diff --git a/.docs/conf.py b/.docs/conf.py index ed223f9..d8e2225 100644 --- a/.docs/conf.py +++ b/.docs/conf.py @@ -51,9 +51,9 @@ # -- Options to show "Edit on GitHub" button --------------------------------- html_context = { - "display_github": True, # Integrate GitHub - "github_user": "aneoconsulting", # Username - "github_repo": "ArmoniK.Api", # Repo name - "github_version": "main", # Version - "conf_py_path": "/.docs/", # Path in the checkout to the docs root + "display_github": True, + "github_user": "aneoconsulting", + "github_repo": "PymoniK", + "github_version": "main", + "conf_py_path": "/.docs/", } diff --git a/.docs/development/contribution.md b/.docs/development/contribution.md index fe2e140..a1ef5dc 100644 --- a/.docs/development/contribution.md +++ b/.docs/development/contribution.md @@ -1,21 +1,104 @@ # Contributing -This doesn't differ from our other projects ([Read ArmoniK.CLI's CONTRIBUTING.md](https://github.com/aneoconsulting/ArmoniK.CLI/blob/main/CONTRIBUTING.md)). - -## Open Issues - -Here's a non-exhaustive list of things that are outright/partially missing from PymoniK that we'd like to see added in: - -- **Unit tests, end-to-end tests** : This project doesn't have any testing associated to it, 0% code coverage. We'd like to change this. -- **More sophisticated examples** : We'd like to add even more examples and tutorials of common use cases that use ArmoniK under the hood. -- **PymoniK logger** : The session created/closed/cancelled prints and the different errors should be logged instead of being printed. -- **Local PymoniK** : Once of the big advantages of the way things have been coded is being able to switch from/to a remote/local context by just removing the `invoke` methods. This could be done even better, by adding a `local=True` flag to PymoniK that makes it so invokes are executed as regular function calls that run locally. The challenge is mainly handling `Pymonik.put`s and the map_invoke. -- **Rename `ResultHandle` to `ObjectHandle`** : The naming of `ResultHandle` was choosen because invocations return results, but as it turns out, these results are also then served as inputs. It'd be better for naming (especially since you can put things onto ArmoniK) to rename `ResultHandle` to the more generic `ObjectHandle`. -- **Cleaner sub-tasking** : Subtasking requires the user to pass in a `delegate=True` flag to invokes, this isn't particularly clean or nice. There must be a better way of doing it. -- **Tasks returning multiple results**: As of right now, a task can only return a single result object (even when you return a tuple). There should be support for cases where you'd like to return multiple results from a task and not have them be grouped up into one (multiple smaller objects being passed onto multiple tasks). We think this change would be easier to implement once sub-tasking is in place, since it also involves analyzing what the user will return. There should be pre-execution tests to check if the user is returning a different number of results in different branches of the task and that should result in a failure (invalid task). -- **`results.as_completed`** : For more sophisticated and better time-to-execute, we'd like to implement a method for `MultiResultHandle` that allows the user to for instance loop through a `MultiResultHandle` and have the code execute as the result is done/retrieved. Moreover, as a side-effect, having this feature would allow for the usage of `tqdm` to create progress bars which is also really nice. -- **Intermediate Objects** : Support being able to create/download ArmoniK Objects (Intermediate results) within tasks. The download part would require `GetDirectData` in the Python API. -- **Remote to local error propagation** : Supply an additional "error name" to created tasks, when a result creation fails, we create this result, locally we can retrieve the remote stack trace using `my_result.error()` if we try-except, either that are we enrich the local result failure grpc exception with the remote one. -- **Test JIT-ing** : jitting tasks using namba/taichi if they're pure for additional performance. (Should just test if it'd work as intended..) -- **More sophisticated result deserialization**: Right now it's just first-level depickling, it'd be nice to be able to pass in a dict that has ResultHandle values for example and be able to dynamically fetch those but that'd add a lot of complexity to data dependencies that'd need to be handled. There has to be a nice way of doing this and it's worth exploring -- **PymoniK Visualizer**: With the current implementation of invoke/map_invoke, we can make it so you can surround your PymoniK context with a Grapher context that dynamically builds up a visualization of your task graph that you can then save and look at/analyze/vizualize later. +Thank you for considering a contribution. PymoniK is a small, +opinionated SDK and we'd like to keep it that way — but there's +plenty to do, and outside perspectives are welcome. + +For repo-wide conventions, see ANEO's +[contribution guidelines on ArmoniK.CLI](https://github.com/aneoconsulting/ArmoniK.CLI/blob/main/CONTRIBUTING.md); +PymoniK follows the same shape. + +## Before you start + +- For non-trivial changes, open an issue to discuss the approach + before writing code. Saves rounds. + +## What we'd particularly like help with + +Roughly in priority order — none of these are claimed; happy to +talk through any of them. + +### Production-readiness + +- **`pymonik image build` CLI.** Read the user's `pyproject.toml`, + render a Dockerfile from a template, run `docker build`, print the + tag. The most-asked-for missing piece. +- **OIDC / bearer-token credentials.** The `Credentials` class only + handles mTLS. Ship a `BearerCredentials(token_provider=...)` that + plugs into the gRPC channel via the metadata callback. +- **`pymonik doctor` CLI.** Hits the cluster's `Versions` and + `Health` services, reports cluster compatibility with the local + pymonik version, surfaces obvious misconfigs (events stream + reachable, partition exists, AKCONFIG sane). + +### Observability + +- **Wire OTel into ArmoniK upstream** so cluster-side spans (polling + agent, control plane, agent sidecar) chain into PymoniK's. The W3C + trace context already propagates; the cluster just needs to emit + spans under it. This is an upstream contribution, not a PymoniK PR. +- **Notebook display hooks.** `Future.__repr__` rendering a progress + bar in Jupyter; `FutureList` showing a per-task heatmap. + +### Performance + +- **Cross-session blob reuse.** Use ArmoniK's `Results.import_data` to + bind a fresh result id to data already uploaded in a prior session, + driven by a local `~/.cache/pymonik/blobs/` hash-to-opaque-id index. +- **Per-session warm subprocess** for `deps=` + `isolate=True`. Spawn + one child Python at session-open time, feed tasks through a Unix + socket. Drops per-task startup from ~500 ms to ~1 ms while + preserving subprocess isolation. + +### Async core + +- Drop the threading completion loop, port the events stream to + `grpc.aio`, unify `Future` on a single `anyio.Event`. The threading + bridge in `Future` is the largest piece of accidental complexity in + the codebase. + +### Tests and examples + +- More end-to-end tests against a `testcontainers`-spun ArmoniK. +- `hypothesis` round-trip tests for the envelope and refs. + +### Documentation + +- This doc tree is a fresh rewrite; it'll have rough edges. Reading + through any of the guides and filing an issue (or PR draft) for + things that confused you is genuinely valuable. +- Worked examples for fault tolerance — show what happens when a + worker pod gets evicted mid-task, and how `retries=` covers it. + +## Small but appreciated + +- Typo fixes, dead-link fixes, doctest fixes. +- Ruff / pyright cleanups in `_internal/`. +- More attribute coverage on existing OTel spans (anything that'd + help filter in a UI). + +## What we generally don't want + +- **Major API churn** — the public surface (decorator, `.spawn`, + `.map`, `Future`, `Blob`, `Materialize`) is mostly settled. + Suggest naming changes via an issue first; don't rename in a PR. +- **Adding heavy dependencies** to the runtime. The current set is + deliberate. New deps need a strong "it would be much worse to + hand-roll this" argument. +- **Hiding ArmoniK from users.** PymoniK wraps the lower-level + `armonik` package; it doesn't try to replace it. Anywhere you'd + reach for `armonik.client.*`, that should still work alongside the + PymoniK API. + +## How to ship a PR (when you have permissions) + +The maintainer's workflow: + +1. Local commits, no force pushes to shared branches. +2. Run the fast suite: `uv run pytest -m "not slow"`. +3. Run pyright: `uv run basedpyright src/pymonik`. +4. Run ruff: `uv run ruff check && uv run ruff format`. +5. If you touched `worker.py` or anything in `_internal/`, rebuild + the worker image and restart the partition; rerun a + representative example end-to-end against the rebuilt cluster. +6. Open the PR with a clear description. diff --git a/.docs/development/development.md b/.docs/development/development.md index 9bebe9d..f728d41 100644 --- a/.docs/development/development.md +++ b/.docs/development/development.md @@ -1,31 +1,118 @@ # Developing PymoniK -We'll be covering some basic information to help you in working on and developing PymoniK +This page covers what you need to know to work *on* PymoniK itself — +not on top of it. -## Requirements +## Prerequisites -We're using `uv` throughout the project, so please make sure that you have it installed. You can refer to their [official `uv` installation guide](https://docs.astral.sh/uv/getting-started/installation/) +- Python 3.11. PymoniK pins to 3.11 because cloudpickle isn't + cross-minor-compatible with the worker image — tests and + `LocalCluster` need to match what the worker runs. +- [`uv`](https://docs.astral.sh/uv/) for project management. +- Docker, if you'll touch worker images or run integration tests + against a real cluster. -## Test client +## Layout -The test client contains some basic examples of working with PymoniK, PymoniK is installed in editable mode `uv add ../pymonik --editable`, it's useful to just create a python file there for testing and then `uv run`ning it to quickly iterate on PymoniK. Keep in mind that if you make changes that affect how the worker functions (obviously like making a change to the `worker.py` file), you'll have to reload the worker image. You can do this by running the following command: +``` +src/pymonik/ # the package + __init__.py # public API re-exports + client.py # PymonikClient + session.py # Session + completion loops + task.py # @task decorator, Task wrapper + future.py # Future, FutureList + options.py # TaskOpts, merge semantics + envelope.py # wire format (msgspec) + blob.py # Blob, Materialize + worker.py # pymonik-worker entrypoint + worker_session.py # from-inside-a-worker submission + context.py # pymonik.current() / WorkerContext + errors.py # PymonikError hierarchy + composition.py # gather, as_completed + testing/ # LocalCluster + cli/ # pymonik CLI (stub today) + _internal/ # not part of the public API + submit.py # shared submission pipeline + refs.py # FutureRef / BlobRef / MaterializeRef + env_builder.py # uv venv + flock for runtime deps + subprocess_dispatch.py # deps + isolate=True path + task_runner.py # subprocess child entrypoint + exec_cache.py # local result cache + query.py # fluent introspection + info.py # TaskInfo / ResultInfo / ... + channel.py # gRPC channel helpers + _otel.py # OpenTelemetry helper + _logging.py # opt-in structlog setup +examples/ # runnable examples (also CI-gated) +tests/ # pytest suite +.docs/ # this documentation (Sphinx + MyST) +worker-image/ # Dockerfile for the harmonic_snake worker +``` + +The `_internal/` prefix marks code that may change without notice. +Anything re-exported from `pymonik/__init__.py` is part of the public +API and follows semver-ish rules within the alpha. -```bash -kubectl rollout restart deployment/compute-plane-pymonik #(1) -n armonik #(2) +## Running tests + +```sh +uv sync # one-time install +uv run pytest # everything +uv run pytest -m "not slow" # skip slow integration tests (no `uv` install) +uv run pytest tests/test_otel.py -v # one file ``` -1. This should be compute-plane-(NAME OF YOUR PYMONIK PARTITION). -2. If you're deploying locally the namespace is typically armonik, otherwise use the namespace of your kubernetes cluster +The test suite is divided: + +- **Fast tests** (~30) — pure unit tests, no network, no `uv venv` + builds. Run in seconds. These are what CI runs on every push. +- **Slow tests** marked `@pytest.mark.slow` — exercise the runtime + deps path with a real `uv` install. Need `uv` on `PATH`. Skip on + Windows (the subprocess wire is POSIX-only for now). + +The `LocalCluster` exercises the same submission pipeline as the +real client, so most behavioural tests don't need a cluster. Only +tests that depend on cluster-side behaviour (partition scheduling, +events stream over the network) need a live ArmoniK; mark those +`@pytest.mark.e2e` and run them separately when you have a deploy. +## Type checking -## Automation Script (`automation.py`) +```sh +uv run basedpyright src/pymonik +``` -The `automation.py` script at the root of the project should help you realize most of your development tasks, it also auto-installs development dependencies. +The codebase aims for `typeCheckingMode = "standard"`. New code +should be fully annotated; private helpers may skip annotations when +obvious. -For example, if you want to access the documentation offline of if you're working on it (thank you!) then you can use the `serve-docs` command. +A few upstream-typing quirks (anyio's `to_thread.run_sync` overload +resolution, armonik's `Result` field types) produce false positives +in `Session` / `WorkerSession` / `task.py`. These predate the +revamp and aren't from new changes — leave them be unless you're +fixing them upstream. -To see a list of all available commands and their general descriptions, you can run: +## Linting and formatting +```sh +uv run ruff check +uv run ruff format ``` -uv run automation.py --help + +Ruff replaces black + flake8 + isort. Configuration is in +`pyproject.toml` under `[tool.ruff]`. + +## Working against a cluster + +If you're touching code that affects the worker (anything in +`worker.py` or `_internal/`), you'll need to rebuild the image and +restart the partition: + +```sh +docker build -t my-org/harmonic_snake:dev worker-image/ +docker push my-org/harmonic_snake:dev # or load directly into your kind cluster +kubectl rollout restart deployment/compute-plane-pymonik -n armonik ``` + +For client-only changes, just `uv sync` (or run with the editable +install) and re-run your client script — no image rebuild needed. diff --git a/.docs/examples/monte_carlo.md b/.docs/examples/monte_carlo.md index 329d15a..759ba3d 100644 --- a/.docs/examples/monte_carlo.md +++ b/.docs/examples/monte_carlo.md @@ -1,3 +1,104 @@ -# Distributed Monte Carlo for PI Estimation +# Monte Carlo: estimating π -This page hasn't been written yet, but the example code for it is in `test_client/estimate_pi.py`. +A short example that hits every basic primitive: a worker function, +`map` for fan-out, `spawn` for fan-in. + +The full source lives at `examples/estimate_pi.py`. + +## The idea + +Monte Carlo estimation of π: throw N random points in the unit square; +the fraction inside the unit quarter-circle is approximately π/4. The +more points, the better the estimate. + +It's embarrassingly parallel — every chunk of points is independent, +and the only fan-in is a sum. Perfect for ArmoniK. + +## The code + +```python +import random +from pymonik import PymonikClient, task + + +@task +def count_inside(n: int, seed: int) -> int: + """How many of `n` random points fall inside the unit quarter circle?""" + rng = random.Random(seed) + inside = 0 + for _ in range(n): + x = rng.random() + y = rng.random() + if x * x + y * y <= 1.0: + inside += 1 + return inside + + +@task +def estimate(total_inside: int, total_points: int) -> float: + """Combine the per-shard counts into a single π estimate.""" + return 4.0 * total_inside / total_points + + +@task +def add_all(xs: list[int]) -> int: + return sum(xs) + + +def run(total_points: int = 10_000_000, num_tasks: int = 32) -> float: + points_per_task = total_points // num_tasks + + with PymonikClient() as client: + with client.session(partition="pymonik") as s: + shards = count_inside.starmap( + (points_per_task, i) for i in range(num_tasks) + ) + total_inside = add_all.spawn(shards) + pi = estimate.spawn(total_inside, num_tasks * points_per_task) + return pi.result(timeout=120) + + +if __name__ == "__main__": + print(f"π ≈ {run()}") +``` + +## What's happening + +- `count_inside.starmap(...)` submits 32 tasks in one gRPC call. Each + one runs in parallel on whatever workers ArmoniK schedules. Returns + a `FutureList[int]`. We use `starmap` because we already have arg + tuples; if the per-task args were single values, `count_inside.map(iter)` + would be cleaner. +- `add_all.spawn(shards)` passes the `FutureList` directly. PymoniK + rewrites each upstream future as a data dependency — `add_all` won't + run until every `count_inside` has finished. The client doesn't + block. +- `estimate.spawn(total_inside, ...)` chains again: `estimate` waits + for `add_all` via the same mechanism. +- Only `pi.result(timeout=120)` blocks. By the time the client wakes + up, the entire DAG (32 + 1 + 1 = 34 tasks) has run. + +## Tweaking + +- **More accuracy?** Bigger `total_points`. Each shard is independent, + so increasing the count just makes the leaves heavier. +- **More parallelism?** Bigger `num_tasks`. Each shard is small enough + that the overhead of submission per task starts to matter at a few + hundred — you'll see diminishing returns. +- **Reproducible?** Drop the seed indirection and pass a fixed seed. + The worker uses Python's stdlib `random`, which is deterministic + given a seed. + +## Variations worth trying + +1. Replace `count_inside` with one that uses `numpy.random` for + speed — declare `client.session(deps=["numpy"])` to make numpy + available without rebuilding the image. See + [Runtime environment](../guides/runtime-environment.md). +2. Use the local exec cache to skip re-running shards: + `PymonikClient(cache=True)` + `@task(cache=True)` on + `count_inside`. Re-running the script with the same args returns + instantly on the second invocation. +3. Use `LocalCluster` to run the whole thing in-process for a unit + test — no cluster needed. See + [Local testing](../guides/local-testing.md). diff --git a/.docs/examples/pong_training.md b/.docs/examples/pong_training.md index 5a36c25..5ed3673 100644 --- a/.docs/examples/pong_training.md +++ b/.docs/examples/pong_training.md @@ -1,3 +1,124 @@ -# Distributed Reinforcement Learning +# Distributed reinforcement learning (Pong) -This example is still a work in progress. +Reinforcement learning workloads benefit from cluster compute when +the bottleneck is **rollout collection** — running game episodes to +gather experience for a learner. Each episode is independent; you +can run dozens or hundreds in parallel on cluster workers, then ship +trajectories back to a single learner. + +This example uses ArmoniK to parallelise the rollout step of training +a pong-playing agent. The full source lives at +`examples/pong_training.py`. + +## Approach + +1. The client owns the model weights and the optimizer. +2. Each iteration: fan out N rollout tasks. Each task plays a few + episodes against the current weights and returns trajectories. +3. The client collects trajectories, runs a gradient step, updates + the weights, repeats. + +The worker doesn't need to know about training — it only needs to +play. The client handles all gradient bookkeeping. + +## Sketch + +```python +from pymonik import PymonikClient, task +import pymonik.blob as blob + +@task +def collect_rollouts(weights_blob, episodes_per_task: int, seed: int) -> dict: + """Play `episodes_per_task` episodes; return trajectories. + + `weights_blob` is a Blob[bytes] — the worker downloads the bytes + on its own and hands `bytes` to this function. We deserialise + locally on the worker. + """ + weights = deserialise_weights(weights_blob) # bytes -> model state + agent = build_agent(weights) + env = build_pong_env(seed=seed) + + trajectories = [] + for _ in range(episodes_per_task): + obs = env.reset() + episode = [] + done = False + while not done: + action = agent.act(obs) + next_obs, reward, done, _ = env.step(action) + episode.append((obs, action, reward)) + obs = next_obs + trajectories.append(episode) + + return {"trajectories": trajectories, "seed": seed} + + +def train(num_iterations: int = 1000, num_workers: int = 32, episodes_per_task: int = 8): + weights = init_weights() + optimizer = build_optimizer(weights) + + with PymonikClient() as client: + with client.session( + partition="pymonik", + deps=["torch", "gymnasium[atari]", "ale-py"], + ) as s: + for it in range(num_iterations): + # Upload current weights once; every rollout task + # references the same blob_id. + wblob = blob.upload(serialise_weights(weights)) + + rollouts = collect_rollouts.starmap( + (wblob, episodes_per_task, it * num_workers + w) + for w in range(num_workers) + ) + + results = rollouts.results(timeout=600) + trajectories = [t for r in results for t in r["trajectories"]] + + loss = update_weights(weights, optimizer, trajectories) + if it % 10 == 0: + print(f"iter {it}: loss={loss:.4f}, episodes={len(trajectories)}") +``` + +## Why blobs matter here + +The model weights are passed to every rollout task. With +`num_workers=32`, naïvely passing `weights_blob` inline would mean 32 +cloudpickled copies of the weights per training iteration. With +`blob.upload(...)` it's one upload + 32 references. PymoniK's +auto-spill kicks in for any arg over 256 KiB, so even without the +explicit `blob.upload` you'd get this dedup behaviour — but doing it +explicitly is clearer in code and skips the cloudpickle round-trip on +every iteration. + +## Why runtime deps matter here + +Running PyTorch and Gymnasium on the worker doesn't require baking +those into the image — declare them on the session: + +```python +client.session( + partition="pymonik", + deps=["torch", "gymnasium[atari]", "ale-py"], +) +``` + +The first task on a fresh worker pod pays the install (multi-minute +for torch); subsequent tasks reuse the venv. For production runs, +bake torch into the image instead — see +[Worker images](../guides/worker-images.md). + +## Things to add + +- **GPU partition.** Move rollouts to a GPU partition for faster + inference: `client.session(partition=["cpu", "gpu"], deps=["torch"])` + + `collect_rollouts.with_options(partition="gpu")`. See + [Multi-partition routing](../guides/multi-partition.md). +- **Async streaming.** Use `as_completed(rollouts)` instead of + `.results()` to start training on the first rollouts that arrive + rather than waiting for the slowest. See + [Async](../guides/async.md). +- **Retries.** Rollouts that crash on a flaky pod aren't fatal — + `@task(retries=3)` on `collect_rollouts` recovers transparently. + See [Retries](../guides/retries.md). diff --git a/.docs/examples/pricing_workflows.md b/.docs/examples/pricing_workflows.md index 2b7f236..217fe34 100644 --- a/.docs/examples/pricing_workflows.md +++ b/.docs/examples/pricing_workflows.md @@ -1,484 +1,215 @@ -# Pricing Workflows with Pymonik +# Pricing workflows -This document provides a detailed overview of two prevalent pricing workflows that are constructed using [**ArmoniK**](https://github.com/aneoconsulting/ArmoniK), a hybrid framework designed to simplify the development of distributed applications, particularly in high-performance computing (HPC) and High Throughput environments, and -[**PymoniK**](https://github.com/aneoconsulting/PymoniK), a Python framework designed to interface with ArmoniK seamlessly. +Two patterns for financial pricing on ArmoniK, side by side: a simple +synchronous workflow that prices one instrument at a time, and a +fan-out/fan-in workflow that decomposes complex pricing into a DAG of +subtasks. Both are runnable; the source lives at +`examples/pricing_workflows.py`. -## Pricing Workflow Scenarios +## Why pricing on ArmoniK -The document specifically covers two distinct scenarios in order to illustrate the usage and versatility of these tools: +Pricing financial instruments is a natural fit for ArmoniK: -* **Scenario 1** – This scenario focuses on a straightforward, synchronous pricing workflow. It is designed to illustrate how pricing tasks can be executed in a linear fashion, with each step waiting for the previous one to complete before moving forward. +- **Embarrassingly parallel** for portfolio pricing (every instrument + is independent). +- **Heterogeneous task sizes** (a vanilla call is microseconds; a + Bermudan swaption with Monte Carlo paths is seconds-to-minutes). +- **Bursty demand** (end-of-day risk runs, intraday what-if analysis, + one-off scenario shocks). -* **Scenario 2** – In contrast, this scenario showcases a more advanced and scalable pricing workflow that incorporates subtasking and dynamic task graphs. This approach allows for a more flexible execution of pricing tasks, enabling the system to adapt to varying demands by breaking down tasks into smaller subtasks that can run concurrently. +PymoniK handles the scheduling; you write the pricing math. -### Breakdown of Each Scenario +## Scenario 1 — Single-instrument synchronous pricing -For both scenarios, the document systematically explains several key aspects: +The simplest possible workflow: one task in, one price out. -* **End-to-End Workflow** – We provide an overview of the entire workflow from the user's point of view. This includes each interaction the user has with the system, detailing how the pricing requests are submitted and processed sequentially or concurrently. - -* **ArmoniK’s Internal Functions** – An in-depth look at what happens behind the scenes within ArmoniK during the workflow execution. This includes explanations of how tasks are scheduled, resources are managed, and results are compiled to ensure efficient processing. - -* **Implementation in Python** – A practical guide on how to implement each workflow scenario using PymoniK within a Python environment. This section offers code snippets and explanations to help users understand how to leverage the library effectively for their pricing needs. - -### Prerequisites for Understanding the Examples - -The examples provided throughout the document are based on the following assumptions : - -* **ArmoniK Cluster** – It is expected that the reader has access to an operational ArmoniK cluster, which serves as the foundation for running distributed pricing tasks. - -* **Python-Based Worker Image** – The document assumes that users are working with a worker image that is compatible with Python, ensuring that the examples can be executed without compatibility issues. - -* **Basic Knowledge of PymoniK** – A fundamental understanding of PymoniK’s tasks, how to invoke them, and how to handle results is assumed. This prior knowledge will enable readers to follow the implementation steps more effectively. - - ---- - -## Scenario 1 – Simple Pricing Workflow - -This scenario illustrates the a basic and direct pricing interaction pattern supported by ArmoniK. -It is intentionally minimal and synchronous, so it is straighforward to understand and ideal as a -starting point for new users. In this workflow: - -- The user prepares all required input data locally, including: - - Market data (e.g., spot prices, rates, volatilities) - - Product or instrument definitions (e.g., an option with a notional) - - Any additional pricing parameters -- The user submits a single pricing task to ArmoniK. -- The user waits synchronously for the task to complete. -- Once execution finishes, the pricing result is retrieved and returned to the user. - -There is no task decomposition, no fan-out/fan-in logic, and no dependency management. -The entire pricing request is handled as one atomic unit of work. - -This interaction model is particularly well suited for: - -- Pricing a single financial instrument -- Lightweight or fast pricing models -- Interactive workflows (e.g., notebooks, scripts, UI-driven tools) -- Situations where immediate feedback is required - ---- - -### Workflow Diagram - -The diagram below shows the logical flow of data and control between the user, ArmoniK, and the pricing function. - - -```mermaid -graph TD - %% Define other nodes - id1["Portfolio"] - id2["Market Data"] - id3((("user"))) - id4["pricer"] - id5["Final Portfolio Price"] - - %% Define connections - id1 --> id4 - id2 --> id4 - id3 -- "1: User provides input data" --> id1 - id3 -- "2: User submits the task" --> id4 - id4 --> id5 - id3 -- "3: User waits for the result availability and downloads the result" --> id5 - -``` - -Hence: - -- The user is responsible for assembling the input data (portfolio definition and market data). -- The pricing task consumes these inputs and performs the computation. -- ArmoniK executes the task remotely and produces a final portfolio price. -- The user explicitly waits for the computation to complete and then retrieves the result. - -### What ArmoniK Does - -From ArmoniK’s point of view, this scenario follows a simple and linear execution path: - -- Receives a single pricing task submission from the client -- Places the task in the scheduler queue -- Assigns the task to an available worker node -- Executes the pricing function in a distributed environment -- Persists the final result in the distributed result store -- Makes the result available for download by the client - -Because the task is fully self-contained: - -- No dynamic task graph is created -- No subtasks are generated -- No dependency resolution is required -- No intermediate results are exposed - -### Example Code - -The following example demonstrates how to define and invoke a simple pricing task using PymoniK. - -```{code-block} python -:linenos: -from pymonik import Pymonik, task +```python +from pymonik import PymonikClient, task -# A simple pricing task @task -def price_vanilla(option, market_data): - # Simplified pricing logic - return option["notional"] * market_data["spot"] * 0.01 - -# User workflow -with Pymonik(endpoint="localhost:5001"): - option = {"notional": 1_000_000} - market_data = {"spot": 105.0} - - result = price_vanilla.invoke(option, market_data).wait().get() - print("Price:", result) +def price_option(market_data: dict, option_def: dict, params: dict) -> dict: + """Black-Scholes price + greeks for a vanilla European option.""" + spot = market_data["spot"] + rate = market_data["rate"] + sigma = market_data["volatility"] + strike = option_def["strike"] + tte = option_def["time_to_expiry"] + is_call = option_def["type"] == "call" + + # ... Black-Scholes math ... + price = compute_bs_price(spot, strike, tte, rate, sigma, is_call) + delta = compute_bs_delta(...) + vega = compute_bs_vega(...) + + return { + "price": price, + "greeks": {"delta": delta, "vega": vega}, + "valuation_id": params["valuation_id"], + } + + +def run() -> dict: + market = {"spot": 100.0, "rate": 0.05, "volatility": 0.2} + option = {"type": "call", "strike": 105.0, "time_to_expiry": 0.5, + "notional": 1_000_000} + params = {"valuation_id": "single-001"} + + with PymonikClient() as client: + with client.session(partition="pymonik") as s: + result = price_option.spawn(market, option, params).result(timeout=30) + return result ``` -#### Step-by-Step Explanation - -##### Task definition: - -* Line 1 imports the ArmoniK client (Pymonik) and the `@task` decorator. -* Line 4 marks `price_vanilla` as a remotely executable ArmoniK task. -* Lines 5–7 define the pricing logic. All required inputs are passed as arguments, making the task fully self-contained and serializable. - -##### User workflow: - -* Line 10 opens a connection to the ArmoniK control plane using a context manager. Here we assume that the cluster is listening on `localhost` at port `5001`. -* Lines 11-12 define the product data and market data locally on the client. -* Line 14 submits the task for execution, waits synchronously for completion, and retrieves the result. -* Line 15 outputs the final price to the user. - -From the user’s perspective, the call behaves much like a local function call, while ArmoniK transparently handles remote execution and scheduling. - -### Take-away messages - -- One task → one result -- The user explicitly waits for completion -- Minimal orchestration logic -- Suitable for straightforward pricing problems - ---- - -## Scenario 2 – Portfolio Pricing with Subtasking and Monte Carlo - -In this scenario, the pricing logic itself takes responsibility for orchestrating the computation. Rather than submitting many independent tasks from the client side, the user submits a single, high-level portfolio pricer task. At runtime, this task dynamically constructs and executes a task graph based on the actual contents of the portfolio. -As a result, orchestration is shifted away from the user and into the pricing logic. This allows complex workflows to be defined within the computation itself, instead of being fixed at submission time. - -At a high level, this means that: - -- The user interacts with ArmoniK only once. -- All task creation, fan-out, and aggregation are handled transparently inside the portfolio pricer. -- The user ultimately receives a single, aggregated portfolio value. - -Because of this design, the execution model is particularly well suited for: - -- Large portfolios containing many instruments. -- Heterogeneous product mixes, including both vanilla and exotic products. -- Computationally intensive models, such as: - - Monte Carlo simulations - - Scenario-based pricing - - Path-dependent products -- Situations in which: - - The computation structure cannot be determined upfront - - Task creation depends on intermediate results - ---- - -### Workflow Diagram - -The diagram below expresses the workflow for this second scenario: - -```mermaid - -flowchart TB - subgraph Inputs [" "] - style Inputs fill:#ffffff, stroke:none; - direction TB - id1["Portfolio"] - id2["Market Data"] - id3["Pricer"] - - id1 --> id3 - id2 --> id3 - end - - id4(("User")) - id4 -- "1: User provides input data" --> id1 - id4 -- "2: User submits the task" --> id3 - - subgraph Subtasks [" "] - style Subtasks fill:#ffffff, stroke:ffffff; - direction TB - v["Vanilla"] - x["Complex Product 1"] - y["Complex Product 2"] - z["Complex Product 3"] +Three things worth noting: - xc1[" "] - xc2[" "] - xc3[" "] +- The function is plain Python — no special imports, no ArmoniK API + surface inside the math. The decorator is the only PymoniK touch. +- All inputs and outputs are JSON-serialisable dicts. PymoniK doesn't + require this — it cloudpickles whatever you pass — but it makes + log/debug output legible. +- One submission, one wait. Suitable for "I want to price this and + see the answer", interactive notebooks, UI-driven tools. - yc1[" "] - yc2[" "] - yc3[" "] +## Scenario 2 — Decomposed pricing with subtasking - zc1[" "] - zc2[" "] - zc3[" "] +For a complex instrument (Bermudan swaption, multi-asset basket, +exotic with path dependencies), you want to: +1. Decompose into independent sub-pricings. +2. Run them in parallel. +3. Aggregate the results. - id2 --> xc1 - id2 --> xc2 - id2 --> xc3 - x --> xc1 - x --> xc2 - x --> xc3 - - id2 --> yc1 - id2 --> yc2 - id2 --> yc3 - y --> yc1 - y --> yc2 - y --> yc3 - - id2 --> zc1 - id2 --> zc2 - id2 --> zc3 - z --> zc1 - z --> zc2 - z --> zc3 - - xd1[" "] - xd2[" "] - xd3[" "] - - xc1 --> xd1 - xc2 --> xd2 - xc3 --> xd3 - - yd1[" "] - yd2[" "] - yd3[" "] - - yc1 --> yd1 - yc2 --> yd2 - yc3 --> yd3 - - zd1[" "] - zd2[" "] - zd3[" "] - - zc1 --> zd1 - zc2 --> zd2 - zc3 --> zd3 - - xa["Aggregate"] - ya["Aggregate"] - za["Aggregate"] - - xd1 --> xa - xd2 --> xa - xd3 --> xa - - yd1 --> ya - yd2 --> ya - yd3 --> ya - - zd1 --> za - zd2 --> za - zd3 --> za - - - xr["Product Price"] - xa --> xr - - yr["Product Price"] - ya --> yr - - zr["Product Price"] - za --> zr - - pa["Aggregate Portfolio"] - - v --> pa - xr --> pa - yr --> pa - zr --> pa - end - - id3 -- "4: The pricer submits a graph for each complex product and the result of all vanilla products" --> Subtasks - - id5["Final Portfolio Price"] - - id4 -- "3: User waits for result availability and downloads the result" --> id5 - pa --> id5 - -``` - -The execution flow proceeds as follows: - -1. The user prepares and provides: - * A portfolio containing multiple financial instruments - * The associated market data required for pricing -2. The user submits one portfolio pricer task to ArmoniK. -3. The portfolio pricer executes and: - * Identifies and prices all vanilla products directly - * Detects complex products requiring advanced models - * Dynamically constructs a computation graph for those products - * Launches Monte Carlo or other heavy computations as subtasks - * Collects and aggregates partial pricing results -4. A final aggregation task computes the total portfolio value. -5. The user retrieves a single portfolio-level result. - -From the client’s point of view, the interaction remains simple and synchronous. The user submits one task and receives one result, even though the internal execution may involve hundreds or thousands of distributed tasks running in parallel. This abstraction is made possible because, within the ArmoniK framework, the portfolio pricer itself can act as a **runtime orchestrator**. - -- It inspects the portfolio composition. - - For vanilla instruments: - - Pricing is performed directly within the main task or via lightweight subtasks. - - For complex instruments: - - A dynamic task graph is built. - - Monte Carlo simulations are split into many independent subtasks. - - Each subtask computes partial statistics (e.g. payoffs, paths). These partial results are progressively collected and combined as the computation advances. - -### What ArmoniK Does - -ArmoniK provides the execution backbone that makes this model possible. In particular, it: - -- Executes the initial portfolio pricer task -- Allows running tasks to submit new tasks dynamically (subtasking) -- Dynamically extends the task graph as new computation paths are discovered -- Tracks and enforces task dependencies to ensure correct execution order -- Manages result propagation so that delegated subtask results are routed back to their parent tasks -- Ensures that the parent task’s result becomes the final aggregated portfolio output - -Despite the complexity of the internal execution, from the user’s perspective this still appears as a single task invocation producing a single result. All orchestration, parallelization, and aggregation are handled transparently by the portfolio pricer and the ArmoniK runtime. - ---- - -## Example Code - -The following example demonstrates how to define and invoke the pricing task explained above using PymoniK. Each code -block is followed by a step-by-step explanation. - -### Supporting Tasks - -```{code-block} python -:linenos: -import numpy as np -from pymonik import task - -@task -def price_vanilla(option, market_data): - return option["notional"] * market_data["spot"] * 0.01 - -@task -def mc_path(product, market_data, seed): - rng = np.random.default_rng(seed) - paths = rng.normal(market_data["spot"], 1.0, size=10_000) - return np.mean(paths) * product["notional"] - -@task -def aggregate_mc_results(results): - return np.mean(results) +```python +from pymonik import PymonikClient, task @task -def aggregate_portfolio(values): - return sum(values) -``` - -- Lines 4–6 handle simple vanilla products directly. -- Lines 8–12 implement a Monte Carlo simulation for complex products. -- Lines 14–20 define aggregation tasks to combine partial results. +def price_leg(market_data: dict, leg_def: dict) -> dict: + """Price one leg of a multi-leg structure.""" + # ... per-leg math ... + return {"leg_id": leg_def["id"], "price": ..., "sensitivities": ...} -### Complex Product Pricing via Subtasking -```{code-block} python -:linenos: @task -def price_complex_product(product, market_data): - # Launch Monte Carlo paths in parallel - mc_results = mc_path.map_invoke([ - (product, market_data, seed) for seed in range(16) - ]) - - # Delegate final product price to aggregation - return aggregate_mc_results.invoke(mc_results, delegate=True) -``` - -- Line 4–6: map_invoke runs multiple Monte Carlo simulations in parallel, each with a different seed. -- Line 9: delegate=True tells ArmoniK that the aggregation result will replace the parent task’s result, making orchestration seamless. +def aggregate_legs(leg_results: list[dict], structure: dict) -> dict: + """Combine leg prices into the structure's overall price.""" + total_price = sum(r["price"] * structure["weights"][r["leg_id"]] for r in leg_results) + return { + "structure_id": structure["id"], + "price": total_price, + "leg_breakdown": leg_results, + } -### Portfolio Pricer (Entry Point) -```{code-block} python -:linenos: @task -def price_portfolio(portfolio, market_data): - vanilla_products = [p for p in portfolio if p["type"] == "vanilla"] - complex_products = [p for p in portfolio if p["type"] == "complex"] - - vanilla_prices = price_vanilla.map_invoke([ - (p, market_data) for p in vanilla_products - ]) - - complex_prices = price_complex_product.map_invoke([ - (p, market_data) for p in complex_products - ]) - - all_prices = vanilla_prices + complex_prices - - # Delegate final portfolio result - return aggregate_portfolio.invoke(all_prices, delegate=True) +def shock_structure(structure: dict, market: dict, shock: dict) -> dict: + """Re-price under a market shock (delta-hedge analysis, scenario PnL).""" + shocked_market = apply_shock(market, shock) + leg_futures = price_leg.starmap( + (shocked_market, leg) for leg in structure["legs"] + ) + return aggregate_legs.spawn(list(leg_futures), structure).result() + # Note: this .result() is OK because shock_structure runs on the + # client side (it's not @task-decorated to be remote, but if it + # were, you'd return the future from aggregate_legs.spawn instead + # — see the sub-task / delegate pattern below). + + +def run_structure(structure: dict, market: dict) -> dict: + with PymonikClient() as client: + with client.session(partition="pymonik") as s: + leg_futures = price_leg.starmap( + (market, leg) for leg in structure["legs"] + ) + result = aggregate_legs.spawn(leg_futures, structure) + return result.result(timeout=300) ``` -- Lines 3–4: Separate the portfolio into vanilla and complex products. -- Lines 6–8: Price all vanilla products in parallel using map_invoke. -- Lines 10–12: Price complex products using the dynamic Monte Carlo subtasks. -- Line 14: Combine all results. -- Line 17: Delegate the final portfolio sum, ensuring the portfolio pricer task returns the total value transparently. - -### User Code +What this gives you: -```{code-block} python -:linenos: -from pymonik import Pymonik +- `price_leg.starmap(...)` submits every leg in one gRPC call. + ArmoniK schedules them in parallel. We use `starmap` because each + leg's args come from a different shape (the market is a constant, + the leg varies); `map(*iterables)` would need parallel iterables of + the same length. +- `aggregate_legs.spawn(leg_futures, structure)` doesn't wait for the + legs on the client. PymoniK rewrites each upstream future as a + `data_dependency`, ArmoniK schedules `aggregate_legs` to run *after* + all legs complete, and `aggregate_legs` receives the resolved list + of leg results. +- Only the terminal `.result(timeout=300)` blocks the client. -portfolio = [ - {"type": "vanilla", "notional": 1_000_000}, - {"type": "complex", "notional": 500_000}, -] +## Scenario 2b — Sub-tasking from inside a worker -market_data = {"spot": 100.0} +When the decomposition itself depends on the input (e.g. a basket +whose number of underlyings isn't known until you parse the trade), +let the worker do it: -with Pymonik(endpoint="localhost:5001", environment={"pip": ["numpy"]}): - result = price_portfolio.invoke(portfolio, market_data).wait().get() - print("Portfolio value:", result) +```python +@task +def price_basket(market: dict, basket: dict) -> dict: + """Price every underlying, then aggregate. The fan-out happens + inside the worker because we don't know `basket["underlyings"]` + until we've parsed the trade. + """ + underlying_results = price_leg.starmap( + (market, u) for u in basket["underlyings"] + ) + # Hand off our expected output to the aggregator subtask. + return aggregate_legs.spawn( + underlying_results, basket, _delegate=True + ) ``` -- Lines 3–6: Define a sample portfolio with one vanilla and one complex product. -- Line 8: Provide market data needed for pricing. -- Line 10: Initialize a PymoniK client session connecting to the ArmoniK server. -- Line 11: Invoke the portfolio pricer task. .wait().get() blocks until the result is ready. -- Line 12: Print the final portfolio value. - ---- - -### Take-away messages - -- Dynamic Orchestration: Complex product pricing uses subtasks that are launched dynamically, depending on the portfolo content. -- Delegation: Aggregation tasks replace parent task results seamlessly, giving the user the appearance of a single synhronous invocation. -- Parallelism: map_invoke allows Monte Carlo paths and vanilla product pricing to execute in parallel, maximizing resorce utilization. -- User Simplicity: From the user perspective, only one task is submitted, and a single portfolio-level result is returned. +The `_delegate=True` flag retargets the child task's +`expected_output_ids` to the parent's. ArmoniK delivers the child's +result to the original awaiter; the parent's own return value is +ignored (it returns a `Future`, which the SDK detects as a tail +call). -## Summary +This is the right pattern for divide-and-conquer when the shape of +the recursion depends on the input. -* **Scenario 1** demonstrates a straightforward request–response pricing model -* **Scenario 2** leverages ArmoniK’s dynamic task graph and subtasking capabilities to scale complex portfolio pricing -* Pymonik allows both workflows to be expressed naturally as Python code while keeping the user-facing API simple +## Putting it together: a portfolio run -From a user’s perspective, both scenarios boil down to: +End-of-day risk runs typically price thousands of instruments under +hundreds of market scenarios. PymoniK turns this into: ```python -result = some_pricer.invoke(...).wait().get() +def run_portfolio(portfolio: list[dict], scenarios: list[dict], + market: dict) -> list[dict]: + with PymonikClient() as client: + with client.session(partition="pymonik") as s: + jobs = [] + for scenario in scenarios: + shocked = apply_shock(market, scenario) + for trade in portfolio: + jobs.append((shocked, trade, {"scenario_id": scenario["id"]})) + + futures = price_option.starmap(jobs) + return futures.results(timeout=3600) ``` -The difference lies entirely in how much intelligence and orchestration is embedded inside the tasks themselves. +For 10,000 trades × 100 scenarios = 1M tasks, this submits in +batched RPCs (32-task batches by default), schedules across however +many workers ArmoniK can give you, and streams results back via the +events stream. Add `events=True` (default) and you'll see results +resolve as they arrive, not all at once at the end. + +## Operational notes + +- **Use blobs for shared market data.** If `market` is large + (full vol surfaces, yield curves), upload it once with + `blob.upload(market)` and pass the blob handle to every task + instead of inlining. See + [Blobs and Materialize](../guides/blobs-and-materialize.md). +- **Use multi-partition for heterogeneous compute.** Vanilla pricing + on a CPU partition; Monte Carlo on a GPU partition. + `client.session(partition=["cpu", "gpu"])` plus + `monte_carlo.with_options(partition="gpu")` per task. +- **Trace it.** Pricing pipelines are exactly the workload OTel was + designed for: a long DAG, occasional slow tasks, fan-in steps that + depend on a thousand upstreams. See + [Observability](../guides/observability.md). +- **Cache for what-if iterations.** Re-running the same pricing with + the same inputs is wasteful. `PymonikClient(cache=True)` and + `@task(cache=True)` short-circuit identical re-submissions. diff --git a/.docs/examples/raytracing.md b/.docs/examples/raytracing.md index 69d5a99..af3a978 100644 --- a/.docs/examples/raytracing.md +++ b/.docs/examples/raytracing.md @@ -1,198 +1,146 @@ -# Distributed Raytracing in Python +# Distributed raytracing -Raytracing is a technique for generating realistic images by tracing the path of light as pixels in an image plane and simulating its effects with virtual objects. While capable of producing stunning visuals, raytracing is computationally intensive, as each pixel's color often requires complex calculations and many rays to be traced, especially for effects like reflections, refractions, and soft shadows. This makes it an excellent candidate for distributed computing. +Raytracing renders an image by tracing the path of light rays through +a 3D scene. It's computationally expensive — every pixel needs many +ray–object intersection tests — and embarrassingly parallel: every +pixel can be computed independently. -PymoniK allows us to distribute the raytracing workload across multiple workers in an ArmoniK cluster, significantly speeding up the rendering process. We can divide the image into smaller sections (tiles) and assign each tile to a PymoniK task for parallel processing. +This example renders an image by splitting it into horizontal tiles +and rendering each tile as a separate task on the cluster. -## Core Concept: Tiled Rendering +The full source lives at `examples/raytracing.py`. -The basic idea is to break down the image rendering into smaller, independent tasks. Each task will be responsible for rendering a horizontal strip (or tile) of the final image. +## Approach -1. **Scene Definition**: We define a 3D scene containing objects (like spheres), light sources, and a camera. -2. **Task Distribution**: The main client script divides the image into a number of horizontal tiles. -3. **PymoniK Task**: A PymoniK task, `render_tile_task`, is defined. Each instance of this task receives: - * The y-coordinates defining the start and end row of the tile it needs to render. - * The overall image dimensions. - * The `Camera` object. - * The `Scene` object (containing all objects and lights). -4. **Pixel Calculation**: Within each task, for every pixel in its assigned tile: - * A ray is generated from the camera through the pixel. - * This ray is traced into the scene to find the closest intersecting object. - * The color of the pixel is determined based on the object's material, lighting, and other effects. -5. **Result Aggregation**: The client script collects the pixel data (colors) for each rendered tile from the completed PymoniK tasks. -6. **Image Assembly**: Finally, the client assembles these tiles into the complete image. +1. Define the scene (objects, lights, camera) on the client. +2. Slice the image into tiles (one task per tile). +3. Submit all tiles via `Task.starmap` (we have arg tuples already). +4. Each task computes its tile's pixels. +5. Client collects the tile results and assembles the final image. ## Prerequisites -Ensure you have the necessary Python packages installed: - -```bash +```sh uv add pymonik Pillow ``` -* `pymonik`: For interacting with the ArmoniK cluster. -* `Pillow`: For image manipulation (creating and saving the final image) on the client side. - -## PymoniK Implementation - -Let's look at the key parts of the Python script. The full script also includes helper classes for 3D vectors (`Vec3`), rays (`Ray`), materials (`Material`), spheres (`Sphere`), lights (`PointLight`), the scene (`Scene`), and the camera (`Camera`). PymoniK will handle the serialization of these custom objects automatically when they are passed as arguments to tasks. - -### The Raytracing Task +`Pillow` is for image assembly on the client. -The core of the distributed computation is the `render_tile_task` function, decorated with `@task` to make it a PymoniK task. +## The render task -```py -import math +```python from pymonik import task -# Assuming Vec3, Ray, Camera, Scene, trace_ray_for_pixel_color etc. are defined elsewhere @task -def render_tile_task(tile_y_start, tile_y_end, image_width, image_height, camera_obj, scene_obj): #(1) - """ - Renders a horizontal strip (tile) of the image. - Accepts scene and camera objects directly. +def render_tile( + y_start: int, + y_end: int, + image_width: int, + image_height: int, + camera, + scene, +) -> tuple[int, list[tuple[int, int, int]]]: + """Render rows [y_start, y_end) of the final image. + + `camera` and `scene` are arbitrary Python objects. PymoniK + cloudpickles them; the worker reconstructs and uses them as if + they were local. Both must be importable on the worker (the + classes — not the instances — need to live in modules the worker + can import). """ - tile_pixel_data = [] # List of (r,g,b) tuples for this tile - - for y in range(tile_y_start, tile_y_end): #(2) - # print(f"Worker rendering row {y}/{image_height}") # Optional: progress within worker + pixels: list[tuple[int, int, int]] = [] + for y in range(y_start, y_end): for x in range(image_width): - # u, v are normalized screen coordinates (0 to 1) - # Add 0.5 for sampling at the center of the pixel - u_norm = (x + 0.5) / image_width - v_norm = (image_height - 1 - y + 0.5) / image_height # Flipped y for typical image coords - - # Use the get_ray method from the camera object - # PymoniK handles sending the camera_obj to the worker - ray = camera_obj.get_ray(u_norm, v_norm) #(3) - - # trace_ray_for_pixel_color uses scene_obj (also sent by PymoniK) - pixel_color_vec3 = trace_ray_for_pixel_color(ray, scene_obj) #(4) - tile_pixel_data.append(pixel_color_vec3.to_color()) - - # Return the starting row index and the pixel data for this tile - return tile_y_start, tile_pixel_data #(5) + u = (x + 0.5) / image_width + v = (image_height - 1 - y + 0.5) / image_height + ray = camera.get_ray(u, v) + color = trace_ray(ray, scene) + pixels.append(color.to_rgb()) + return y_start, pixels ``` -1. It receives `camera_obj` and `scene_obj` directly. PymoniK takes care of serializing these objects and sending them to the worker where the task executes. -2. It iterates over its assigned rows (`tile_y_start` to `tile_y_end`) and columns (`image_width`). -3. For each pixel, it uses `camera_obj.get_ray()` to generate a ray. -4. `trace_ray_for_pixel_color(ray, scene_obj)` performs the actual raytracing logic for that single ray against the scene. -5. It returns the starting y-coordinate of the tile and a list of pixel colors for that tile. - -### Main Client Logic +The function returns the tile's start row and its pixel list — the +client uses the start row to know where each tile goes in the final +image. -The client-side script sets up the scene, camera, connects to PymoniK, divides the work, submits tasks, and then assembles the final image. +## Submitting and assembling ```python -# --- Main Application Logic (Client Side) --- -# Assuming imports for os, Pymonik, Image, math, and helper classes like Vec3, Scene, Camera etc. +import math, os +from PIL import Image +from pymonik import PymonikClient + + +def main() -> None: + image_width = 600 + image_height = 400 + num_tasks = int(os.getenv("NUM_RAYTRACING_TASKS", "16")) + rows_per = math.ceil(image_height / num_tasks) + + camera = build_camera(image_width, image_height) + scene = build_scene() + + task_args = [] + for i in range(num_tasks): + y_start = i * rows_per + y_end = min((i + 1) * rows_per, image_height) + if y_start >= y_end: + continue + task_args.append( + (y_start, y_end, image_width, image_height, camera, scene) + ) + + with PymonikClient() as client: + with client.session(partition="pymonik") as s: + tiles = render_tile.starmap(task_args) + results = tiles.results(timeout=600) + + image = Image.new("RGB", (image_width, image_height)) + flat: list[tuple[int, int, int]] = [(255, 0, 255)] * (image_width * image_height) + for y_start, pixels in results: + for j, color in enumerate(pixels): + x = j % image_width + y = y_start + j // image_width + flat[y * image_width + x] = color + image.putdata(flat) + image.save("raytraced.png") + if __name__ == "__main__": - # Image dimensions - img_width = 600 - img_height = 400 - - # Scene setup (materials, objects, lights) - # ... (material_red, material_green, etc.) - # ... (scene_objects list of Sphere instances) - # ... (scene_lights list of PointLight instances) - # ... (scene_background Vec3) - # main_scene = Scene(scene_objects, scene_lights, scene_background) - - # Camera setup - # ... (look_from, look_at, vup, vfov, aspect_ratio) - # main_camera = Camera(look_from, look_at, vup, vfov, aspect_ratio) - - - with Pymonik(endpoint="localhost:5001"): - print("Successfully connected to Pymonik.") - - # Divide work: each task renders a few rows - num_tasks = int(os.getenv("NUM_RAYTRACING_TASKS", "10")) - num_tasks = max(1, min(num_tasks, img_height)) - rows_per_task = math.ceil(img_height / num_tasks) - - task_args_list = [] - for i in range(num_tasks): - y_start = i * rows_per_task - y_end = min((i + 1) * rows_per_task, img_height) - if y_start >= y_end: - continue - # Pass main_camera and main_scene objects directly - task_args_list.append( - (y_start, y_end, img_width, img_height, main_camera, main_scene) # main_camera and main_scene are actual objects - ) - - if not task_args_list: - print("Error: No tasks generated. Check image dimensions and num_tasks.") - exit() - - print(f"Submitting {len(task_args_list)} raytracing tasks to Pymonik...") - # map_invoke submits all tasks in parallel - results_handle = render_tile_task.map_invoke(task_args_list) - - print("Waiting for tasks to complete...") - results_handle.wait() # Wait for all tasks to finish - print("All tasks completed. Fetching results...") - - # Prepare to assemble the image - final_image = Image.new("RGB", (img_width, img_height)) - - rendered_tiles_data = {} - for task_idx in range(len(task_args_list)): - try: - # results_handle is a MultiResultHandle, access individual results by index - tile_y_start, tile_pixel_data = results_handle[task_idx].get() - rendered_tiles_data[tile_y_start] = tile_pixel_data - # ... (logging) - except Exception as e: - # ... (error handling) - - print("Assembling final image...") - # ... (Logic to iterate through rendered_tiles_data and put pixels into final_image) - # Example: - # flat_pixel_list = [ (255,0,255) ] * (img_width * img_height) # Default for missing - # for y_start_key, tile_pixels in rendered_tiles_data.items(): - # # ... (detailed logic to place tile_pixels into flat_pixel_list) - # final_image.putdata(flat_pixel_list) - - - output_filename = "pymonik_raytraced_image.png" - try: - final_image.save(output_filename) - print(f"Image saved as {output_filename}") - except Exception as e: - print(f"Error saving or showing image: {e}") - - print("Raytracing finished.") + main() ``` - -- **Objects as Arguments**: The `main_camera` and `main_scene` objects are passed directly when building `task_args_list`. PymoniK handles their distribution. -- **`map_invoke`**: This PymoniK method is used to submit multiple instances of `render_tile_task` with different arguments (different tiles) in parallel. It returns a `MultiResultHandle`. -- **Result Handling**: `results_handle.wait()` blocks until all tasks are complete. Then, individual results are fetched using `results_handle[task_idx].get()`. -- **Image Assembly**: The `Pillow` library is used to create a new image and populate it with the pixel data returned by the tasks. - -!!! tip "Full Code" - The snippets above are excerpts. You would need the full definitions for classes like `Vec3`, `Sphere`, `Camera`, `Scene`, and the `trace_ray_for_pixel_color` function for a complete runnable example. Please check `examples/raytracing` for that - -## Running the Example - -1. Save the complete Python script (including helper classes and the PymoniK logic shown above) as a `.py` file (e.g., `distributed_raytracer.py`). -2. Ensure your ArmoniK cluster is running and accessible. -3. Either set the AKCONFIG to your ArmoniK deployment or supply the endpoint (If you've deployed ArmoniK locally it should be "localhost:5001") -4. You can also control the number of tasks (and thus, tiles) using the `NUM_RAYTRACING_TASKS` environment variable. - ```bash - export NUM_RAYTRACING_TASKS=20 # Example: divide into 20 tiles - ``` -5. Run the script: - ```bash - python distributed_raytracer.py - ``` - -## Expected Output - -After the script completes, you should find an image file named `pymonik_raytraced_image.png` (or similar, based on your output filename) in the same directory. This image will be the result of the distributed raytracing computation. - -The console output will show connection messages, task submission progress, and final assembly messages. +## Things worth noting + +**Auto-spill on the camera and scene.** The camera and scene objects +are passed to every task. If their cloudpickled size exceeds 256 KiB +(the default `spill_threshold`), PymoniK uploads them once as blobs +and the workers download them — instead of inlining them into every +task's payload. You don't have to do anything for this to happen; +see [Blobs and Materialize](../guides/blobs-and-materialize.md) for +the explicit form. + +**Multi-file project.** `Vec3`, `Camera`, `Scene`, `trace_ray`, etc. +are typically in their own modules in a real raytracing project. +Workers need to be able to import those modules. Either: + +- Bake the project into your worker image (production), or +- Call `cloudpickle.register_pickle_by_value(my_raytracer_pkg)` at + the client's entrypoint (fast iteration). See + [Important considerations](../important-considerations.md#cloudpickle-and-multi-file-projects). + +**Trace it.** This is a great workload to point at Jaeger — `map` +produces a fan of `pymonik.task.run` spans, each tagged with its +tile's row range. See [Observability](../guides/observability.md). + +## Tuning + +- **`NUM_RAYTRACING_TASKS`** controls fan-out. Too few and individual + tasks dominate wall time; too many and submission overhead does. + For a 600×400 image, 16-32 is a reasonable starting point. +- **`spill_threshold`** on the client controls when scene/camera get + blobbed. Default 256 KiB is fine for medium scenes. +- **Scene complexity scales the per-pixel cost**. More spheres, more + lights, deeper recursion = longer tasks. ArmoniK's per-task + scheduling latency disappears into the noise once tasks take more + than a second. diff --git a/.docs/getting-started.md b/.docs/getting-started.md index 9390004..ddd0988 100644 --- a/.docs/getting-started.md +++ b/.docs/getting-started.md @@ -1,291 +1,263 @@ # Getting started -- We'll be using `uv` as our Python project manager, so if you haven't installed it yet, follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/). +This page walks you from "no PymoniK installed" to "I just ran a +distributed computation on my cluster." -- Moreover, we assume that you have a partition in your Armonik cluster with the name `pymonik` and that is using a pymonik worker image. You can either build your own or use the pre-prepared one for Python 3.10.12 (Python 3.10 should work just as well): `ineedzesleep/harmonic_snake`. If your partition is named differently then you need to pass in the name of the partition to Pymonik. +## Prerequisites -```py -pymonik = Pymonik(partition="my_pymonik_partition") -``` +- An ArmoniK cluster you can talk to (any deploy: local quick-deploy, + k8s, etc.) with a partition that runs a PymoniK-compatible worker + image (see [Worker images](guides/worker-images.md)). The default + partition name we'll use throughout is `pymonik`. +- [`uv`](https://docs.astral.sh/uv/) for project management. +- Python **3.11** locally — must match the worker's Python version + (cloudpickle isn't cross-minor-compatible; see + [Important considerations](important-considerations.md)). -## Creating a new project +## Install ```sh mkdir hello_pymonik && cd hello_pymonik -uv init --python 3.10.12 -``` - -Install the pymonik package - -``` +uv init --python 3.11 uv add pymonik ``` -## Pymonik basics - -It's best to learn by example +## Point at your cluster -```py -from pymonik import Pymonik, task +PymoniK reads cluster connection info from a YAML file the same way +the ArmoniK CLI does. Three precedence levels: -@task #(1) -def add(a, b): #(2) - return a + b +1. **Pass `endpoint=` (and `credentials=`) explicitly** to + `PymonikClient(...)`. +2. **Pass `akconfig=/path/to/armonik-cli.yaml`** — loads the endpoint + and (optionally) the CA / client cert / key for mTLS. +3. **Set `AKCONFIG=/path/to/armonik-cli.yaml`** in the environment. + The constructor picks it up automatically. -with Pymonik(endpoint="localhost:5001"): #(3) - result = add.invoke("Hello", " World!").wait().get() - print(result) -``` - -1. To create a task for ArmoniK, it suffices to use the `@task` decorator, if you're working with other decorators, make sure that `@task` is applied last -2. You can define your Python function as usual, you don't need to worry about anything. Just be aware that this is to be executed remotely. -3. Tasks invoked inside a Pymonik context will be executed in the Armonik cluster associated with said context. - - -This simple example basically creates an ArmoniK task to add up two strings. To execute a Python function on a remote cluster you `.invoke` it, passing in the same parameters you would've if you called it locally. To execute a Python function locally you can call it like you would have usually. - -At the end of a PymoniK call you get a handle to the execution result. I can at any point block my execution to `wait` for a certain result or continue executing my code. To wait for a result, you call the `wait` method. Note however that the wait method does not return the actual value of the result. To do so, you'll need to call `get`. `get` will download the execution result to your local machine. - -In essence, you'll be working with result handles throughout your Pymonik programs. Let's add a new task to our previous code to multiply two numbers. - -```py -@task -def add(a, b): - return a + b - -@task -def multiply(a,b): - return a*b - -with Pymonik(endpoint="localhost:5001"): - intermediate_result = add.invoke(2, 3) #(1) - final_result = multiply.invoke(intermediate_result, 5).wait().get() - print(final_result) -``` - -1. We don't need to block and wait for the result of the addition, we can just pass the ResultHandle to multiply task and it'll execute when this result is ready. - - -Running this code should yield the result `25`. This isn't really exciting though, let's try running some operations on arrays, we'll be using numpy. - -First, let's install numpy locally: +Most users export `AKCONFIG` once and never pass anything to +`PymonikClient()` again: ```sh -uv add numpy +export AKCONFIG=/path/to/generated/armonik-cli.yaml ``` -```py -from pymonik import Pymonik, task -import numpy as np +## Hello, world -@task -def add(a, b): - return a + b +```python +from pymonik import PymonikClient, task @task -def multiply(a,b): - return a*b - -with Pymonik(endpoint="localhost:5001", environment={"pip":["numpy"]}): #(1) - intermediate_result = add.invoke(np.array([1,2,3]), np.array([2,1,0])) - final_result = multiply.invoke(intermediate_result, 2).wait().get() - print(final_result) -``` - -1. To specify a global execution environment for all your tasks, you just need to add an environment argument - -You can pass in a list of packages to install in your remote workers via the environment argument to the Pymonik client. If you'd like to specify specific versions, you just need to pass in a tuple with `(package_name, version_specifier)`. This example should yield the result `[6 6 6]`. - -The environment argument also supports the "env_variables" key which allows you to pass in a dictionary with environment variables to set on the worker. - -Now let's take this another notch and invoke multiple tasks in parallel. To do this, we use `map_invoke`. - -```py -from pymonik import Pymonik, task -import numpy as np - -@task -def add(a, b): +def add(a: int, b: int) -> int: return a + b -@task -def sum_arrays(arrays): - return np.sum(arrays) - -with Pymonik(endpoint="localhost:5001", environment={"pip":["numpy"]}): - intermediate_result = add.map_invoke( #(1) - [(np.random.randint(0,10, size=(3,3)) ,np.random.randint(0,10, size=(3,3))) for _ in range(10)] - ) - final_result = sum_arrays.invoke(intermediate_result).wait().get() - print(final_result) +# Local call still works exactly like a plain function. No client, +# no session, no setup — useful for tests, sanity checks, and +# debugging the function in isolation. +assert add(2, 3) == 5 +with PymonikClient() as client: # reads $AKCONFIG + with client.session(partition="pymonik") as s: + result = add.spawn(2, 3).result(timeout=60) + print(result) # 5 ``` -1. We pass in a list of the arguments that we'd like to execute remotely. You should provide the same arguments that a function expects in the form of a tuple. A `map_invoke` returns a MultiResultHandle. - -`map_invoke` allows you to submit multiple tasks to your cluster in parallel and returns a MultiResultHandle. If you `wait` here then it'll wait until all results ready. +What's happening: -A MultiResultHandle behaves like a list, this allows you to selectively wait for results or split the computation. +- `@task` wraps `add` so it gets two callable shapes: + - `add(2, 3)` — plain Python call, runs in the current process, + returns `5`. The decorator doesn't change this. + - `add.spawn(2, 3)` — remote submission, returns a `Future[int]`. +- `PymonikClient()` opens a gRPC channel; the `with` block closes it. +- `client.session(partition="pymonik")` creates an ArmoniK session + bound to that partition. Tasks submitted inside its `with` block run + there. +- `Future.result(timeout=60)` blocks until the result is delivered or + the timeout fires. `await fut` is the async equivalent — see + [Async](guides/async.md). -I can for instance write: +## Many tasks at once -```py -from pymonik import Pymonik, task -import numpy as np +```python +@task +def square(x: int) -> int: + return x * x -@task -def add(a, b): +@task +def add(a: int, b: int) -> int: return a + b -@task -def sum_arrays(arrays): - return np.sum(arrays) - -with Pymonik(endpoint="localhost:5001", environment={"pip":["numpy"]}): - intermediate_result = add.map_invoke( - [(np.random.randint(0,10, size=(3,3)) ,np.random.randint(0,10, size=(3,3))) for _ in range(10)] - ) - partial_final_1 = sum_arrays.invoke(intermediate_result[:5]) #(1) - partial_final_2 = sum_arrays(intermediate_result[5:].wait().get()) #(2) - final_result = add(partial_final_1.wait().get(), partial_final_2) #(3) - print(final_result) +with PymonikClient() as client: + with client.session(partition="pymonik") as s: + squares = square.map(range(32)) + print(squares.results(timeout=120)) # [0, 1, 4, 9, ...] + sums = add.map(range(32), range(1, 33)) + print(sums.results(timeout=120)) # [1, 3, 5, 7, ...] ``` -1. I execute this part of the computation remotely using half of the results of the previous step, without retrieving them. -2. I retrieve the other half of the results and run this part of the computation locally. -3. I retrieve partial_final_1 and add the results locally - -## Anonymous Tasks +`Task.map(*iterables)` mirrors Python's built-in `map`: it zips its +iterables (stopping at the shortest) and submits one task per zipped +tuple. The whole batch is one gRPC round-trip, not N. Returns a +`FutureList[T]`; use `.results(timeout=...)` to wait for everything in +submission order. -You can create tasks from lambda functions by directly creating a Task object, for instance: +If you already have your arguments as tuples and just want each tuple +unpacked positionally, use `starmap` (the equivalent of +`itertools.starmap`): -```py -add_task = Task(lambda a, b: a+b, func_name="add") -add_task.map_invoke([(1,2), (1,3)]) +```python +pairs = [(1, 2), (3, 4), (5, 6)] +sums = add.starmap(pairs) ``` -!!! warning - - Please note that when creating anonymous tasks using lambda functions, it's imperative that you give it a name on your own. +`map` is the right shape for "I have parallel lists of inputs"; +`starmap` is the right shape for "I already have a list of arg +tuples." Pick the one that doesn't make you build the wrong shape. +## Composing tasks (pipelining) -Anonymous tasks are particularly useful when you want to "armonikize" code from other Python packages. For instance: - -```py -numpy_sum = Task(np.sum) -``` -## Subtasking +A `Future` (and a `FutureList`) is a first-class argument. Pass it to +another `.spawn()` and the SDK rewrites it as an ArmoniK data +dependency: -Subtasking is an ArmoniK feature that allows you to dynamically change your task graph based mid-task execution. This is best illustrated with the following scenario. Say we've implemented a vector addition task as follows: - -```py +```python @task -def vec_add(a: np.ndarray, b: np.ndarray) -> np.ndarray: +def add(a: int, b: int) -> int: return a + b -``` - -One way to enhance this operation through subtasking is by making it so the `vec_add` task checks the size of the vectors to add. If the size is bigger than a certain threshold, then we can split the input into two parts and then invoke the same task for these smaller inputs. - -Here is a sample code for this (check `test_client/adaptive_vector_addition.py` for the full example) - -```py -VECTOR_SIZE_THRESHOLD = 512 - -@task -def aggregate_results(result_1, result_2) -> np.ndarray: - return np.concatenate([result_1, result_2]) @task -def vec_add(a: np.ndarray, b: np.ndarray) -> np.ndarray: - if a.size > VECTOR_SIZE_THRESHOLD: - mid_point = a.size // 2 #(1)! - a1, a2 = np.split(a, [mid_point]) - b1, b2 = np.split(b, [mid_point]) - - result_handle1 = vec_add.invoke(a1, b1) #(2)! - result_handle2 = vec_add.invoke(a2, b2) - - return aggregate_results.invoke(result_handle1, result_handle2, delegate=True) #(3)! - else: - return a + b #(4)! +def total(xs: list[int]) -> int: + return sum(xs) + +with PymonikClient() as client: + with client.session(partition="pymonik") as s: + partials = add.map(range(32), range(1, 33)) # FutureList[int] + final = total.spawn(partials) # pass it directly + print(final.result(timeout=120)) ``` -1. Split vectors into two chunks, ideally you'd have a chunk size and you'd split into miltiple chunks. A half-way split was chosen here to highlight subtasking. -2. We invoke the `vec_add` task for each split. Note that you cannot wait or get there task results here. As Armonik's design philosophy is centered around workers being ephemeral. We can still invoke other tasks that make use of these results. -3. Aggregate the results using the aggregate_results task. We directly use the result handles from the sub-tasks that were invoked. The `delegate=True` basically tells ArmoniK that the result of vec_add will be the result of this task. So on the user side of things, you don't get a ResultHandle wrapped around a ResultHandle. This is sub-tasking. The result of the parent task will be set to the result of the delegated sub-task. -4. If the vectors are of adequate size, we sum them up as usual and return their value. +Two important properties: -There is another much simpler example of subtasking in `test_client/subtasking.py` +- The client never blocks on `partials`. ArmoniK schedules `total` to + run after every upstream `add` completes — the dependency edge is + enough. +- `total` receives the *resolved* values as a plain `list[int]`. The + SDK rewrites each upstream `Future` as a data dependency, downloads + the result bytes on the worker, and substitutes them before calling + your function. From the worker's perspective, it's just a list. +You can mix `Future` arguments with plain values freely — anything +that isn't a `Future` / `FutureList` / `Blob` / `Materialize` rides +inline (or auto-spills if it's too big; see +[Blobs and Materialize](guides/blobs-and-materialize.md)). -## Context +## Errors -Sometimes, you might want to log messages from your tasks. To do that, you can add a `PymonikContext` to your task : +Tasks that raise on the worker surface as `TaskFailed` on the client: -```py -@task(require_context=True) -def my_task(ctx): - ctx.logger.info("This is an info log") - ctx.logger.error("This is an error log", my_keyword="hello from pymonik") #(1)! +```python +from pymonik import TaskFailed + +@task +def maybe_blow_up(x: int) -> int: + if x < 0: + raise ValueError("x must be non-negative") + return x * 2 + +with PymonikClient() as client: + with client.session(partition="pymonik") as s: + try: + maybe_blow_up.spawn(-1).result(timeout=30) + except TaskFailed as e: + print(e.task_id, e.worker_message) ``` -1. I can add additional information/metadata to display on Seq. +Other typed exceptions in `pymonik`: -You can also use the context to access the current environment, and in particular to install packages in that specific task. Although this isn't recommended as it will just cause environment contamination. It's preferred to have a single environment that you define from your PymoniK client. +- `TaskCancelled` — task or session was cancelled. +- `TaskTimeout` — `.result(timeout=...)` exceeded. +- `NotInSessionError` — you called `.spawn()` outside a session block. +- `PymonikError` — base class; everything above derives from it. +Catch them with `try/except`, or `try/except*` if you're using +`gather()` / `as_completed()` and want to fan in error groups (see +[Async](guides/async.md)). -The context also gives you direct access to the task handler for that worker, if you ever feel the need to do more advanced work with the low level Python API for ArmoniK. (Not recommended) -## Storing objects in the cluster and reusing them +## Per-task options -You might happen into scenarios where you'd like to store an object in your ArmoniK cluster and reuse it throughout. For that, you can `put` it into the ArmoniK cluster. +`@task` accepts task-level overrides: -```py -from pymonik import ResultHandle +```python +from datetime import timedelta -with Pymonik() as pymonik: - df = pd.read_csv("some_data.csv") #(1)! - df_handle: ResultHandle[pd.Dataframe] = pymonik.put(df) #(2)! - some_operation.invoke(df_handle) #(3)! - some_other_operation.invoke(df_handle) +@task(retries=3, partition="gpu", timeout=timedelta(minutes=5), priority=10) +def render(scene: bytes, frame: int) -> bytes: + ... ``` -1. Dataframe is read locally. -2. Dataframe is uploaded to the ArmoniK cluster, you get back a reusable handle that points to this remote object. -3. Invoke multiple operations on the ArmoniK cluster that reuse the same dataframe. - -This is really useful for larger objects because it minimizes transfer time to the cluster, moreover, you might be able to benefit from worker level caching whenever it's implemented. +The same options are settable per call via `.with_options(...)` +(returns a new bound task — never mutates the decorated function): +```python +fast_lane = render.with_options(partition="gpu-a100", priority=20) +fast_lane.spawn(scene, 0).result() +``` -!!! warning - - You're not required to do this for every object that you're dealing with, you can just pass everything into your tasks and ArmoniK will take care of everything; `pymonik.put` is just an additional optimization when you're reusing the same object over and over again (same object being passed over to multiple tasks). - If you end up modifying your object after the put then PymoniK will not synchronize these changes over to the workers. It's better to think of the sent objects as constants in that sense to avoid making mistakes. - - -There is also a `put_many` if you want to store multiple objects at the same time. (This is more efficient than looping through a list of objects and `put`-ing them individually). - -You can also give your object a name, this makes it easy to see the objects you're putting when looking through the ArmoniK dashboard or if you want to search for it in the ArmoniK.CLI. -## Connecting to ArmoniK +Merge order at submission time is **session default ← `@task(...)` ← +`.with_options(...)`**. `client.session(default_options=...)` lets you +set session-wide baselines: -If you've deployed ArmoniK on your own, you should've been prompted to run a command for setting the `AKCONFIG` environment variable. +```python +from pymonik import TaskOpts -```sh -export AKCONFIG=... +with client.session( + partition="pymonik", + default_options=TaskOpts(retries=2, timeout=timedelta(seconds=30)), +) as s: + ... ``` -This environment variables points to a config file that contains everything needed to connect to this cluster. If you set it before executing your client, then you can just write `pymonik=Pymonik()` and it'll connect automatically to the exported Armonik cluster. -If you want to connect to multiple Armonik clusters, the invoke methods can accept a pymonik client argument. Which allows you to do something like: +## Sub-tasking -```py -pymonik1 = Pymonik( """""" ) #(1) -pymonik2 = Pymonik( """""" ) #(2) +A task can delegate its output to a child task using `task.tail(...)`: -my_task.invoke(arg1, arg2, pymonik=pymonik1) #(3) -my_task.invoke(arg1, arg2, pymonik=pymonik2) #(4) +```python +@task +def adaptive_add(a: list[int], b: list[int]) -> list[int]: + if len(a) > 1024: + mid = len(a) // 2 + return concat.tail( + adaptive_add.spawn(a[:mid], b[:mid]), + adaptive_add.spawn(a[mid:], b[mid:]), + ) + return [x + y for x, y in zip(a, b)] ``` -1. Specify the connection options and environment configuration for your first cluster. -2. Specify the connection options and environment configuration for your second cluster. -3. This task is invoked in the first cluster. -4. This task is invoked in the second cluster. +`tail()` returns a lazy promise; the framework binds it to the parent's +expected output id and submits the child task with that binding. The +child writes the parent's output directly. Use this for divide-and- +conquer; for fan-out / fan-in, plain `map` + `spawn` is simpler. + +A task can also produce multiple named outputs via `MultiResult` — +downstream tasks then depend on individual fields, not the whole +result. See the [Sub-tasking and multi-output](guides/sub-tasking-and-multi-output.md) +guide. + +## What's next + +You now know enough to ship simple workloads. The guides cover +specific topics: + +- [Runtime environment](guides/runtime-environment.md) — install pip + packages on workers, set environment variables. +- [Blobs and Materialize](guides/blobs-and-materialize.md) — large + arguments, file/directory materialisation. +- [Multi-partition routing](guides/multi-partition.md) — mix CPU and + GPU partitions in one session. +- [Retries](guides/retries.md) — cluster-side vs client-side. +- [Local testing](guides/local-testing.md) — `LocalCluster` for unit + tests. +- [Observability](guides/observability.md) — OTel + Jaeger, end-to-end. +- [Async](guides/async.md) — `await fut`, structured concurrency. +- [Worker images](guides/worker-images.md) — bake your project into a + worker image for production. diff --git a/.docs/guides/adding-worker-image.md b/.docs/guides/adding-worker-image.md deleted file mode 100644 index 8041f81..0000000 --- a/.docs/guides/adding-worker-image.md +++ /dev/null @@ -1,62 +0,0 @@ -# Adding a worker image - -You can add a new worker image to your ArmoniK cluster by creating a partition, inside your control plane - -```tf - # Partition for the PymoniK worker - pymonik = { - # number of replicas for each deployment of compute plane - replicas = 0 #(1)! - # ArmoniK polling agent - polling_agent = { - limits = { - cpu = "2000m" - memory = "2048Mi" - } - requests = { - cpu = "50m" - memory = "50Mi" - } - } - # ArmoniK workers - worker = [ - { - image = "dockerhubaneo/harmonic_snake" - tag = "python-YOUR_PYTHON_VERSION-PYMONIK_VERSION_TO_USE" #(2)! - limits = { - cpu = "1000m" - memory = "1024Mi" - } - requests = { - cpu = "50m" - memory = "50Mi" - } - } - ] - hpa = { - type = "prometheus" - polling_interval = 15 - cooldown_period = 300 - min_replica_count = 0 - max_replica_count = 5 - behavior = { - restore_to_original_replica_count = true - stabilization_window_seconds = 300 - type = "Percent" - value = 100 - period_seconds = 15 - } - triggers = [ - { - type = "prometheus" - threshold = 2 - }, - ] - } - }, -``` - -1. By default this partition will start with no workers and scale up as needed, you can change this behavior for faster cold starts -2. Don't forget to set the version of python that you're using here, it **must match** the version of python that you're using for your client. The second part of the tag is for the PymoniK package version to use. - -For the list of available docker images tags, please refer to [our repository](https://hub.docker.com/r/dockerhubaneo/harmonic_snake) diff --git a/.docs/guides/async.md b/.docs/guides/async.md new file mode 100644 index 0000000..70820ac --- /dev/null +++ b/.docs/guides/async.md @@ -0,0 +1,155 @@ +# Async + +PymoniK exposes the same surface twice: a sync API for scripts and +notebooks, an async API for asyncio / trio applications. The async +version is built on `anyio`, so the same code runs under either +backend. + +## Sync vs async, side by side + +```python +# Sync +with PymonikClient() as client: + with client.session(partition="pymonik") as s: + result = add.spawn(2, 3).result(timeout=30) + +# Async +async with PymonikClient() as client: + async with client.session_async(partition="pymonik") as s: + result = await add.spawn(2, 3) +``` + +`Future` works in either context: call `result()` to block, `await +fut` to suspend. + +## When to use the async API + +- Your application is already async (FastAPI, an asyncio service, a + trio application). +- You need to interleave PymoniK submissions with other I/O without + blocking. +- You want structured concurrency primitives (`anyio.create_task_group`) + to bound parallelism, propagate cancellation, and fan in errors as + `ExceptionGroup`s. + +If your code is plain sync (a notebook, a CLI, a batch job), the sync +API is simpler and gives you the same performance. + +## Awaiting many futures + +Plain `await` on a `FutureList` is one common pattern: + +```python +async with client.session_async(partition="pymonik") as s: + futures = add.map(range(32), range(1, 33)) + results = await futures.results_async(timeout=60) +``` + +Or stream completions as they arrive (order: ready first): + +```python +from pymonik import as_completed + +async with client.session_async(partition="pymonik") as s: + futures = work.map(many_args) + async for done in as_completed(futures): + value = await done + # process as it lands +``` + +Or collect with structured fan-in errors: + +```python +from pymonik import gather, TaskFailed + +async with client.session_async(partition="pymonik") as s: + futures = work.map(many_args) + try: + results = await gather(futures) + except* TaskFailed as eg: + for failed in eg.exceptions: + log.warning("retry candidate", task=failed.task_id) +``` + +The `try/except*` syntax (PEP 654) lets you catch one exception type +out of an `ExceptionGroup` while letting others propagate — the right +shape for "tell me about all the failures, then re-raise the rest." + +## Submission off the event loop + +`.spawn()` and `.map()` stay sync from async code — they're a few +gRPC calls, returning a `Future` is fast. If your event loop is +sensitive to even small blocks, use `.spawn_async()` / `.map_async()` +which offload the submission RPCs to a worker thread: + +```python +async with client.session_async(partition="pymonik") as s: + fut = await heavy_work.spawn_async(big_arg) + result = await fut +``` + +For typical workloads the difference is invisible — submission +latency is dominated by network round-trip, not local CPU. + +## Both backends: asyncio and trio + +`async with PymonikClient()` works on either: + +```python +# asyncio +import asyncio +asyncio.run(my_pipeline()) + +# trio +import trio +trio.run(my_pipeline) +``` + +Internally PymoniK's completion machinery currently uses an asyncio +loop (the trio backend bridges through `anyio`'s blocking portal). +The user-facing primitives are anyio.Event and anyio.create_task_group, +so user code reads the same on both. + +## Cancellation + +A cancel scope around a `await fut` propagates to the cluster: + +```python +import anyio + +async with anyio.create_task_group() as tg: + fut = work.spawn(...) + with anyio.move_on_after(30): # 30s deadline + result = await fut + if not fut.done: + # The cancel scope exited; PymoniK has already issued + # CancelTasks on the cluster. The future is resolved with + # TaskCancelled. + ... +``` + +(Today the +client-side cancel works; the cluster-side teardown is best-effort.) + +## When you're sync but the rest of your app is async + +If you're embedding PymoniK in a long-running asyncio service: + +```python +# Use the async API directly on the service's loop: +async def my_handler(): + async with PymonikClient() as client: + async with client.session_async(...) as s: + return await work.spawn(...) +``` + +Don't open `with PymonikClient()` (sync) on a thread inside an +asyncio service — that spins up a second asyncio loop in a portal +thread. It works, it's just slower than using the async API +directly. + +## Mixing async and sync across processes + +The wire format is the same. A sync client can submit work whose +results an async client awaits in another process — they share a +session id and a result id is the only handle you need. diff --git a/.docs/guides/blobs-and-materialize.md b/.docs/guides/blobs-and-materialize.md new file mode 100644 index 0000000..b7bd318 --- /dev/null +++ b/.docs/guides/blobs-and-materialize.md @@ -0,0 +1,138 @@ +# Blobs and Materialize + +A `Blob[T]` is a typed handle to bytes that live in ArmoniK's object +store. The handle is what your client passes around; on the worker, +the function receives the resolved value (or a path, for +`Materialize`). + +Three shapes, one mental model: + +- `blob.upload(obj)` — cloudpickles a Python object and uploads. + Worker function receives the object. +- `blob.upload(Path("file"))` — uploads raw file bytes. Worker + function receives `bytes`. +- `blob.materialize(...)` — uploads bytes (or a zipped directory) and + asks the worker to write them to a path on disk. Worker function + receives a `pathlib.Path`. + +## Why use blobs explicitly + +Most arguments don't need this — PymoniK auto-spills any cloudpickled +arg above the threshold (default 256 KiB) to a blob during +submission, transparently. You only reach for `blob.upload` when you +want to **share the same bytes across many tasks** and skip +re-uploading: + +```python +import pymonik.blob as blob + +with client.session(partition="pymonik") as s: + weights = blob.upload(Path("model.bin")) # uploaded once + for shard in shards: + infer.spawn(weights, shard) # all tasks share the blob_id +``` + +The session's blob cache deduplicates by SHA-256 of the bytes — even +without the explicit `blob.upload`, two `.spawn()` calls passing the +same large value would dedupe at auto-spill time. + +## Files: bytes on the worker + +```python +import pymonik.blob as blob +from pymonik import Blob, task +from pathlib import Path + +@task +def parse_config(data: bytes) -> dict: + import tomllib + return tomllib.loads(data.decode()) + +with client.session(partition="pymonik") as s: + cfg = blob.upload(Path("config.toml")) # Blob[bytes] + parse_config.spawn(cfg).result() +``` + +`blob.upload(Path)` reads file bytes verbatim. The function receives +`bytes`; what you do with them is your call. + +## Materialize: write a file at a path on the worker + +When a library you call needs a file path on disk, `materialize` +puts the bytes there: + +```python +from pymonik import task +import pymonik.blob as blob +from pathlib import Path + +@task +def run_with_config(config_path: Path) -> str: + return Path(config_path).read_text() + +with client.session(partition="pymonik") as s: + cfg = blob.materialize(Path("./local.toml"), at="/etc/app.toml") + run_with_config.spawn(cfg).result() +``` + +The worker writes the bytes to `/etc/app.toml` *before* the task +runs, then passes `pathlib.Path("/etc/app.toml")` as the argument. +Parent directories are created if they don't exist. + +## Materialize a whole directory + +`blob.materialize(dir_path, at=...)` detects a directory and zips it +client-side: + +```python +import pymonik.blob as blob + +@task +def use_assets(assets_dir: Path) -> list[str]: + return [p.name for p in assets_dir.rglob("*") if p.is_file()] + +with client.session(partition="pymonik") as s: + assets = blob.materialize(Path("./assets"), at="/opt/assets") + files = use_assets.spawn(assets).result() +``` + +The zip happens client-side with deterministic ordering (sorted +`rglob`) so two uploads of the same directory contents produce +identical bytes — and identical `result_id`s, so the session's blob +cache deduplicates them. On the worker, the bytes are unpacked into +`/opt/assets` and the task receives a `Path` to it. + +Limits to be aware of: + +- The zip is held in memory client-side. Multi-GB assets will hurt; + consider baking those into the worker image instead. +- File permissions inside the zip are normalised by Python's + `zipfile`. If you need executable bits or symlinks, materialise + individual files yourself and set `chmod` inside the task. +- Existing files at the target path are overwritten by `extractall`. + +## Auto-spill: when arguments get too big + +You don't have to call `blob.upload` for arguments to flow through +the object store. PymoniK cloudpickles each top-level positional / +keyword argument and uploads anything above `spill_threshold` +(default 256 KiB) automatically: + +```python +PymonikClient(spill_threshold=64 * 1024) # spill arg > 64 KiB +``` + +The function receives the deserialised object exactly as if it had +been passed inline. Sub-elements aren't introspected — a list with +a million ints spills as one blob, not a million. + +Auto-spill has the same dedup behaviour: two tasks receiving the +same big value share one blob upload. + +## Cross-session blob reuse (planned) + +Within a session, identical bytes upload once. Across sessions, each +session re-uploads. A planned mechanism using ArmoniK's +`Results.import_data` will let a fresh result id bind to data already +in the object store from a prior session. Until that lands: re-upload, +or bake large static assets into the worker image. diff --git a/.docs/guides/custom-worker.md b/.docs/guides/custom-worker.md index 5724c08..4de8d5c 100644 --- a/.docs/guides/custom-worker.md +++ b/.docs/guides/custom-worker.md @@ -1,21 +1,114 @@ -# Creating your own PymoniK worker +# Custom worker entrypoints -It's pretty simple to create your own ArmoniK worker, you can start by modifying the pymonik_worker image. In terms of code you just need to call the `run_pymonik_worker` method. +The `pymonik-worker` console script is enough for almost everyone: +it runs the dispatch loop, decodes task envelopes, calls your +`@task` functions, and ships results back. But sometimes you want to +do something *before* PymoniK takes over the process — emit a metric, +configure logging, monkey-patch a library, set up a tracer. -```py -from pymonik import run_pymonik_worker +## Wrapping the dispatcher -run_pymonik_worker() +Write a tiny Python entrypoint that calls `pymonik.worker.run()` +yourself: + +```python +# my_worker.py +import logging +import os + +import pymonik +from pymonik.worker import run + + +def main() -> None: + pymonik.enable_logging(level=os.getenv("LOG_LEVEL", "INFO")) + logging.getLogger("my_app").setLevel(logging.DEBUG) + + # Any one-time setup the worker process needs. + _configure_internal_metrics() + _patch_third_party_lib() + + run() # blocks until ArmoniK tears the pod down + + +if __name__ == "__main__": + main() +``` + +Then point your image's `ENTRYPOINT` at it: + +```dockerfile +COPY --chown=armonikuser:armonikuser my_worker.py /app/ +ENTRYPOINT ["python", "/app/my_worker.py"] ``` -But other than that, you're free to do anything in the worker. You can create your own ArmoniK worker image by using the `pymonik_worker`'s as a starting point. The most important part is properly configuring the armonikuser: +`pymonik.worker.run()` does exactly what the `pymonik-worker` +console script does. It: + +1. Calls `pymonik.enable_logging(level=$PYMONIK_WORKER_LOG_LEVEL)`. +2. Sets up OTel if `OTEL_*` env vars are present. +3. Patches the upstream worker class to route the gRPC context to + `WorkerContext.cancel_if_requested()`. +4. Hands control to the upstream `armonik_worker()` framework, which + serves tasks until the pod is killed. + +## Inside a task: WorkerContext + +User code running inside a `@task` function can reach a worker-side +context via `pymonik.current()`: -```Dockerfile -RUN groupadd --gid 5000 armonikuser && \ - useradd --home-dir /home/armonikuser --create-home --uid 5000 --gid 5000 --shell /bin/sh --skel /dev/null armonikuser && \ - mkdir /cache && \ - chown armonikuser: /cache && \ - chown -R armonikuser: /app +```python +import pymonik +from pymonik import task -USER armonikuser +@task +def long_running(x: int) -> int: + ctx = pymonik.current() + + ctx.log.info("starting", input=x, attempt=ctx.attempt) + + for i in range(x): + ctx.cancel_if_requested() # raises TaskCancelled if cluster cancelled + # ... work ... + + return x * 2 ``` + +`WorkerContext` exposes: + +- `task_id`, `session_id` — for logs and external IDs. +- `attempt` — 1 for the original submission, 2+ for retries. Useful + for idempotency-aware code that needs to know "this is a re-run." +- `log` — a `structlog`-style logger pre-bound with `task_id` / + `session_id`. +- `cancel_if_requested()` — polls whether the gRPC server context is + still active. Raises `TaskCancelled` if not. + +## Don't override the dispatcher + +The temptation is to write your own task processor — read the +envelope, call user code, ship the result. **Don't.** PymoniK's +dispatcher handles a lot of the wire format that's easy to get +wrong: + +- msgspec envelope decoding with version checks. +- cloudpickle minor-version validation. +- `Future` / `Blob` / `Materialize` argument resolution. +- `data_dependencies` substitution. +- Sub-task delegate handling. +- Subprocess vs splice routing for `deps=`. +- OTel context extraction and span wrapping. +- Cooperative cancellation observation. + +If you want to extend the worker, do it *around* `run()` (logging, +metrics, OTel) — not in place of it. + +## Image hygiene + +Whatever your entrypoint, the image still needs: + +- A non-root `armonikuser` (uid 5000) owning `/app` and `/cache`. +- `pymonik` importable in the runtime venv. +- The Python minor matching the client's. + +See [Worker images](worker-images.md) for the full Dockerfile. diff --git a/.docs/guides/local-testing.md b/.docs/guides/local-testing.md new file mode 100644 index 0000000..f5eb50d --- /dev/null +++ b/.docs/guides/local-testing.md @@ -0,0 +1,128 @@ +# Local testing + +`LocalCluster` is a drop-in replacement for `PymonikClient` that runs +tasks in an in-process thread pool. The same `@task` definitions, the +same `.spawn()` / `.map()` / blob upload / `Future` API — no gRPC, no +cluster, no network. + +It's the right tool for unit tests, fast iteration on task logic, and +CI lanes that exercise PymoniK without depending on a deployment. + +## Quick start + +```python +from pymonik import task +from pymonik.testing import LocalCluster + +@task +def add(a: int, b: int) -> int: + return a + b + +def test_add(): + with LocalCluster() as client: + with client.session() as s: + assert add.spawn(2, 3).result(timeout=5) == 5 +``` + +`LocalCluster()` opens a thread pool (default 16 threads); each +`.spawn()` enqueues the task; the pool runs it. + +## What's exercised vs. what isn't + +`LocalCluster` runs the **same submission pipeline** the real client +uses. Specifically: + +- Arguments are walked for `Future` / `Blob` / `Materialize` and + rewritten into wire refs (`extract_deps`). +- Auto-spill kicks in for oversize args. +- The envelope is encoded with msgspec. +- A worker thread decodes the envelope, resolves data dependencies + from a session-local dict, runs the function, and pickles the + result. + +So bugs in envelope encoding, ref resolution, blob auto-spill, or +runtime-deps env management surface here the same way they would on +the cluster. + +What's local-only: + +- No pod scheduling latency, no partition routing, no autoscaling. +- No worker isolation — every task runs in the host process. +- ArmoniK's cluster-side `max_retries` (infra-failure retries) isn't + emulated. Client-side `@task(retry_on=...)` retries *do* run end-to- + end through the same code path the real session uses. + +## Async + +```python +import pytest +import anyio + +@pytest.mark.anyio +async def test_add_async(): + async with LocalCluster() as client: + async with client.session_async() as s: + assert await add.spawn(2, 3) == 5 +``` + +Both asyncio and trio backends work via `pytest-anyio`. + +## Runtime deps in tests + +```python +@task(deps=["numpy"]) +def numpy_sum(n: int) -> int: + import numpy as np + return int(np.arange(n).sum()) + +def test_numpy_dep(tmp_path, monkeypatch): + monkeypatch.setenv("PYMONIK_ENVS_ROOT", str(tmp_path / "envs")) + with LocalCluster() as client: + with client.session() as s: + assert numpy_sum.spawn(100).result(timeout=600) == 4950 +``` + +`LocalCluster` exercises the *real* env builder — `uv` runs, a venv +is built at `PYMONIK_ENVS_ROOT//.venv`, the splice (or +subprocess) path runs identically to the worker. The first call pays +the install; subsequent ones reuse the env. + +`PYMONIK_ENVS_ROOT` defaults to `~/.cache/pymonik/envs`; override per +test so runs don't pollute each other. + +## Multi-task pool sizing + +Default is 16 worker threads. A pipeline with deeper in-flight depth +can deadlock — every thread parks waiting for an upstream that needs +a thread to compute. Increase the pool for deep DAGs: + +```python +LocalCluster(max_workers=64) +``` + +(The deadlock is a known limitation of the in-process dispatcher; +a future anyio refactor will drop it.) + +## Cache and OTel + +Both work the same as on the real client: + +```python +LocalCluster(cache=True) # exec cache enabled +# OTEL_EXPORTER_OTLP_ENDPOINT=... exports spans normally +``` + +See [Observability](observability.md) and the exec-cache section in +[Important considerations](../important-considerations.md). + +## When LocalCluster isn't enough + +Two cases need a real cluster: + +- **Cluster-side `max_retries`** — only ArmoniK enforces infra + retries. +- **Things that depend on the polling agent** — partition queue + depth, pod scheduling latency, image pull behaviour. + +For those, use `pytest -m e2e` and a `testcontainers`-spun ArmoniK or +a dev-deploy. Most behavioural tests don't need either. diff --git a/.docs/guides/multi-partition.md b/.docs/guides/multi-partition.md new file mode 100644 index 0000000..799606a --- /dev/null +++ b/.docs/guides/multi-partition.md @@ -0,0 +1,99 @@ +# Multi-partition routing + +A single PymoniK session can submit tasks to more than one partition. +This is how you mix CPU and GPU work in one logical workflow without +opening two clients. + +## Single partition (the default) + +```python +with client.session(partition="pymonik") as s: + add.spawn(2, 3).result() +``` + +The session is bound to one partition. Any task that tries to route +elsewhere (`@task(partition="gpu")`) fails at submit time with a +`PymonikError`: + +```text +task 'render' requested partition 'gpu', but the session is only bound to ['pymonik']. +Pass that partition to client.session(partition=[...]) to enable it. +``` + +## Multiple partitions on one session + +Pass a list: + +```python +with client.session(partition=["cpu", "gpu", "io"]) as s: + ... +``` + +- The **first** partition is the default for tasks that don't pick + one explicitly. +- The full list is what the session advertises to ArmoniK on create. +- Per-task partition selection (`@task(partition="gpu")` or + `.with_options(partition="gpu")`) must be one of the declared + partitions. + +```python +@task +def cheap(x): return x + +@task(partition="gpu") +def render(scene): ... + +with client.session(partition=["cpu", "gpu"]) as s: + cheap.spawn(1) # routes to "cpu" (default) + render.spawn(scene) # routes to "gpu" (explicit) + + fast = render.with_options(partition="gpu-a100") # NOT in the set + fast.spawn(scene) # raises PymonikError at submit time +``` + +## When to use this + +- **Tasks need different hardware.** GPU tasks on a GPU partition, CPU + pre/post-processing on a CPU partition, all stitched together with + data dependencies. +- **Different worker images per route.** Partition A runs an image + with TensorFlow; partition B runs one with PyTorch. Both bound to + the session, tasks pick at submit time. +- **Quota or priority isolation.** Some operators put noisy + experiments on a separate partition and route only specific tasks + there. + +## When not to use this + +- **One partition is fine.** Don't list multiple if you don't need + them; the validation cost is real (every submission checks partition + membership), and the cluster sees a session it can route to N + partitions even if you only ever use one. +- **Cross-cluster routing.** A session is bound to one cluster. To + submit work to multiple clusters, open multiple `PymonikClient` + instances. + +## Inspecting a session's partitions + +Both attributes are available on a `Session`: + +```python +with client.session(partition=["cpu", "gpu"]) as s: + s.partition # "cpu" — the default + s.partitions # ("cpu", "gpu") — the full set +``` + +## Discovering what's available + +The cluster's partition catalogue is reachable from the client (no +session needed): + +```python +with PymonikClient() as client: + for p in client.partitions.list(): + print(p.id, p.priority, p.preemption_percentage) +``` + +Use this to decide what to bind in `client.session(partition=[...])`, +or to script a "give me the lowest-priority partition with a free +slot" allocation. diff --git a/.docs/guides/observability.md b/.docs/guides/observability.md new file mode 100644 index 0000000..8730500 --- /dev/null +++ b/.docs/guides/observability.md @@ -0,0 +1,149 @@ +# Observability + +PymoniK emits OpenTelemetry spans for the whole task lifecycle: +session open, batch submission, blob upload, worker-side execution, +client-side waits. Spans propagate from client to worker via a W3C +trace context embedded in the task envelope, so a single trace covers +your submission *and* the work that ran on the cluster. + +It's opt-in (no overhead when off) and the visualisation story is one +container. + +## Install the optional dependency + +```sh +uv add 'pymonik[otel]' +``` + +This pulls in `opentelemetry-api`, `opentelemetry-sdk`, and the OTLP +gRPC exporter. Without these installed, every OTel call site in +PymoniK is a no-op — the library runs unchanged. + +## The minimum-infra visualisation + +Run [Jaeger all-in-one](https://www.jaegertracing.io/docs/getting-started/) +locally — UI, OTLP collector, and storage in one container: + +```sh +docker run --rm -d --name jaeger \ + -p 16686:16686 \ + -p 4317:4317 \ + -e COLLECTOR_OTLP_ENABLED=true \ + jaegertracing/all-in-one:latest +``` + +- `16686` — Jaeger UI ([http://localhost:16686](http://localhost:16686)) +- `4317` — OTLP/gRPC ingress + +Point the client at it via standard OTel env vars: + +```sh +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +export OTEL_SERVICE_NAME=pymonik-client # optional; default "pymonik" +``` + +Run any pymonik script. Spans appear in Jaeger under the `pymonik` +service. + +## Enabling tracing in code + +PymoniK auto-detects when standard OTel env vars are set and turns on +tracing without further configuration. To force-enable / force- +disable from code: + +```python +from pymonik import PymonikClient + +with PymonikClient(otel=True) as client: # force on, regardless of env + ... + +with PymonikClient(otel=False) as client: # force off, even if env vars set + ... +``` + +If your application already configures a `TracerProvider` (e.g. you +use OTel for other things), PymoniK detects and uses it — no second +provider, no double export. + +## What spans you get + +``` +pymonik.session.open +└── pymonik.submit (count=N, func=..., partition=...) + ├── pymonik.task.run [worker] (task_id=..., attempt=1) + ├── pymonik.task.run [worker] + └── ... (one per task in the batch) +pymonik.future.wait +pymonik.blob.upload +``` + +Span attributes you can filter on in the UI: + +- `pymonik.func` — the decorated function name +- `pymonik.task_id` — the ArmoniK task id +- `pymonik.partition` — the partition the task was submitted to +- `pymonik.count` — batch size for `.map()` calls +- `pymonik.attempt` — 1 for fresh submissions, ≥2 for retries +- `pymonik.bytes` — size of an uploaded blob +- `pymonik.subprocess` / `pymonik.local` — task ran in subprocess + (deps + isolate=True) or in LocalCluster + +The submit span and the worker's `pymonik.task.run` span share a +trace id; the worker span's parent is the submit span. So in Jaeger +you click into one trace and see the whole batch. + +## Workers in a real cluster + +The local Jaeger container only sees client-side spans by default — +worker pods need to reach the same collector to export their spans. +Two ways: + +1. **Bake the env vars into the worker image** when you build it (see + [Worker images](worker-images.md)). The image's pod template gets + `OTEL_EXPORTER_OTLP_ENDPOINT` baked in, pointing at an in-cluster + collector reachable by both client and worker. +2. **Set the env at the partition level** via your ArmoniK Terraform + variables (`workers[*].env`) so the polling agent injects the env + var into worker pods at scale-up time. + +For local-cluster setups (kind, Docker Desktop), `host.docker.internal:4317` +usually points to the host's Jaeger from within the cluster. + +## End-to-end example + +`examples/with_otel.py` ships with a runnable end-to-end demo against +LocalCluster. Run with the Jaeger container above: + +```sh +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \ + uv run python examples/with_otel.py +``` + +Open Jaeger, find the trace, see the tree. + +## Sampling and cost + +By default head-sampling is 100% (every trace exported). For a +production cluster doing thousands of tasks per second, that's a lot +of span volume. Configure sampling via the standard OTel env vars: + +```sh +export OTEL_TRACES_SAMPLER=parentbased_traceidratio +export OTEL_TRACES_SAMPLER_ARG=0.01 # 1% +``` + +The client samples; if it decides to keep a trace, the W3C trace +context tells the worker to keep its span too. Sampling is consistent +end-to-end. + +## What's not yet covered + +ArmoniK itself (control plane, polling agent, agent sidecar) doesn't +emit OTel spans yet — that's an upstream item on their roadmap. Until +that lands, traces show a gap between the client's `pymonik.submit` +span and the worker's `pymonik.task.run` span: the polling agent's +wait time, queue depth, and dispatch latency happen there but don't +appear as spans. The trace tree is correct; it's just sparse in the +middle. Once ArmoniK ships native OTel, the W3C context PymoniK +already propagates feeds into their spans automatically — no PymoniK +changes needed. diff --git a/.docs/guides/retries.md b/.docs/guides/retries.md new file mode 100644 index 0000000..cc099a4 --- /dev/null +++ b/.docs/guides/retries.md @@ -0,0 +1,105 @@ +# Retries + +Two flavours, both opt-in: + +- **Cluster-side retries** — `@task(retries=N)` alone. ArmoniK retries + the task up to N times for any failure (infra crash or user-code + exception). Cheap; nothing on the client wakes up between attempts. +- **Client-side retries** — `@task(retries=N, retry_on=(...))`. The + SDK observes the failure type, sleeps a backoff, and re-spawns. The + cluster's `max_retries` is held at 2 (still covers infra crashes); + the application retry loop runs in your process. + +## Cluster-side: blanket retry on any failure + +```python +@task(retries=3) +def flaky(x: int) -> int: + ... +``` + +ArmoniK retries up to 3 times. The task is identified by a fresh +`task_id` per attempt; from the client's perspective, the +`Future.result()` either delivers the eventual success or surfaces +the final failure as `TaskFailed`. + +Use this when you don't care *why* a task failed and a re-attempt is +likely to work — transient network errors, temporary resource +contention, ArmoniK pod restarts. + +## Client-side: filterable retries with backoff + +```python +from pymonik import task + +@task( + retries=5, + retry_on=(ConnectionError, TimeoutError), + retry_backoff="exponential", +) +def call_external_api(url: str) -> str: + ... +``` + +The SDK retries when the worker raised `ConnectionError` or +`TimeoutError`, up to 5 times, with exponential backoff between +attempts. Other exceptions surface immediately as `TaskFailed` — no +retry. + +Backoff strategies: + +- `"exponential"` (default) — `0.5, 1.0, 2.0, 4.0, ...` capped at 30s. +- `"linear"` — `0.5, 1.0, 1.5, 2.0, ...`. +- `"constant"` — 1 second between every attempt. +- A number — fixed seconds. +- A callable `attempt -> seconds` — total control. + +```python +@task(retries=10, retry_on=(MyTransientError,), retry_backoff=lambda a: 2 ** a + 1) +def very_specific(...): ... +``` + +`attempt` is 0 for the first retry, 1 for the second, etc. + +## When to use which + +- **Use cluster retries** when retries are infrastructure-driven: + ArmoniK pod went away, gRPC blip, agent restart. The cluster handles + it; your client doesn't need to know. +- **Use client retries** when retries are application-driven: a third- + party API rate-limited you, a database is recovering, a network + partition is healing. The application knows the right backoff and + the right exception types. + +You can combine: `@task(retries=5, retry_on=(MyError,))` gives you 5 +client-side application retries *plus* the cluster's default 2 infra +retries underneath. + +## How a retry surfaces to the user + +The client-side retry is invisible to your `await fut` / +`.result()`. The same `Future` object is rewired in place — its +`task_id` and `result_id` change between attempts; awaiters keep +waiting. The `attempt` field on the envelope (visible to the worker +via `pymonik.current().attempt`) lets idempotency-aware code see +which try this is. + +The PymoniK logger emits one `task retrying` line per attempt with +the delay and old/new task ids — useful when something is +retry-storming. + +## Retries in batches + +If you `.map(args)` and one of the N tasks fails, only the failing +one is retried. The other futures in the `FutureList` resolve +normally. Each retry is a single-task re-submission, not a re-batch. + +## What doesn't retry + +- **Submission failures** — if the gRPC call to submit the batch + itself raises (control-plane down, auth refused), no retry. The + exception propagates out of `.spawn()` / `.map()` immediately. +- **Cancelled tasks** — `TaskCancelled` is never retried. Cancellation + is intentional. +- **`PymonikError`s that aren't subclasses of the listed types** — + `retry_on` filters strictly. Catch what you mean. diff --git a/.docs/guides/runtime-environment.md b/.docs/guides/runtime-environment.md new file mode 100644 index 0000000..980b77e --- /dev/null +++ b/.docs/guides/runtime-environment.md @@ -0,0 +1,155 @@ +# Runtime environment + +PymoniK lets you control the Python environment a task runs in +without rebuilding the worker image: install pip dependencies on +demand, set environment variables, point at a private package index. +Useful when you want to iterate on code that depends on libraries +that aren't in the base worker image. + +## Adding pip dependencies + +Declare them on the session: + +```python +with client.session( + partition="pymonik", + deps=["numpy", "polars>=1", "scikit-learn==1.5.*"], +) as s: + ... +``` + +`deps` is a list of [PEP 508](https://peps.python.org/pep-0508/) +specifier strings — exactly what you'd put in `requirements.txt`. + +The first task into a worker pod with these deps installs them into +a content-addressed venv at `/cache/internal/envs//.venv` +(via `uv pip install`). Subsequent tasks reuse that venv with no +install cost. Two sessions on the same cluster declaring the same +`deps` resolve to the same `env_id` and share the same venv. + +The wire footprint is the deps strings only — never a lockfile. +ArmoniK's polling agent caches the venv across tasks within a pod's +lifetime; pod restarts wipe it. + +## Per-task overrides + +Different tasks in one session can declare extra deps: + +```python +@task(deps=["torch==2.6"]) +def gpu_inference(x): ... + +@task # no deps — uses session's set +def cheap_aggregate(xs): ... +``` + +Or per call via `with_options`: + +```python +heavy = analyze.with_options(deps=["polars>=1.20"]) +heavy.spawn(df).result() +``` + +Merge order is the same as for other options: session ← `@task` ← +`.with_options`. + +## How tasks run with deps: subprocess vs in-process + +When `deps` is non-empty, the worker has two modes for actually +running the task: + +| Mode | When to use | How it works | +|------|-------------|--------------| +| **In-process splice** (default, `isolate=False`) | Compute-light tasks, single session per pod. ~1 ms per task once warm. | Worker adds the env's `site-packages` to `sys.path` and calls the function inline. | +| **Subprocess** (`isolate=True`) | Concurrent sessions on the same pod with conflicting deps, or tasks that mutate global module state. | Worker spawns a fresh Python interpreter from the env's venv per task. ~400-500 ms startup with numpy. | + +Default is in-process splice because it's drastically faster for the +common case (numpy alone costs ~400 ms to import; subprocess pays +that on every task). The trade-off: import state persists across +tasks on the same pod. Two tasks in the same session that import +`numpy` see the same module; if a third task imported a *different* +numpy version on the same pod, the first import would win. + +Opt into subprocess isolation when that matters: + +```python +with client.session( + partition="pymonik", + deps=["torch==2.6"], + isolate=True, +) as s: + ... +``` + +## Environment variables + +Set per-session env vars alongside (or instead of) deps: + +```python +with client.session( + partition="pymonik", + deps=["numpy"], + env={"OMP_NUM_THREADS": "4", "MY_FEATURE_FLAG": "true"}, +) as s: + ... +``` + +Per-task and per-call overrides work the same way (`@task(env=...)`, +`.with_options(env=...)`). Merges are key-wise — the per-task dict +adds to / overrides the session's, it doesn't replace it. + +`env` works *with or without* `deps`. If you don't need extra packages +but want env vars on a task, `client.session(env={...})` alone is +enough — no venv is built. + +Env vars participate in the `env_id` hash when deps are also +declared, so two sessions with the same deps but different env vars +get distinct venvs. This is intentional: env vars often change install +behaviour (CUDA build selection, `PIP_INDEX_URL`, etc.), and treating +them as part of identity prevents accidental cross-contamination. + +## Private package indexes + +```python +with client.session( + partition="pymonik", + deps=["my-private-pkg>=2"], + index_url="https://idx.example.com/simple/", +) as s: + ... +``` + +`index_url` is forwarded to `uv pip install --index-url`. For +indexes that need credentials, either bake the credential into the +URL (`https://user:token@idx.example.com/`) — keeping in mind that +`index_url` is part of the `env_id` hash, so two URLs that differ +only in credential will produce different venvs — or set +`UV_INDEX_URL` / `UV_EXTRA_INDEX_URL` in the worker pod environment +where it stays out of envelope payloads. + +## When to use this vs baking an image + +| Use `deps=` (this guide) | Bake an image | +|--------------------------|---------------| +| Iterating on what packages your tasks need | Production deploys you want pinned | +| Mixing different deps in different sessions on shared workers | The same set every time | +| Tasks that import third-party libraries | Tasks that import third-party libraries *plus your own multi-file project* | +| Need a private package on a one-off basis | C-extension libraries with system deps you'd otherwise need to install at runtime | + +For multi-file projects, see the +[multi-file projects section in Important considerations](../important-considerations.md#cloudpickle-and-multi-file-projects). +For image baking, see [Worker images](worker-images.md). + +## Inspecting the cache + +Venvs live at `/cache/internal/envs//` on the worker pod. +The `env_id` is logged at submission time when the events stream +delivers a result, and it's stable across runs given the same deps — +so you can grep your worker logs for it. + +Eviction is the polling agent's responsibility. A pod restart wipes +the cache (it's an `emptyDir`); a fresh pod re-runs `uv pip install` +on the first task with that env_id. Wheel downloads are shared across +env builds via `UV_CACHE_DIR=/cache/internal/uv`, so the second +session to install torch on a given pod pays the install but not the +download. diff --git a/.docs/guides/sub-tasking-and-multi-output.md b/.docs/guides/sub-tasking-and-multi-output.md new file mode 100644 index 0000000..913c543 --- /dev/null +++ b/.docs/guides/sub-tasking-and-multi-output.md @@ -0,0 +1,284 @@ +# Sub-tasking and multi-output tasks + +Two related primitives, both about controlling how a task's output flows +into the cluster: + +- **`task.tail(*args)`** — sub-tasking. Lets a `@task` body delegate its + output to another task. Replaces what other frameworks call + "tail-call" or "delegation." +- **`MultiResult(field=value, ...)`** — multi-output tasks. A single task + produces N independently-named outputs that downstream consumers + depend on individually. + +They compose: a multi-output task can tail-call to another multi-output +task, and a multi-output task can use `tail()` to delegate just one +field's computation to a child task. + +## Sub-tasking with `task.tail()` + +```python +from pymonik import task + +@task +def base(n: int) -> int: + return n + 1 + +@task +def adaptive(n: int) -> int: + if n < 1024: + return base.tail(n) # delegate to base; base writes our output + return n +``` + +When `adaptive(2)` is invoked remotely, the worker: + +1. Runs the function. It returns `base.tail(2)` — a `TailPromise`. +2. Submits `base` as a child task whose ArmoniK + `expected_output_ids` is set to *adaptive's* expected output id. +3. Returns. The cluster delivers `base`'s result to whoever was + awaiting `adaptive`. + +The user's submission code never sees the difference: + +```python +with client.session(partition="pymonik") as s: + print(adaptive.spawn(2).result(timeout=10)) # 3, via base + print(adaptive.spawn(2000).result(timeout=10)) # 2000, no delegation +``` + +### `tail()` is lazy + +`task.tail(*args)` does **not** submit a task immediately. It returns a +`TailPromise` that the framework will submit later, with whichever +output id is appropriate (the parent's output, or a specific +MultiResult field's output). Three rules: + +- **Awaiting a `TailPromise` directly is an error.** It hasn't been + submitted; there's nothing to await. If you want the result, use + `task.spawn(...)` instead. +- **A `TailPromise` is only valid as a return value of a `@task`** — + either returned directly, or as a field value inside a + `MultiResult`. Anywhere else it's an error. +- **A worker that constructs a `TailPromise` and drops it on the + floor** (returns something else without binding it) leaks: the + child task is never submitted. There's no warning today; we may + add one. + +### `tail()` chains + +A tail-called task can itself tail-call: + +```python +@task +def increment_chain(n: int, acc: int) -> int: + if n == 0: + return acc + return increment_chain.tail(n - 1, acc + 1) +``` + +Each link's child writes to the *original* parent's output id (since +the chain unwinds — every intermediate task's output id is the same). +ArmoniK handles arbitrary depth. + +## Multi-output tasks with `MultiResult` + +```python +from pymonik import MultiResult, task + +@task +def split(x: int): + return MultiResult(double=x * 2, triple=x * 3) +``` + +`split.spawn(7)` returns a `MultiResultHandle`, not a `Future`: + +```python +with client.session(partition="pymonik") as s: + out = split.spawn(7) + out.double.result() # 14 — only blocks on the `double` output + out.triple.result() # 21 — only blocks on the `triple` output + view = out.result() # MultiResultView — blocks on every field + view.double # 14 (attribute access) + view["double"] # 14 (dict-style access) + dict(view) # {"double": 14, "triple": 21} +``` + +Each field is its own ArmoniK output id. A downstream task that +consumes one field doesn't wait on the other: + +```python +@task +def double_plus_one(d: int) -> int: + return d + 1 + +with client.session(partition="pymonik") as s: + out = split.spawn(7) + answer = double_plus_one.spawn(out.double) # depends only on `double` + answer.result() # 15 — runs before `triple` finishes +``` + +That independent-scheduling behaviour is the reason to use +`MultiResult` rather than returning a dataclass: a slow `triple` +doesn't gate consumers of `double`. + +### How the schema is extracted + +The `@task` decorator walks the function body's AST at decoration +time, finds every `MultiResult(...)` literal, and validates that all +branches use the same field set: + +```python +@task +def conditional(x: int): + if x > 0: + return MultiResult(a=x, b=-x) + return MultiResult(a=-x, b=x) # ← same field set; OK +``` + +Branches with inconsistent shapes raise at decoration: + +```python +@task +def bad(x: int): + if x > 0: + return MultiResult(a=x, b=x) + return MultiResult(a=x, b=x, c=x) # ← raises PymonikError on import +``` + +The error has the offending lines. Bugs that would silently mis-write +outputs in production show up at module-load time. + +### Limitations of AST extraction + +- **Helpers don't count.** `MultiResult(...)` constructed in a helper + function the task calls is invisible to the AST walk. +- **`**kwargs` expansion is rejected.** `MultiResult(**dynamic)` would + produce a non-static field set; the decorator raises. +- **Aliased imports work.** `from pymonik import MultiResult as MR; + return MR(a=..., b=...)` is fine — the walker tracks top-level + imports. + +If you need to construct `MultiResult` outside the task body, declare +the schema explicitly via the decorator: + +```python +@task(outputs=("a", "b")) +def via_helper(x): + return _build_outputs(x) + +def _build_outputs(x): + return MultiResult(a=x, b=-x) +``` + +`outputs=(...)` overrides the AST walk; the decorator trusts your +declared field set. + +### `MultiResult` returning the wrong shape fails the task + +Even with AST extraction in place, what actually flows at runtime is +checked again on the worker. A task that declared +`MultiResult(a=int, b=int)` but returns `MultiResult(a=int)` (perhaps +via a helper) fails: + +```text +TaskFailed: MultiResult shape mismatch (missing ['b']). +Declared: ['a', 'b']; returned: ['a']. +``` + +## Per-field tail-call: `MultiResult(a=other.tail(...))` + +A `MultiResult` field's value can be a plain Python value (cloudpickled +and written by the parent worker) **or** a `TailPromise` (delegated to +a child task that writes that one field's output): + +```python +@task +def heavy_compute(x: int) -> int: + # ... slow ... + return x * 100 + +@task +def split(x: int): + return MultiResult( + cheap=x + 1, # written by split's worker + expensive=heavy_compute.tail(x),# delegated; heavy_compute's worker writes it + ) +``` + +The cluster runs: + +- `split`'s worker writes `cheap`'s bytes to its output id and submits + `heavy_compute` as a child with the `expensive` output id. +- `heavy_compute` runs (possibly on a different partition / pod) and + writes its result to `expensive`'s output id. +- A downstream consumer of `out.cheap` runs immediately; a consumer of + `out.expensive` waits for `heavy_compute`. + +### Rules for `MultiResult` fields + +- **Plain values** — pickled, written directly. Use for fast-to-compute + fields. +- **`TailPromise` from `task.tail(...)`** — delegated to a child task. + Use when one field is expensive enough to warrant its own task. +- **`Future` from `task.spawn(...)`** — **error**. The Future has its + own output id (allocated when `spawn` ran inside the parent worker); + binding it to a MultiResult field would mean re-routing already- + submitted work, which the cluster can't do cheaply. Use `tail()` + instead. +- **Multi-output children** (`MultiResult(a=other_split.tail(x))` + where `other_split` is itself multi-output) — **error**. Per-field + delegation requires a single-output child. To wire up a nested + multi-output result, insert a passthrough single-output task that + forwards just the field you want. + +## Whole-task tail-call to a multi-output child + +A multi-output task can tail-call another multi-output task — the +shapes must match: + +```python +@task +def rebranded(x: int): + return MultiResult(a=x * 2, b=x * 3) + +@task +def parent(x: int): + if x > 100: + return rebranded.tail(x) # OK: child declares same shape + return MultiResult(a=x, b=x * 5) +``` + +If the schemas differ, the worker raises clearly: + +```text +worker error: tail-called task 'wrong_shape' declares ['x', 'y', 'z'] +but parent declares ['a', 'b'] — shapes must match for whole-task tail-call. +``` + +## Cancellation + +`MultiResultHandle.cancel()` cancels the task that produces all of the +handle's outputs. ArmoniK's `Tasks.CancelTasks` operates per-task — +there's no "cancel just one output of a task." Every field's Future +resolves to `TaskCancelled`. + +For tail-call chains, cancelling the parent cancels the chain: ArmoniK +propagates cancellation to children when a parent is cancelled. + +## Cluster behaviour matches local + +Everything documented here works the same under `LocalCluster` for +testing — same envelope encoding, same dispatch logic. See the +[Local testing](local-testing.md) guide. + +## When to reach for which + +- **Want a single result?** Plain `@task` returning a single value. +- **Want a single result, decided dynamically by another task?** + `task.tail(...)` returned from the body. +- **Want N results that downstream tasks consume independently?** + `MultiResult(...)` with the field set extracted at decoration. +- **Want a structured result that downstream consumers always read + whole?** Plain `@task` returning a dataclass — no need for + `MultiResult`. One ArmoniK output, one data-dependency edge per + consumer; less ceremony. diff --git a/.docs/guides/worker-images.md b/.docs/guides/worker-images.md new file mode 100644 index 0000000..f49370a --- /dev/null +++ b/.docs/guides/worker-images.md @@ -0,0 +1,181 @@ +# Worker images + +A PymoniK worker pod runs the `pymonik-worker` console script +inside a Docker image. The image needs: + +- A Python interpreter matching the client's minor version. +- The `pymonik` package installed. +- Whatever Python libraries (and your project code) the tasks import. +- A non-root `armonikuser` and a writable `/cache` directory. + +The official base image (`dockerhubaneo/harmonic_snake`) gives you +the first three; you bake on top of it for project-specific deps. + +## Choosing the right image + +| Situation | What to do | +|-----------|------------| +| You're prototyping, no project code on workers, all deps fit `pymonik[deps]=...` | Use the base image as-is, declare deps via `client.session(deps=[...])`. See [Runtime environment](runtime-environment.md). | +| You have a multi-file project and want fast iteration | Base image, run `cloudpickle.register_pickle_by_value(mypkg)` at your client's entry. See [Important considerations](../important-considerations.md#cloudpickle-and-multi-file-projects). | +| You're shipping production: pinned deps, your project code, no install cost on every pod scale-out | Bake your own image. | + +## Adding the worker partition to your cluster + +The PymoniK worker runs as a partition in your ArmoniK Terraform +config: + +```hcl +pymonik = { + replicas = 0 # scale-from-zero; HPA below brings up pods on demand + polling_agent = { + limits = { cpu = "2000m", memory = "2048Mi" } + requests = { cpu = "50m", memory = "50Mi" } + } + worker = [ + { + image = "dockerhubaneo/harmonic_snake" + tag = "python-3.11-2.0.0a3" # MATCH your client's Python + pymonik version + limits = { cpu = "1000m", memory = "1024Mi" } + requests = { cpu = "50m", memory = "50Mi" } + } + ] + hpa = { + type = "prometheus" + polling_interval = 15 + cooldown_period = 300 + min_replica_count = 0 + max_replica_count = 5 + behavior = { + restore_to_original_replica_count = true + stabilization_window_seconds = 300 + type = "Percent" + value = 100 + period_seconds = 15 + } + triggers = [ + { type = "prometheus", threshold = 2 }, + ] + } +} +``` + +The image tag has the form `python--`. Find +available tags at the +[official Docker Hub repository](https://hub.docker.com/r/dockerhubaneo/harmonic_snake). + +The Python minor in the tag **must match** your client's Python +minor — cloudpickle isn't cross-minor-compatible. + +## Building your own image + +Two reasons: + +1. **Pinned dependencies** — your tasks import a fixed set of + libraries. Baking them in skips the first-task install cost on + every fresh pod. +2. **Multi-file project** — your tasks import from `mypkg.foo`. The + worker needs `mypkg` importable; baking is the production answer. + +Start from the official base image: + +```dockerfile +ARG PYTHON_VERSION=3.11 +FROM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-bookworm-slim AS base + +RUN groupadd --gid 5000 armonikuser \ + && useradd --uid 5000 --gid 5000 --home-dir /home/armonikuser --create-home armonikuser \ + && mkdir /cache && chown armonikuser: /cache + +USER armonikuser +WORKDIR /app + +# Copy your project metadata + lockfile + source. +COPY --chown=armonikuser:armonikuser pyproject.toml uv.lock README.md* ./ +COPY --chown=armonikuser:armonikuser src/ ./src/ + +# Build a frozen venv. `--no-dev` skips dev deps (pytest, ruff, ...). +RUN uv venv /app/.venv && uv sync --frozen --no-dev + +ENV PATH="/app/.venv/bin:${PATH}" +ENV PYTHONUNBUFFERED=1 + +# The worker entrypoint that ArmoniK launches. +ENTRYPOINT ["pymonik-worker"] +``` + +Build: + +```sh +docker build -t my-org/my-pymonik-worker:v3 . +``` + +Push to whatever registry your cluster pulls from: + +```sh +docker push my-org/my-pymonik-worker:v3 +``` + +Update your Terraform `worker[0].image` and `tag`, apply, and +restart the partition: + +```sh +kubectl rollout restart deployment/compute-plane-pymonik -n armonik +``` + +## What "must match" means concretely + +For an image to function as a PymoniK worker: + +- **Python minor** matches the client's. If your `pyproject.toml` has + `requires-python = "==3.11.*"`, the image must run Python 3.11. +- **`pymonik` is installed** in the venv at `/app/.venv` (or wherever + `PATH` points). The console script `pymonik-worker` must be on + `PATH`. +- **`armonikuser` (uid 5000)** owns `/cache` and `/app` (the polling + agent expects these paths writable by the same uid that + `pymonik-worker` runs as). + +The simplest sanity check: locally, `docker run --rm -it + bash` and run `pymonik-worker --help`. If that prints +help, the image is structurally fine. + +## Custom worker code + +If you have a reason to add your own logic to the worker process — +metrics emission, custom signal handling, monkey-patching a library +before any task runs — write a small Python entrypoint that calls +`pymonik.worker.run()` and use that as the image's `ENTRYPOINT`: + +```python +# my_worker.py +import logging + +import pymonik +from pymonik.worker import run + +def main() -> None: + logging.getLogger("my_app").setLevel(logging.INFO) + pymonik.enable_logging("INFO") + # Your one-time setup here. + run() + +if __name__ == "__main__": + main() +``` + +```dockerfile +# ... same base as above ... +COPY --chown=armonikuser:armonikuser my_worker.py /app/ +ENTRYPOINT ["python", "/app/my_worker.py"] +``` + +`pymonik.worker.run()` does the same thing the `pymonik-worker` +console script does — registers the dispatch loop with the upstream +`armonik` worker framework and serves tasks until shut down. + +## Future: `pymonik image build` + +A `pymonik image build` CLI subcommand is planned: +read your `pyproject.toml`, render a Dockerfile from a template, run +`docker build`, print the tag. Until that lands, hand-write the +Dockerfile above; it's ~15 lines and changes rarely. diff --git a/.docs/important-considerations.md b/.docs/important-considerations.md index 1f6ebc5..bf9339c 100644 --- a/.docs/important-considerations.md +++ b/.docs/important-considerations.md @@ -1,3 +1,168 @@ -## Can I use different Python versions for the client and worker ? +# Important considerations -Although you can install different python packages on your execution environment, you're constrained to a single python version on both your worker and client. There is currently no support for switching between different Python versions. If you're using the harmonic_snake worker then you need to use Python 3.10.12 for your client. \ No newline at end of file +A small number of constraints don't show up in the API but bite if you +don't know about them. Read this once before shipping. + +## Python version pinning + +**The client's Python minor version must match the worker's.** PymoniK +ships your function as a `cloudpickle` blob; cloudpickle bytecode is +not cross-minor-compatible. A 3.11 client against a 3.12 worker will +SIGSEGV during unpickle in the worker process — usually with no +traceback, just a non-zero exit. + +PymoniK's wire envelope embeds `sys.version_info` and the worker +rejects mismatches with a typed `ValueError`, but the cleaner fix is +to pin your client's Python to whatever the worker image was built +with. The default worker image is built for Python 3.11; if your +project uses 3.11 too, you're set. To run a different version you'll +need to bake a matching worker image (see +[Worker images](guides/worker-images.md)). + +In `pyproject.toml`: + +```toml +[project] +requires-python = "==3.11.*" +``` + +## Cloudpickle and multi-file projects + +Single-file scripts work out of the box: cloudpickle pickles +functions in `__main__` *by value* — bytecode + globals — so the +worker doesn't need the source on disk. + +Multi-file projects don't. cloudpickle pickles functions in normal +modules *by reference*: it stores `(module_name, qualname)` and the +worker re-imports `module_name` to look the function up. If the +worker doesn't have your project installed, the import fails and the +task does too. + +Two answers, both supported: + +1. **Bake your project into the worker image.** The recommended + production path — once the image has `pip install .` of your code, + every task can find every helper. See + [Worker images](guides/worker-images.md). +2. **Tell cloudpickle to pickle your package by value too.** At your + client's entrypoint: + + ```python + import cloudpickle + import mypkg + + cloudpickle.register_pickle_by_value(mypkg) + ``` + + Now functions in `mypkg.tasks`, `mypkg.utils`, etc. are pickled the + same way `__main__` functions are. The worker doesn't need + `mypkg` installed — it reconstructs from the pickled bytes. + +The first is right for production; the second is right for fast +iteration without rebuilding the image on every change. + +A future `additional_modules=` option will automate (2). For now, +`register_pickle_by_value` is the primitive. + +## Partitions and routing + +A session is bound to one or more partitions on the cluster. By +default `client.session(partition="pymonik")` allows only that +partition; if a task tries to route to anything else (`@task(partition="gpu")`), +submission is rejected at the client. + +To allow a task to choose, declare the set up front: + +```python +with client.session(partition=["cpu", "gpu"]) as s: + fast = render.with_options(partition="gpu").spawn(scene) +``` + +The first partition in the list is the default for tasks that don't +specify one. See [Multi-partition routing](guides/multi-partition.md). + +## Result delivery: events vs polling + +By default the client opens a server-streamed gRPC `Events.GetEvents` +call to receive completions. Latency from "result ready" to "future +resolved" is a few ms. + +If the events stream misbehaves in your environment (proxies, network +policies that mangle long-lived streams), fall back to polling: + +```python +PymonikClient(events=False, polling_interval=1.0, polling_chunk=200) +``` + +Polling does one `Tasks.list_results` RPC every `polling_interval` +seconds, batched into chunks of `polling_chunk` ids. Higher latency, +no streaming connection. + +## Argument size and auto-spill + +Anything you pass to `.spawn()` rides in the task's payload — except +when the cloudpickled bytes exceed `spill_threshold` (default 256 KiB, +configurable on the client). Large args are uploaded as blobs and +referenced via `data_dependencies` automatically; you don't need to +think about it. + +If you're passing the same big object to many tasks, upload it once +explicitly: + +```python +import pymonik.blob as blob + +shared = blob.upload(big_dict) # uploaded once +for i in range(1000): + process.spawn(shared, i) # all 1000 tasks share the same blob_id +``` + +See [Blobs and Materialize](guides/blobs-and-materialize.md). + +## Worker-side blocking is illegal + +Inside a `@task` body, `Future.result()` / `await future` raises: + +```python +@task +def parent() -> int: + child = other.spawn(...) + return child.result() # PymonikError — workers don't poll for results +``` + +ArmoniK tasks are ephemeral; blocking inside one ties up a pod +indefinitely. Pass the future to another `.spawn()` (creates a data +dependency edge so ArmoniK runs the next task once this one +completes), or return it with `_delegate=True` to hand off your +expected output. See [Sub-tasking in Getting Started](getting-started.md#sub-tasking). + +## Returning multiple results + +A task returns one Python object. If you `return a, b, c`, the worker +pickles the tuple and downstream tasks receive the tuple. There's no +way today to declare "this task produces three independent outputs." +If you need that, return a dict and have downstream tasks pick keys, +or split into three tasks. + +## Cancellation propagation + +`session.cancel()` and `future.cancel()` issue ArmoniK +`CancelTasks`/`CancelSession` RPCs and resolve pending futures locally +with `TaskCancelled`. Worker-side cancellation observation +(`pymonik.current().cancel_if_requested()`) requires a small +upstream-armonik change that's not yet merged, so for now the worker +only learns about cancellation when its gRPC channel is torn down. + +## Logging + +The library is **silent by default** (uses a `NullHandler`). To see +PymoniK's structured logs: + +```python +import pymonik +pymonik.enable_logging("INFO") +``` + +Workers always log — operators rely on the polling-agent → k8s +pipeline to surface what each pod is doing. Set +`PYMONIK_WORKER_LOG_LEVEL` in the worker environment to override. diff --git a/.docs/index.rst b/.docs/index.rst index 2455064..5ceaba5 100644 --- a/.docs/index.rst +++ b/.docs/index.rst @@ -1,18 +1,66 @@ -Welcome to PymoniK's documentation! -===================================== +PymoniK +======= + +A dead-simple Python SDK for `ArmoniK `_. + +PymoniK turns a regular Python function into a remote task with a single +decorator. Tasks compose into pipelines via plain function calls; the +SDK takes care of submission, data dependencies, retries, and result +delivery. The same code runs locally for tests and on a cluster of +hundreds of pods for production. + +.. code-block:: python + + from pymonik import PymonikClient, task + + @task + def add(a: int, b: int) -> int: + return a + b + + @task + def total(xs: list[int]) -> int: + return sum(xs) + + with PymonikClient() as client: + with client.session(partition="pymonik") as s: + parts = add.map(range(16), range(1, 17)) + print(total.spawn(parts).result(timeout=60)) .. toctree:: :maxdepth: 2 - :caption: Contents: + :caption: First steps introduction getting-started - guides/adding-worker-image + important-considerations + +.. toctree:: + :maxdepth: 2 + :caption: Guides + + guides/runtime-environment + guides/blobs-and-materialize + guides/sub-tasking-and-multi-output + guides/multi-partition + guides/retries + guides/local-testing + guides/observability + guides/async + guides/worker-images guides/custom-worker + +.. toctree:: + :maxdepth: 2 + :caption: Examples + examples/monte_carlo - examples/pong_training examples/raytracing + examples/pong_training examples/pricing_workflows + +.. toctree:: + :maxdepth: 2 + :caption: Development + development/development development/contribution - important-considerations diff --git a/.docs/introduction.md b/.docs/introduction.md index 426fbae..3557cd1 100644 --- a/.docs/introduction.md +++ b/.docs/introduction.md @@ -1,54 +1,96 @@ -## Quick introduction +# Introduction +PymoniK is a Python framework for writing distributed programs that run +on an [ArmoniK](https://github.com/aneoconsulting/ArmoniK) cluster. It +sits on top of the lower-level `armonik` Python client and gives you a +decorator-first API that feels like calling regular functions. -PymoniK is a dead simple Python framework for writing distributed programs that run on an ArmoniK cluster. It's a wrapper around the low-level `ArmoniK.API` that allows you to easily make your Python programs distributed. +## What it gives you -- Make your functions run in the cloud using a simple decorator. +**A decorator turns any function into a remote task.** + +```python +from pymonik import task -```py @task -def hello_worlder(): +def hello() -> str: return "hello world" - -with Pymonik(): - print(hello_worlder.invoke().wait().get()) ``` -- Run multiple tasks in parallel: +Inside a session, `hello.spawn()` submits the function for remote +execution and returns a `Future[str]`. `hello()` still calls the +function locally — the decoration doesn't get in your way during +debugging. -```py -@task -def add(a,b): - return a+b +**Many tasks at once, batched into one round-trip.** -with Pymonik(): - results = add.map_invoke([(i, i+1) for i in range(32)]) - print(results.wait().get()) +```python +@task +def add(a: int, b: int) -> int: + return a + b +results = add.map(range(32), range(1, 33)) ``` -- Easily construct and run complex task graphs, interweave local and remote code execution: +`map` zips its iterables (Python-stdlib semantics) and packs all 32 +submissions into a single gRPC call. Returns a `FutureList[int]`. -```py -@task -def get_constant() - return 2 +**Pipelines compose by passing futures as arguments.** +```python @task -def add(a,b): - return a+b - -with PymoniK(): - my_constant = get_constant.invoke() - results = add.map_invoke([(my_constant, i) for i in range(32)]) - sum_task = Task(sum) - remote_partial_result = sum_task.invoke(results[:16]) - local_partial_result = sum_task(results[16:].wait().get()) - final_result = remote_partial_result.wait().get() + local_partial_result - print(final_result) +def total(xs: list[int]) -> int: + return sum(xs) + +partials = add.map(range(32), range(1, 33)) +final = total.spawn(partials) +print(final.result(timeout=60)) ``` -- Define your remote execution environment (specify Python packages), subtasking and more. +`final` doesn't wait for `partials` on the client. PymoniK rewrites +each `Future` into an ArmoniK data dependency edge. The cluster runs +`total` as soon as the upstream `add` tasks complete; the client only +blocks on the terminal `result()`. `total`'s function body receives a +plain `list[int]` — the SDK resolves the futures on the worker before +calling. + +**Local execution is a flag away.** -If you're interested in using PymoniK, please take a look at our [getting started guide](getting-started.md). +```python +from pymonik.testing import LocalCluster + +with LocalCluster() as client: + with client.session() as s: + assert add.spawn(2, 3).result(timeout=5) == 5 +``` +`LocalCluster` is a drop-in for `PymonikClient` that runs tasks in a +thread pool. Same envelope encoding, same dispatch pipeline — pytest +without a cluster. + +## Where it fits + +PymoniK is the highest-level Python SDK for ArmoniK. Underneath, it +uses the official `armonik` Python client for control-plane RPCs and +the standard worker framework for the agent sidecar. Anything you can +do with the lower-level SDK (filters, sessions, multi-partition, +priorities, retries) is reachable from PymoniK without dropping down. + +The library is opinionated about ergonomics — `Future[T]` over result +handles, decorators over registries, structured exceptions over raw +gRPC errors — but it doesn't hide ArmoniK from you. When you need a +filter query, the polling agent's cache, or the partition catalogue, +they're a property access away (`client.tasks`, `client.partitions`, +`client.results`, etc.). + +## Where to go next + +- [Getting started](getting-started.md) — install, configure, run your + first task. +- [Important considerations](important-considerations.md) — the small + number of constraints that bite if you don't know about them + (Python version pinning, cloudpickle minor compatibility, multi-file + project shipping). +- The guides cover specific topics: runtime dependencies, blobs and + file materialisation, multi-partition routing, retries, local + testing, observability, async usage, and worker image building. diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml index e33e029..f9c1c22 100644 --- a/.github/workflows/publish-images.yml +++ b/.github/workflows/publish-images.yml @@ -1,5 +1,13 @@ name: Publish Docker images +# Builds and pushes the v2 worker image to Docker Hub on every published +# GitHub release. The Dockerfile lives at worker-image/Dockerfile and uses +# the build context rooted at the repo so it can COPY the package in. +# +# Python matrix: 3.11 and 3.12. Cloudpickle is not cross-minor, so users +# must run a client whose Python minor version matches the image they +# point their session at. + on: release: types: ["published"] @@ -8,25 +16,30 @@ on: jobs: docker: strategy: - matrix: - python_version: [3.10.12, 3.11] + matrix: + python_version: ["3.11", "3.12"] runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Login to Docker Hub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v3 - name: Build and push uses: docker/build-push-action@v6 with: + context: . + file: worker-image/Dockerfile platforms: linux/amd64,linux/arm64 - file: pymonik_worker/Dockerfile push: true - build-args: "USE_PYTHON_VERSION=${{ matrix.python_version }}" + build-args: "PYTHON_VERSION=${{ matrix.python_version }}" tags: | dockerhubaneo/harmonic_snake:python-${{ matrix.python_version }}-${{ github.ref_name }} dockerhubaneo/harmonic_snake:python-${{ matrix.python_version }} diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml index 76206a2..bd4b21a 100644 --- a/.github/workflows/publish-package.yml +++ b/.github/workflows/publish-package.yml @@ -1,5 +1,9 @@ name: "Publish PymoniK package to PyPI" +# Builds and publishes the pymonik package on every published GitHub +# release. The repo is now flat (pyproject.toml at root, src/ layout), +# so no working-directory hop is needed. + on: release: types: ["published"] @@ -16,13 +20,13 @@ jobs: uses: astral-sh/setup-uv@v3 - name: Set up Python - run: uv python install 3.10.12 # - working-directory: pymonik + # Match the requires-python = "==3.11.*" floor in pyproject.toml. + # Cloudpickle is not cross-minor, so the published wheel pins to + # 3.11; a future release matrix can publish 3.12 separately. + run: uv python install 3.11 - name: Build run: uv build - working-directory: pymonik - name: Publish run: uv publish -t ${{ secrets.PYPI_TOKEN }} - working-directory: pymonik diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 142c987..3aa34b5 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -21,4 +21,3 @@ sphinx: python: install: - requirements: .docs/requirements.txt - diff --git a/README.md b/README.md index 9336bfa..fe27583 100644 --- a/README.md +++ b/README.md @@ -5,16 +5,124 @@ [![Publish Docker images](https://github.com/aneoconsulting/PymoniK/actions/workflows/publish-images.yml/badge.svg?branch=main&event=release)](https://github.com/aneoconsulting/PymoniK/actions/workflows/publish-images.yml) ![GitHub Release](https://img.shields.io/github/v/release/aneoconsulting/PymoniK) -PymoniK is a dead simple Python framework for writing distributed programs that run on an ArmoniK cluster. +A dead-simple Python SDK for [ArmoniK](https://github.com/aneoconsulting/ArmoniK). -[Documentation](https://pymonik.readthedocs.io/en/latest) -[Getting Started](https://pymonik.readthedocs.io/en/latest/getting-started.html) -[Contributing](https://pymonik.readthedocs.io/en/latest/development/contribution.html) +## Quick start + +```python +from pymonik import PymonikClient, task + +@task +def add(a: int, b: int) -> int: + return a + b + +@task +def sum_all(xs: list[int]) -> int: + return sum(xs) + +with PymonikClient() as client: # reads $AKCONFIG + with client.session(partition="pymonik") as s: + # Pipelining: pass futures as args. No client-side blocking — ArmoniK + # chains the tasks via data_dependencies. Only the terminal .result() + # actually waits. + parts = add.map(range(16), range(1, 17)) + total = sum_all.spawn(parts) + print(total.result()) +``` + +`Task.map(*iterables)` zips its iterables and submits one task per +zipped tuple — exactly Python's built-in `map` shape. If you already +have arg tuples, use `Task.starmap(args_iter)` instead. + +Async too: + +```python +import asyncio +from pymonik import PymonikClient, gather, task + +@task +def double(x: int) -> int: + return x * 2 + +async def main(): + async with PymonikClient() as client: + async with client.session_async(partition="pymonik") as s: + futures = double.map(range(8)) + results = await gather(futures) + print(results) + +asyncio.run(main()) +``` + +Multiple named outputs from one task — downstream consumers depend on +fields, not the whole result, so a slow field doesn't gate the others: + +```python +from pymonik import MultiResult, task + +@task +def split(x: int): + return MultiResult(double=x * 2, triple=x * 3) + +with PymonikClient() as client: + with client.session(partition="pymonik") as s: + out = split.spawn(7) + print(out.double.result(), out.triple.result()) # 14 21 + print(out.result()) # {double: 14, triple: 21} +``` + +Sub-tasking: a `@task` body can delegate its output to another task +via `task.tail(...)` — the parent's expected output is fulfilled by +the child, no intermediate hops: + +```python +@task +def adaptive(n: int) -> int: + if n < 1024: + return base.tail(n) # base writes our output directly + return n +``` + +No cluster handy? Run the same code in-process with `LocalCluster`: + +```python +from pymonik import task +from pymonik.testing import LocalCluster + +@task +def add(a, b): return a + b + +with LocalCluster() as client: + with client.session() as s: + assert add.spawn(2, 3).result() == 5 +``` + +## Layout + +``` +src/pymonik/ Python package + _internal/ implementation details (submit pipeline, refs, cache) + cli/ `pymonik` CLI (click) + testing/ LocalCluster / LocalSession +worker-image/ Dockerfile baking the worker entrypoint +examples/ Live, runnable examples +.docs/ Sphinx documentation (Sphinx + MyST) +tests/ pytest suite (unit + slow integration via LocalCluster) +``` + ## Requirements -PymoniK is a wrapper around the low level APIs of ArmoniK, and thus requires you to use an ArmoniK cluster. (PS: It's not that hard) -- For more information on deploying ArmoniK please read the [getting started with ArmoniK guide](https://armonik.readthedocs.io/en/latest/content/armonik/getting-started.html). -- For more information on using a PymoniK worker in ArmoniK, please refer to [this guide](TODO) +- Python ≥ 3.11 and < 3.13 (cloudpickle is not cross-minor; the worker + image's Python must match the client's). +- An ArmoniK cluster — see the [ArmoniK getting-started guide](https://armonik.readthedocs.io/en/latest/content/armonik/getting-started.html). +- For local-only tests: nothing else; `LocalCluster` runs in-process. + +## Documentation +The full guide tree lives under `.docs/` (built with Sphinx + MyST). +Topics: getting started, runtime pip dependencies, blobs and file +materialization, multi-partition routing, retries, local testing, +observability (OTel + Jaeger), async, sub-tasking and multi-output +tasks, worker image building. diff --git a/automation.py b/automation.py deleted file mode 100644 index 884f68f..0000000 --- a/automation.py +++ /dev/null @@ -1,357 +0,0 @@ -# /// script -# requires-python = ">=3.10" -# dependencies = [ -# "mkdocs-material[imaging]", -# "rich-click", -# "ruff", -# ] -# /// - -import rich_click as click -import subprocess -import shutil -import os -from pathlib import Path - -# TODO: Consider switching to zxpy -# Configure rich-click to use Rich for help text and styling -click.rich_click.USE_RICH_MARKUP = True -click.rich_click.SHOW_ARGUMENTS = True -click.rich_click.GROUP_ARGUMENTS_OPTIONS = True -click.rich_click.STYLE_ERRORS_SUGGESTION = "magenta italic" -click.rich_click.ERRORS_SUGGESTION = "Try running the --help flag for more information." -click.rich_click.ERRORS_EPILOGUE = "To find out more, read our [link=https://aneoconsulting.github.io/PymoniK/]developer's guide[/link]" - -@click.group() -def cli(): - """ - A CLI for managing common development tasks for Pymonik. - """ - pass - -@cli.command("build-docker") -@click.option( - "--image-name", - "-i", - default="pymonik_worker", - show_default=True, - help="Name of the Docker image to build.", -) -@click.option( - "--python-version", - "-pv", - default="3.10.12", - show_default=True, - help="Python version to use in the Docker image.", -) -@click.option( - "--dockerfile-path", - "-df", - default="pymonik_worker/Dockerfile", # Assuming this is the path - show_default=True, - help="Path to the Dockerfile relative to the current directory.", -) -@click.option( - "--context-path", - "-c", - default=".", - show_default=True, - help="Build context path for Docker.", -) -@click.option( - "--refresh-namespace", - default="armonik", - show_default=True, - help="Namespace to use when refreshing the image used in the kubernetes deployment. Useful during development." -) -@click.option( - "--refresh-partition", - default="pymonik", - show_default=True, - help="Partition to use when refreshing the image used in the kubernetes deployment. Useful during development." -) -@click.option( - "--refresh", - is_flag=True, - help="Refresh the image used in the kubernetes deployment.", -) -@click.option( - "--push", - is_flag=True, - help="Push the image to Docker Hub after a successful build.", -) -def build_docker(image_name: str, python_version: str, dockerfile_path: str, context_path: str, refresh_namespace:str, refresh_partition:str, refresh:bool, push: bool): - """ - Builds a Docker image. - - Example: - `python your_script_name.py build-docker -i my_image --python-version 3.11 --push` - """ - click.secho(f"Building Docker image '{image_name}' with Python {python_version}...", fg="cyan") - - # Check if Dockerfile exists - if not Path(dockerfile_path).exists(): - click.secho(f"Error: Dockerfile not found at '{dockerfile_path}'. Please specify the correct path.", fg="red") - raise click.Abort() - - build_command = [ - "docker", - "build", - "-t", - image_name, - "-f", - dockerfile_path, - "--build-arg", - f"USE_PYTHON_VERSION={python_version}", - context_path, - ] - - try: - click.secho(f"Running command: {' '.join(build_command)}", fg="yellow") - subprocess.run(build_command, check=True) - click.secho(f"Docker image '{image_name}' built successfully.", fg="green") - - if push: - click.secho(f"Pushing image '{image_name}' to Docker Hub...", fg="cyan") - push_command = ["docker", "push", image_name] - click.secho(f"Running command: {' '.join(push_command)}", fg="yellow") - subprocess.run(push_command, check=True) - click.secho(f"Image '{image_name}' pushed successfully.", fg="green") - - except subprocess.CalledProcessError as e: - click.secho(f"Error during Docker operation: {e}", fg="red") - click.secho(f"Command output:\n{e.stdout}\n{e.stderr}", fg="red") - raise click.Abort() - except FileNotFoundError: - click.secho("Error: Docker command not found. Is Docker installed and in your PATH?", fg="red") - raise click.Abort() - - if refresh: - click.secho(f"Refreshing image used in the kubernetes deployment...", fg="cyan") - refresh_command = [ - "kubectl", - "rollout", - "restart", - f"deployment/compute-plane-{refresh_partition}", - "--namespace", - refresh_namespace, - ] - try: - click.secho(f"Running command: {' '.join(refresh_command)}", fg="yellow") - subprocess.run(refresh_command, check=True) - click.secho(f"Image '{image_name}' refreshed successfully in the kubernetes deployment.", fg="green") - except subprocess.CalledProcessError as e: - click.secho(f"Error during kubernetes operation: {e}", fg="red") - click.secho(f"Command output:\n{e.stdout}\n{e.stderr}", fg="red") - raise click.Abort() - except FileNotFoundError: - click.secho("Error: kubectl command not found. Is kubectl installed and in your PATH?", fg="red") - raise click.Abort() - click.secho("Refreshed image.", fg="green") - - -@cli.command("serve-docs") -@click.option( - "--port", - "-p", - default=8000, - show_default=True, - help="Port to serve the documentation on.", - type=int, -) -def serve_docs(port: int): - """ - Serves the MkDocs documentation locally. - Requires MkDocs to be installed. - """ - click.secho(f"Serving MkDocs documentation on http://127.0.0.1:{port}...", fg="cyan") - command = ["mkdocs", "serve", "--dev-addr", f"127.0.0.1:{port}"] - try: - click.secho(f"Running command: {' '.join(command)}", fg="yellow") - subprocess.run(command, check=True) - except subprocess.CalledProcessError as e: - click.secho(f"Error serving documentation: {e}", fg="red") - raise click.Abort() - except FileNotFoundError: - click.secho("Error: mkdocs command not found. Is MkDocs installed and in your PATH?", fg="red") - raise click.Abort() - - -@cli.command("publish-docs") -@click.option( - "--message", - "-m", - help="Commit message for publishing the documentation.", -) -@click.option( - "--force", - is_flag=True, - help="Force push the documentation. Use with caution.", -) -def publish_docs(message: str | None, force: bool): - """ - Builds and deploys the MkDocs documentation, typically to GitHub Pages. - This command uses `mkdocs gh-deploy`. - """ - click.secho("Publishing MkDocs documentation...", fg="cyan") - command = ["mkdocs", "gh-deploy"] - if message: - command.extend(["--message", message]) - if force: - command.append("--force") - - try: - click.secho(f"Running command: {' '.join(command)}", fg="yellow") - subprocess.run(command, check=True) - click.secho("Documentation published successfully.", fg="green") - except subprocess.CalledProcessError as e: - click.secho(f"Error publishing documentation: {e}", fg="red") - raise click.Abort() - except FileNotFoundError: - click.secho("Error: mkdocs command not found. Is MkDocs installed and in your PATH?", fg="red") - raise click.Abort() - - -@cli.command("publish-project") -@click.option( - "--project-dir", - "-d", - default="./pymonik", - show_default=True, - help="Path to the Python project directory managed by UV.", - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), -) -@click.option( - "--token", - envvar="UV_PYPI_TOKEN", # Example environment variable, adjust as needed - help="Authentication token for publishing. Can also be set via environment variable (e.g., UV_PYPI_TOKEN).", -) -def publish_project(project_dir: str, token: str | None): - """ - Builds and publishes a Python project using UV. - Assumes necessary environment variables for authentication are set if --token is not provided. - """ - click.secho(f"Publishing Python project in '{project_dir}' using UV...", fg="cyan") - - # Step 1: Ensure the project is built (create sdist and wheel) - build_command = ["uv", "build"] - click.secho(f"Running build command in {project_dir}: {' '.join(build_command)}", fg="yellow") - try: - subprocess.run(build_command, cwd=project_dir, check=True) - click.secho("Project built successfully.", fg="green") - except subprocess.CalledProcessError as e: - click.secho(f"Error building project with UV: {e}", fg="red") - click.secho(f"Command output:\n{e.stdout}\n{e.stderr}", fg="red") - raise click.Abort() - except FileNotFoundError: - click.secho("Error: uv command not found. Is UV installed and in your PATH?", fg="red") - raise click.Abort() - - # Publish the built distributions - publish_command = ["uv", "publish"] - - publish_env = os.environ.copy() - if token: - publish_env["UV_PUBLISH_TOKEN"] = token - click.secho("Using provided token for publishing.", fg="yellow") - click.secho(f"Running publish command: {' '.join(publish_command)}", fg="yellow") - try: - subprocess.run(publish_command, cwd=project_dir, check=True, env=publish_env) - click.secho(f"Successfully published project to PyPI.", fg="green") - except subprocess.CalledProcessError as e: - click.secho(f"Error publishing project with UV: {e}", fg="red") - click.secho(f"Command output:\n{e.stdout}\n{e.stderr}", fg="red") - except FileNotFoundError: - click.secho("Error: uv command not found. Is UV installed and in your PATH?", fg="red") - raise click.Abort() - finally: - # Delete the dist folder - dist_path = Path(project_dir) / "dist" - if dist_path.exists() and dist_path.is_dir(): - try: - shutil.rmtree(dist_path) - click.secho(f"Successfully deleted directory: {dist_path}", fg="green") - except OSError as e: - click.secho(f"Error deleting directory {dist_path}: {e}", fg="red") - else: - click.secho(f"Directory not found (or not a directory): {dist_path}", fg="yellow") - click.secho("Project publishing process completed.", fg="green") - - -@cli.command("clean") -def clean(): - """ - Cleans the project: - - Deletes the 'site/' directory (MkDocs build output). - - Cleans UV projects in 'pymonik/' and 'test_client/' by removing common build artifacts. - """ - click.secho("Cleaning project...", fg="cyan") - - # Delete site/ directory - site_dir = Path("site") - if site_dir.exists() and site_dir.is_dir(): - try: - shutil.rmtree(site_dir) - click.secho(f"Successfully deleted directory: {site_dir}", fg="green") - except OSError as e: - click.secho(f"Error deleting directory {site_dir}: {e}", fg="red") - else: - click.secho(f"Directory not found (or not a directory): {site_dir}", fg="yellow") - - # Clean specified UV project directories - project_dirs_to_clean = ["pymonik", "test_client"] - for project_path_str in project_dirs_to_clean: - project_path = Path(project_path_str) - click.secho(f"Cleaning UV project in '{project_path}'...", fg="cyan") - if project_path.exists() and project_path.is_dir(): - # Common directories/files to remove for a "clean" operation - # `uv clean` itself is more about the global cache. - # For project cleaning, we remove typical build/cache outputs. - items_to_remove = [ - ".venv", - "__pycache__", - ".pytest_cache", - "build", - "dist", - "*.egg-info", # Glob pattern for .egg-info directories - ".ruff_cache", - ".mypy_cache" - ] - - for item_name in items_to_remove: - if "*" in item_name: # Handle glob patterns - for matching_item in project_path.glob(item_name): - try: - if matching_item.is_dir(): - shutil.rmtree(matching_item) - click.secho(f" Removed directory: {matching_item}", fg="green") - elif matching_item.is_file(): - matching_item.unlink() - click.secho(f" Removed file: {matching_item}", fg="green") - except OSError as e: - click.secho(f" Error removing {matching_item}: {e}", fg="red") - else: - item_path = project_path / item_name - if item_path.exists(): - try: - if item_path.is_dir(): - shutil.rmtree(item_path) - click.secho(f" Removed directory: {item_path}", fg="green") - elif item_path.is_file(): - item_path.unlink() - click.secho(f" Removed file: {item_path}", fg="green") - except OSError as e: - click.secho(f" Error removing {item_path}: {e}", fg="red") - else: - click.secho(f" Item not found: {item_path}", fg="yellow") - click.secho(f"Cleaning for '{project_path}' complete.", fg="green") - else: - click.secho(f"Directory not found for cleaning: {project_path}", fg="yellow") - - - click.secho("Project cleaning finished.", fg="green") - -# TODO: format command - -if __name__ == "__main__": - cli() diff --git a/examples/adaptive_vector_addition.py b/examples/adaptive_vector_addition.py new file mode 100644 index 0000000..c7d3288 --- /dev/null +++ b/examples/adaptive_vector_addition.py @@ -0,0 +1,68 @@ +"""Recursive subtasking + fan-in: adaptive vector addition. + +Splits a vector in half until each chunk is under threshold, then +component-wise-adds the base case and concatenates up the tree. The +aggregation at each level is delegated to a sub-task so the parent's +expected output is fulfilled by the child (no intermediate hops). + +Uses plain Python lists instead of numpy so the default worker image +doesn't need scipy/numpy baked in. + + uv run python examples/adaptive_vector_addition.py --partition pymonikv1 +""" + +from __future__ import annotations + +import argparse + +from pymonik import PymonikClient, current, task +import pymonik + +CHUNK_THRESHOLD = 256 + + +@task +def vec_add(a: list[int], b: list[int]) -> list[int]: + """Recursive divide-and-conquer add. Delegates aggregation to a sub-task.""" + if len(a) != len(b): + raise ValueError("vector length mismatch") + + if len(a) > CHUNK_THRESHOLD: + current().log.info("splitting", size=len(a)) + mid = len(a) // 2 + left = vec_add.spawn(a[:mid], b[:mid]) + right = vec_add.spawn(a[mid:], b[mid:]) + # Delegate: the concat task's output *is* our output. When concat + # completes, ArmoniK marks this task's result as ready too. + return concat.tail(left, right) # type: ignore[return-value] + + return [x + y for x, y in zip(a, b)] + + +@task +def concat(a: list[int], b: list[int]) -> list[int]: + return a + b + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument("--partition", default="pymonikv1") + ap.add_argument("--size", type=int, default=4096) + args = ap.parse_args() + + vec_a = list(range(args.size)) + vec_b = [x * 2 for x in vec_a] + expected = [a + b for a, b in zip(vec_a, vec_b)] + + with PymonikClient() as client: + with client.session(partition=args.partition) as s: + result = vec_add.spawn(vec_a, vec_b).result(timeout=300) + if result == expected: + print(f"adaptive add verified; size={args.size}, head={result[:6]} … tail={result[-6:]}") + else: + print(f"MISMATCH: got head={result[:6]} expected head={expected[:6]}") + + +if __name__ == "__main__": + main() diff --git a/examples/async_hello.py b/examples/async_hello.py new file mode 100644 index 0000000..0e3490b --- /dev/null +++ b/examples/async_hello.py @@ -0,0 +1,59 @@ +"""Async entry points — ``async with PymonikClient()`` and ``await future``. + +Mirrors ``examples/hello.py`` but runs on the user's asyncio loop. +Submission stays sync (spawn returns a Future), waiting is asynchronous. + + uv run python examples/async_hello.py --partition pymonikv1 +""" + +from __future__ import annotations + +import argparse +import asyncio +import time + +from pymonik import PymonikClient, task +import pymonik + + +@task +def add(a: int, b: int) -> int: + return a + b + + +@task +def double(x: int) -> int: + return x * 2 + + +@task +def sum_all(xs: list[int]) -> int: + return sum(xs) + + +async def main(partition: str) -> None: + pymonik.enable_logging() + t0 = time.monotonic() + async with PymonikClient() as client: + async with client.session_async(partition=partition) as s: + # Composition: spawn is sync, await is async. Submission returns + # immediately; ArmoniK holds `doubled` and `total` in PENDING + # via data_dependencies until their inputs complete. + seed = add.spawn(2, 3) + doubled = double.spawn(seed) # depends on seed + leaves = add.map(range(8), range(1, 9)) + total = sum_all.spawn(leaves) # depends on all leaves + + # Await two separate DAG terminals concurrently via asyncio.gather. + a, b = await asyncio.gather(doubled, total) + print(f"doubled(2+3) = {a}") + print(f"sum(1,3,5,...,15) = {b}") + + print(f"took {time.monotonic() - t0:.1f}s") + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--partition", default="pymonikv1") + args = ap.parse_args() + asyncio.run(main(args.partition)) diff --git a/examples/blobs.py b/examples/blobs.py new file mode 100644 index 0000000..36b0f77 --- /dev/null +++ b/examples/blobs.py @@ -0,0 +1,94 @@ +"""Blobs — explicit upload, materialize-to-path, and auto-spill. + +Three flows in one example: + +1. ``blob.upload(Path(...))`` — file contents delivered to the task as bytes. +2. ``blob.materialize(Path(...), at=...)`` — file written to the worker FS at a + specific path; the task parameter receives a ``pathlib.Path`` to it. +3. Auto-spill — a large plain-Python arg (above the spill threshold) is + transparently uploaded and rewired as a data dependency. User code looks + identical to the inline form. + +Also demonstrates content-hash dedup: the second upload of the same bytes +reuses the first result id and skips the network round-trip. + + uv run python examples/blobs.py --partition pymonikv1 +""" + +from __future__ import annotations + +import argparse +import tempfile +from pathlib import Path + +from pymonik import PymonikClient, blob, current, task +import pymonik + + +@task +def fingerprint_bytes(label: str, payload: bytes) -> str: + ctx = current() + ctx.log.info("task got bytes", label=label, size=len(payload)) + return f"{label}: {len(payload)} bytes; head={payload[:8]!r}" + + +@task +def read_config(cfg: Path) -> str: + ctx = current() + ctx.log.info("task reads materialized file", path=str(cfg)) + return f"config at {cfg} says: {cfg.read_text().strip()!r}" + + +@task +def sum_samples(samples: list[float]) -> float: + """Receives a (possibly auto-spilled) big list. User code is oblivious.""" + current().log.info("summing", n=len(samples)) + return sum(samples) + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument("--partition", default="pymonikv1") + ap.add_argument("--samples", type=int, default=200_000) + args = ap.parse_args() + + # Make two tiny local files we can blob/materialize. + with tempfile.TemporaryDirectory() as tmp: + weights_path = Path(tmp) / "weights.bin" + weights_path.write_bytes(b"WEIGHTS" + b"\x01" * 10_000) + + cfg_path = Path(tmp) / "app.toml" + cfg_path.write_text("mode='production'\nvalue=42\n") + + with PymonikClient() as client: + with client.session(partition=args.partition) as s: + # (1) Explicit blob: file bytes → delivered as `bytes`. + weights = blob.upload(weights_path) + print("uploaded:", weights) + + # Dedup: second call finds the cache and returns the same handle shape. + weights_again = blob.upload(weights_path) + assert weights.result_id == weights_again.result_id, "dedup failed" + print("second upload reused:", weights_again.result_id[:8] + "…") + + # (2) Materialize: worker writes the bytes to this path before the task. + cfg = blob.materialize(cfg_path, at="/tmp/pmk_app.toml") + print("materialize:", cfg) + + # (3) Auto-spill: a half-million-float list is well above 256 KiB + # cloudpickled; submission will quietly turn it into a Blob. + big = [x * 0.01 for x in range(args.samples)] + print(f"big list size = {args.samples} floats") + + f1 = fingerprint_bytes.spawn("weights", weights) + f2 = read_config.spawn(cfg) + f3 = sum_samples.spawn(big) + + print(f1.result(timeout=120)) + print(f2.result(timeout=120)) + print(f"sum_samples -> {f3.result(timeout=120):.2f}") + + +if __name__ == "__main__": + main() diff --git a/examples/cancellation.py b/examples/cancellation.py new file mode 100644 index 0000000..5319bcc --- /dev/null +++ b/examples/cancellation.py @@ -0,0 +1,90 @@ +"""Cancellation — client-initiated, cooperatively honoured on the worker. + +Two flows: + +1. ``future.cancel()`` — cancels a single task via ArmoniK ``CancelTasks``. + The future resolves locally with :class:`TaskCancelled` immediately; + the task may run briefly longer on the worker until it checks in via + ``pymonik.current().cancel_if_requested()``. + +2. ``session.cancel()`` — cancels every in-flight task in the session + via ``CancelSession``. All pending futures resolve with + :class:`TaskCancelled`. + +Cooperative on the worker side: the ``@task`` body must periodically call +``pymonik.current().cancel_if_requested()``. A task that never checks +runs to ``max_duration`` regardless of cluster state. + + uv run python examples/cancellation.py --partition pymonikv1 +""" + +from __future__ import annotations + +import argparse +import threading +import time + +from pymonik import PymonikClient, TaskCancelled, current, task +import pymonik + + +@task +def slow(steps: int) -> int: + """Cooperative long task. Checks in every iteration.""" + ctx = current() + for i in range(steps): + ctx.cancel_if_requested() # raises TaskCancelled if so + if i % 5 == 0: + ctx.log.info("tick", i=i, steps=steps) + time.sleep(0.3) + return steps + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument("--partition", default="pymonikv1") + args = ap.parse_args() + + with PymonikClient() as client: + with client.session(partition=args.partition) as s: + # ---- 1. cancel a single future ---- + print("test 1: single future.cancel()") + fut = slow.spawn(30) # would take ~9 s + + def cancel_after(sec: float): + time.sleep(sec) + print(f" client: cancelling after {sec}s") + fut.cancel() + + threading.Thread(target=cancel_after, args=(2.0,), daemon=True).start() + t0 = time.monotonic() + try: + fut.result(timeout=30) + print(" UNEXPECTED success") + except TaskCancelled as e: + print(f" cancelled as expected after {time.monotonic() - t0:.2f}s: {e}") + + # ---- 2. cancel the whole session ---- + print("test 2: session.cancel()") + futs = [slow.spawn(30) for _ in range(3)] + + def cancel_session_after(sec: float): + time.sleep(sec) + print(f" client: session.cancel() after {sec}s") + s.cancel() + + threading.Thread(target=cancel_session_after, args=(1.0,), daemon=True).start() + + t0 = time.monotonic() + cancelled = 0 + for f in futs: + try: + f.result(timeout=30) + except TaskCancelled: + cancelled += 1 + print(f" {cancelled}/{len(futs)} cancelled after {time.monotonic() - t0:.2f}s") + + +if __name__ == "__main__": + main() diff --git a/examples/estimate_pi.py b/examples/estimate_pi.py new file mode 100644 index 0000000..2e1e185 --- /dev/null +++ b/examples/estimate_pi.py @@ -0,0 +1,55 @@ +"""Estimate π via parallel Monte-Carlo sampling. + +Map N parallel Monte-Carlo estimates, then reduce via a single task whose +inputs are the fan-out futures. Client blocks only on the terminal result. + + uv run python examples/estimate_pi.py --partition pymonikv1 --n 32 --samples 200000 +""" + +from __future__ import annotations + +import argparse +import random +import time + +from pymonik import PymonikClient, current, task +import pymonik + + +@task +def estimate_pi_partial(num_samples: int) -> tuple[int, int]: + # pymonik.current() gives structured-logging + task/session ids on the worker. + current().log.info("shard start", samples=num_samples) + hits = 0 + for _ in range(num_samples): + x, y = random.random(), random.random() + if x * x + y * y <= 1.0: + hits += 1 + return hits, num_samples + + +@task +def reduce_pi(partials: list[tuple[int, int]]) -> float: + hits = sum(h for h, _ in partials) + samples = sum(n for _, n in partials) + return 4 * hits / samples + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument("--partition", default="pymonikv1") + ap.add_argument("--n", type=int, default=32) + ap.add_argument("--samples", type=int, default=200_000) + args = ap.parse_args() + + t0 = time.monotonic() + with PymonikClient() as client: + with client.session(partition=args.partition) as s: + shards = estimate_pi_partial.map([args.samples] * args.n) + pi = reduce_pi.spawn(shards).result(timeout=300) + print(f"pi ≈ {pi:.6f} ({args.n * args.samples} samples, {time.monotonic()-t0:.1f}s)") + + +if __name__ == "__main__": + main() diff --git a/examples/exec_cache.py b/examples/exec_cache.py new file mode 100644 index 0000000..abaf5d7 --- /dev/null +++ b/examples/exec_cache.py @@ -0,0 +1,99 @@ +"""Local execution cache. + +Two-knob opt-in: + +1. ``PymonikClient(cache=True)`` (or a ``Path``) enables the cache + *infrastructure* — without this the cache directory is never touched. +2. ``@task(cache=True)`` declares one specific task pure-and-cacheable. + +When both are set, ``.spawn()`` / ``.map()`` consult the on-disk cache +*before* submitting. A hit returns a Future that's already resolved +with the cached value — zero RPCs. A miss submits as normal and the +result is written back when it lands. + +Caching skips automatically when: + +- An arg is a ``Future`` (upstream value not yet known). +- A leaf isn't picklable. + +Run twice. First run hits the cluster (or LocalCluster); second run +shows hits and finishes in milliseconds. + + uv run python examples/exec_cache.py + uv run python examples/exec_cache.py # second run = hits + uv run pymonik cache stats # peek inside + uv run pymonik cache clear --yes # wipe between experiments + +This example uses LocalCluster so it works without a deployed cluster. +""" + +from __future__ import annotations + +import argparse +import time +from pathlib import Path + +from pymonik import current, task +import pymonik +from pymonik.testing import LocalCluster + + +@task(cache=True) +def expensive_pure(n: int) -> int: + """Pretend-expensive computation; deterministic so the cache is valid.""" + current().log.info("running expensive_pure", n=n) + time.sleep(0.3) # simulate real work + return sum(i * i for i in range(n)) + + +@task # NOT cached — no @task(cache=True) +def cheap_uncached(n: int) -> int: + return n * 2 + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument( + "--cache-dir", + default=str(Path.home() / ".cache" / "pymonik-example"), + help="Cache root for this demo (default keeps it out of the global cache).", + ) + args = ap.parse_args() + + print(f"using cache at {args.cache_dir}") + + with LocalCluster(cache=args.cache_dir) as client: + with client.session() as s: + inputs = [10_000, 20_000, 30_000, 40_000] + + t0 = time.monotonic() + futures = expensive_pure.map(inputs) + results = [f.result() for f in futures] + print(f" expensive map -> {results} ({time.monotonic() - t0:.2f}s)") + + # Same inputs again — should be all hits. + t0 = time.monotonic() + again = [expensive_pure.spawn(n).result() for n in inputs] + print(f" same inputs again -> {again} ({time.monotonic() - t0:.2f}s)") + assert again == results, "cache returned different value!" + + # New input → miss for that one only. + t0 = time.monotonic() + mixed = [expensive_pure.spawn(n).result() for n in [10_000, 50_000, 30_000]] + print(f" one new + two cached -> {mixed} ({time.monotonic() - t0:.2f}s)") + + # Uncached task: never goes to cache regardless of how many times. + t0 = time.monotonic() + r = cheap_uncached.spawn(7).result() + print(f" cheap_uncached(7) -> {r} ({time.monotonic() - t0:.2f}s; not cached)") + + # ``.with_options(cache=False)`` can opt a single call out even + # when the @task decorator says cache=True. + t0 = time.monotonic() + r = expensive_pure.with_options(cache=False).spawn(10_000).result() + print(f" cache=False override -> {r} ({time.monotonic() - t0:.2f}s; bypassed cache)") + + +if __name__ == "__main__": + main() diff --git a/examples/fanout.py b/examples/fanout.py new file mode 100644 index 0000000..6a29daa --- /dev/null +++ b/examples/fanout.py @@ -0,0 +1,55 @@ +"""Fan-out example: submit N tasks in parallel, collect results. + +Demonstrates that the session's background poller can resolve many +futures at once. + + uv run python examples/fanout.py --endpoint localhost:5001 --partition pymonik --n 20 +""" + +from __future__ import annotations + +import argparse +import random +import time + +from pymonik import PymonikClient, task +import pymonik + + +@task +def estimate_pi_partial(num_samples: int) -> tuple[int, int]: + hits = 0 + for _ in range(num_samples): + x, y = random.random(), random.random() + if x * x + y * y <= 1.0: + hits += 1 + return hits, num_samples + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument("--endpoint", default="localhost:5001") + ap.add_argument("--partition", default="pymonik") + ap.add_argument("--n", type=int, default=20, help="number of worker tasks") + ap.add_argument("--samples", type=int, default=200_000, help="samples per task") + args = ap.parse_args() + + t0 = time.monotonic() + with PymonikClient(endpoint=args.endpoint) as client: + with client.session(partition=args.partition) as s: + futures = [estimate_pi_partial.spawn(args.samples) for _ in range(args.n)] + print(f"submitted {args.n} tasks; waiting") + total_hits = 0 + total_samples = 0 + for f in futures: + hits, samples = f.result(timeout=300) + total_hits += hits + total_samples += samples + pi = 4 * total_hits / total_samples + elapsed = time.monotonic() - t0 + print(f"pi ≈ {pi:.6f} ({total_samples} samples, {elapsed:.1f}s)") + + +if __name__ == "__main__": + main() diff --git a/examples/gather_async.py b/examples/gather_async.py new file mode 100644 index 0000000..f3c55d1 --- /dev/null +++ b/examples/gather_async.py @@ -0,0 +1,60 @@ +"""Async fan-in: ``pymonik.gather`` and ``pymonik.as_completed``. + +Same flavour as ``asyncio.gather`` / ``asyncio.as_completed`` but takes +``Future`` / ``FutureList`` directly. Two demos in one script: + +1. Submit a fan-out, gather everything in submission order. +2. Submit a fan-out, consume results as-they-complete with the typed + Future yielded back. + + uv run python examples/gather_async.py --partition pymonikv1 +""" + +from __future__ import annotations + +import argparse +import asyncio +import random +import time + +from pymonik import PymonikClient, as_completed, gather, task +import pymonik + + +@task +def slow_double(x: int) -> int: + # Random short delay so the fan-out has interesting completion order. + import time as _t + _t.sleep(0.1 + 0.6 * random.random()) + return x * 2 + + +async def main(partition: str, n: int) -> None: + pymonik.enable_logging() + async with PymonikClient() as client: + async with client.session_async(partition=partition) as s: + # ---- gather() ---- + print(f"submitting {n} tasks for gather()") + t0 = time.monotonic() + futs = slow_double.map(range(n)) + results = await gather(futs) # results in submission order + print(f" gather -> {results} ({time.monotonic() - t0:.1f}s)") + + # ---- as_completed() ---- + print(f"submitting {n} more for as_completed()") + t0 = time.monotonic() + futs2 = slow_double.map(range(100, 100 + n)) + received: list[int] = [] + async for done in as_completed(futs2): + value = await done # the typed Future is yielded back + received.append(value) + print(f" +{value:>3} (running for {time.monotonic() - t0:.2f}s)") + print(f" total {sum(received)} ({time.monotonic() - t0:.1f}s)") + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--partition", default="pymonikv1") + ap.add_argument("--n", type=int, default=8) + args = ap.parse_args() + asyncio.run(main(args.partition, args.n)) diff --git a/examples/hello.py b/examples/hello.py new file mode 100644 index 0000000..96f2741 --- /dev/null +++ b/examples/hello.py @@ -0,0 +1,49 @@ +"""Minimal smoke test: submit a single task and print its result. + +Easiest: + + export AKCONFIG=/path/to/generated/armonik-cli.yaml + uv run python examples/hello.py + +Or explicit: + + uv run python examples/hello.py --endpoint --partition pymonik +""" + +from __future__ import annotations + +import argparse + +from pymonik import PymonikClient, task +import pymonik + + +@task +def add(a: int, b: int) -> int: + return a + b + + +@task +def greet(name: str) -> str: + return f"hello, {name}, from the ArmoniK worker" + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument("--endpoint", default=None, help="overrides AKCONFIG if given") + ap.add_argument("--partition", default="pymonik") + args = ap.parse_args() + + with PymonikClient(endpoint=args.endpoint) as client: + with client.session(partition=args.partition) as s: + f1 = add.spawn(2, 3) + f2 = greet.spawn("pymonik") + print("submitted; waiting for results") + + print("add(2, 3) ->", f1.result(timeout=120)) + print("greet('pymonik') ->", f2.result(timeout=120)) + + +if __name__ == "__main__": + main() diff --git a/examples/introspection.py b/examples/introspection.py new file mode 100644 index 0000000..ce19570 --- /dev/null +++ b/examples/introspection.py @@ -0,0 +1,123 @@ +"""Fluent introspection: ``client.tasks.where(...).list()`` and friends. + +Cluster-wide queries on the client, session-scoped queries on the +session, mutation verbs on each: + +- ``client.tasks.where(status=ERROR).cancel()`` +- ``session.results.where(status=COMPLETED).download()`` +- ``client.sessions.where(status=PAUSED).resume()`` + +Field names are homogenised: ``id`` works on every resource (task, +result, session, partition); the upstream-native names (``task_id``, +``result_id``, ``session_id``) also resolve so either shape is fine. + +This example walks through the read paths against your real cluster +and demonstrates the query / mutation verbs without actually destroying +anything. + + uv run python examples/introspection.py --partition pymonikv1 +""" + +from __future__ import annotations + +import argparse + +from armonik.common import SessionStatus, TaskStatus + +from pymonik import PymonikClient, task +import pymonik + + +@task +def add(a: int, b: int) -> int: + return a + b + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument("--partition", default="pymonikv1") + args = ap.parse_args() + + with PymonikClient() as client: + # ---- cluster-wide reads ---- + print("== partitions ==") + for p in client.partitions.order_by("priority").list(): + print(f" {p.id:<14} priority={p.priority} pod_max={p.pod_max}") + + print("\n== sessions (newest 5, completed only) via .list() ==") + completed_sessions = ( + client.sessions + .where(status=SessionStatus.CLOSED) + .order_by("-status") # only STATUS is filterable upstream + .limit(5) + .list() + ) + for s in completed_sessions: + print(f" {s.id} status={s.status.name} parts={s.partition_ids}") + + print(f"\n== sessions count by status ==") + for st_name in ("RUNNING", "PAUSED", "CLOSED", "CANCELLED"): + try: + st = getattr(SessionStatus, st_name) + n = client.sessions.where(status=st).count() + print(f" {st_name:<10} {n}") + except AttributeError: + pass # status name differs across armonik versions + + # ---- submit a few tasks so we have something to query ---- + with client.session(partition=args.partition) as s: + print(f"\n== running 4 tasks in session {s.session_id[:8]}… ==") + futs = add.map(range(4), range(1, 5)) + results = [f.result(timeout=60) for f in futs] + print(f" results: {results}") + + # ---- session-scoped reads ---- + print("\n== this session's tasks (homogenised .id) ==") + for t in s.tasks.order_by("created_at").list(): + print( + f" id={t.id[:8]}… status={t.status.name} " + f"partition={t.partition_id}" + ) + + print("\n== count by status (cluster total + this session) ==") + cluster_completed = client.tasks.where(status=TaskStatus.COMPLETED).count() + sess_completed = s.tasks.where(status=TaskStatus.COMPLETED).count() + print(f" COMPLETED — cluster={cluster_completed} session={sess_completed}") + + print("\n== results in this session, by status ==") + n_completed = s.results.where(status=2).count() # ResultStatus.COMPLETED == 2 + print(f" results in session: {n_completed} completed") + + print("\n== results.first() (id = homogenised result_id) ==") + r = s.results.first() + if r: + print(f" first: id={r.id[:8]}… status={r.status.name} " + f"size={r.size_bytes} name={r.name!r}") + + # ---- iteration with limits ---- + print("\n== async iteration over a fan-out (limit=3) ==") + for t in s.tasks.where(status=TaskStatus.COMPLETED).limit(3): + print(f" -> {t.id[:8]}… completed at {t.ended_at}") + + # ---- mutations: kept conservative; no actual delete here ---- + print("\n== predicate suffixes ==") + print(f" by id__in: {s.tasks.where(id__in=[t.id for t in s.tasks.list()[:2]]).count()}") + print(f" by status__ne: {s.tasks.where(status__ne=TaskStatus.ERROR).count()}") + try: + print(f" by partition__startswith='py': " + f"{client.tasks.where(partition_id__startswith='py').limit(5).count()}") + except ValueError as e: + # not all clusters support all suffixes on every field + print(f" startswith query unsupported on this cluster: {e}") + + print("\n== mutation surface (NOT firing — just shape) ==") + print(" s.tasks.where(status=TaskStatus.ERROR).cancel() # int") + print(" s.results.where(status=2).delete(batch_size=100) # int") + print(" s.results.where(status=2).download() # dict[id, bytes]") + print(" s.results.where(status=2).download_to('./out') # int (files)") + print(" client.sessions.where(status=...).pause() / .resume() / .close() / .cancel() / .delete() / .purge()") + + +if __name__ == "__main__": + main() diff --git a/examples/lambda_tasks.py b/examples/lambda_tasks.py new file mode 100644 index 0000000..dbf59bb --- /dev/null +++ b/examples/lambda_tasks.py @@ -0,0 +1,34 @@ +"""Wrap an arbitrary callable (here a lambda) as a task by building the +``Task`` wrapper directly. + + uv run python examples/lambda_tasks.py --partition pymonikv1 +""" + +from __future__ import annotations + +import argparse + +from pymonik import PymonikClient, Task +import pymonik + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument("--partition", default="pymonikv1") + args = ap.parse_args() + + # Task(func, name=...) for anything you can't put @task on — lambdas, + # third-party callables, partial()s. The name surfaces in worker logs. + add = Task(lambda a, b: a + b, name="add_lambda") + mul = Task(lambda a, b: a * b, name="mul_lambda") + + with PymonikClient() as client: + with client.session(partition=args.partition) as s: + f1 = add.spawn(1, 2) + f2 = mul.spawn(f1, 10) # f1 is a Future → wired as data_dependency + print("(1 + 2) * 10 =", f2.result(timeout=60)) + + +if __name__ == "__main__": + main() diff --git a/examples/local_cluster.py b/examples/local_cluster.py new file mode 100644 index 0000000..c9c38b2 --- /dev/null +++ b/examples/local_cluster.py @@ -0,0 +1,107 @@ +"""LocalCluster — same task code, no ArmoniK cluster, no Docker. + +``pymonik.testing.LocalCluster`` is a drop-in for ``PymonikClient`` that +runs tasks in a thread pool. Useful for unit tests, examples in CI, and +fast iteration on @task functions before committing to a deployment. + +What works the same way as the real cluster: + +- ``@task`` + ``.spawn()`` / ``.map()`` / ``.with_options(...)``. +- Futures as data dependencies. +- ``pymonik.gather`` / ``as_completed`` (sync and async). +- ``pymonik.current()`` inside tasks (logger, task_id, attempt, cancel check). +- Decorator-level retries (``retry_on=`` / ``retry_backoff=``). +- ``future.cancel()`` / ``session.cancel()``. +- Sub-tasking (returning a Future from a @task is awaited and forwarded). +- Blobs and Materialize (in-memory, file written for materialize). + +Run: + + uv run python examples/local_cluster.py +""" + +from __future__ import annotations + +import asyncio +import time + +from pymonik import ( + TaskFailed, + as_completed, + current, + gather_sync, + task, +) +from pymonik.testing import LocalCluster + + +# ---- ordinary task ---- +@task +def add(a: int, b: int) -> int: + return a + b + + +# ---- composition: pass futures as args ---- +@task +def sum_all(xs: list[int]) -> int: + return sum(xs) + + +# ---- worker-context use ---- +@task +def slow_square(x: int) -> int: + ctx = current() + ctx.log.info("squaring", x=x, attempt=ctx.attempt) + time.sleep(0.05) + return x * x + + +# ---- retries ---- +@task(retries=3, retry_on=(TaskFailed,), retry_backoff="constant") +def flaky() -> str: + # Use `current().attempt` rather than mutable globals: cloudpickle's + # round-trip resets module-level state on every attempt, and on the + # real cluster each retry may land on a different worker pod anyway. + n = current().attempt + if n < 3: + raise RuntimeError(f"failure on attempt {n}") + return f"ok after {n} attempts" + + +def sync_demo() -> None: + print("=== sync demo ===") + with LocalCluster() as client: + with client.session(partition="local") as s: + # 1) basic spawn + assert add.spawn(2, 3).result() == 5 + + # 2) composition via futures-as-args; ArmoniK-style data dep + leaves = add.map(range(8), range(1, 9)) + total = sum_all.spawn(leaves).result() + assert total == 64, total + print(f" sum DAG -> {total}") + + # 3) gather over a fan-out + squares = gather_sync(*[slow_square.spawn(i) for i in range(6)]) + print(f" squares -> {squares}") + + # 4) retries — `current().attempt` reflects the retry attempt + assert flaky.spawn().result() == "ok after 3 attempts" + print(" retried -> 3 attempts (driven by current().attempt)") + print() + + +async def async_demo() -> None: + print("=== async demo ===") + async with LocalCluster() as client: + async with client.session_async(partition="local") as s: + futs = slow_square.map(range(8)) + t0 = time.monotonic() + async for done in as_completed(futs): + v = await done + print(f" square ready: {v} ({time.monotonic() - t0:.2f}s)") + + +if __name__ == "__main__": + sync_demo() + asyncio.run(async_demo()) diff --git a/examples/pipeline.py b/examples/pipeline.py new file mode 100644 index 0000000..e5286e8 --- /dev/null +++ b/examples/pipeline.py @@ -0,0 +1,52 @@ +"""Pipeline example: futures as arguments, no client-side blocking. + +Demonstrates that ``sum_all.spawn(add.spawn(...), add.spawn(...))`` builds +an ArmoniK DAG via data_dependencies — the sum task runs only after its +inputs complete, and the client blocks only on the terminal .result(). + + uv run python examples/pipeline.py --partition pymonikv1 +""" + +from __future__ import annotations + +import argparse +import time + +from pymonik import PymonikClient, task +import pymonik + + +@task +def add(a: int, b: int) -> int: + return a + b + + +@task +def sum_all(xs: list[int]) -> int: + return sum(xs) + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument("--partition", default="pymonikv1") + ap.add_argument("--n", type=int, default=32, help="leaf additions") + args = ap.parse_args() + + t0 = time.monotonic() + with PymonikClient() as client: + with client.session(partition=args.partition) as s: + # Leaves: N parallel adds. + leaves = add.map(range(args.n), range(1, args.n + 1)) + + # Fan-in: sum_all depends on every leaf. Submitted immediately; + # runs only after all leaves complete. Client never touches the + # intermediate values. + total = sum_all.spawn(leaves) + print(f"submitted DAG: {args.n} leaves + 1 reducer") + print("total ->", total.result(timeout=300)) + print(f"took {time.monotonic()-t0:.1f}s") + + +if __name__ == "__main__": + main() diff --git a/examples/raytracing/.python-version b/examples/raytracing/.python-version deleted file mode 100644 index c8cfe39..0000000 --- a/examples/raytracing/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.10 diff --git a/examples/raytracing/pyproject.toml b/examples/raytracing/pyproject.toml deleted file mode 100644 index 7c8d803..0000000 --- a/examples/raytracing/pyproject.toml +++ /dev/null @@ -1,13 +0,0 @@ -[project] -name = "pong-ml" -version = "0.1.0" -description = "Add your description here" -readme = "README.md" -requires-python = ">=3.10.12" -dependencies = [ - "pillow>=11.2.1", - "pymonik", -] - -[tool.uv.sources] -pymonik = { path = "../../pymonik", editable = true } diff --git a/examples/raytracing/raytracing.py b/examples/raytracing/raytracing.py deleted file mode 100644 index 8790fbd..0000000 --- a/examples/raytracing/raytracing.py +++ /dev/null @@ -1,313 +0,0 @@ -import math -import os -from pymonik import Pymonik, task -from PIL import Image - -# --- Helper Classes for Raytracing --- - -class Vec3: - """A simple 3D vector class with basic operations.""" - def __init__(self, x=0.0, y=0.0, z=0.0): - self.x, self.y, self.z = float(x), float(y), float(z) - - def __add__(self, other): - return Vec3(self.x + other.x, self.y + other.y, self.z + other.z) - - def __sub__(self, other): - return Vec3(self.x - other.x, self.y - other.y, self.z - other.z) - - def __mul__(self, other): - if isinstance(other, Vec3): - return Vec3(self.x * other.x, self.y * other.y, self.z * other.z) - elif isinstance(other, (int, float)): - return Vec3(self.x * other, self.y * other, self.z * other) - else: - return NotImplemented - - def __rmul__(self, other): - if isinstance(other, (int, float)): - return Vec3(self.x * other, self.y * other, self.z * other) - else: - return NotImplemented - - def dot(self, other): - return self.x * other.x + self.y * other.y + self.z * other.z - - def cross(self, other): - return Vec3( - self.y * other.z - self.z * other.y, - self.z * other.x - self.x * other.z, - self.x * other.y - self.y * other.x - ) - - def length_squared(self): - return self.x*self.x + self.y*self.y + self.z*self.z - - def length(self): - return math.sqrt(self.length_squared()) - - def normalize(self): - l = self.length() - if l == 0: return Vec3(0,0,0) - return Vec3(self.x / l, self.y / l, self.z / l) - - def to_tuple(self): # Still useful for other purposes, like debugging or if needed elsewhere - return (self.x, self.y, self.z) - - def to_color(self): - r = int(max(0, min(255, self.x * 255.999))) - g = int(max(0, min(255, self.y * 255.999))) - b = int(max(0, min(255, self.z * 255.999))) - return (r, g, b) - - @staticmethod - def from_tuple(t): # Still potentially useful - return Vec3(t[0], t[1], t[2]) - - -class Ray: - def __init__(self, origin, direction): - self.origin = origin - self.direction = direction.normalize() - - def point_at_parameter(self, t): - return self.origin + self.direction * t - -class Material: - def __init__(self, color, ambient=0.1, diffuse=0.9, specular=0.3, shininess=32): - self.color = color - self.ambient = ambient - self.diffuse = diffuse - self.specular = specular - self.shininess = shininess - - -class Sphere: - def __init__(self, center, radius, material): - self.center = center - self.radius = float(radius) - self.material = material - - def intersect(self, ray): - oc = ray.origin - self.center - a = ray.direction.dot(ray.direction) - b = 2.0 * oc.dot(ray.direction) - c = oc.dot(oc) - self.radius * self.radius - discriminant = b*b - 4*a*c - if discriminant < 0: - return None - else: - if abs(a) < 1e-6: - return None - sqrt_discriminant = math.sqrt(discriminant) - t1 = (-b - sqrt_discriminant) / (2.0*a) - t2 = (-b + sqrt_discriminant) / (2.0*a) - epsilon = 0.001 - valid_ts = [] - if t1 > epsilon: - valid_ts.append(t1) - if t2 > epsilon: - valid_ts.append(t2) - if not valid_ts: - return None - return min(valid_ts) - - -class PointLight: - def __init__(self, position, color, intensity=1.0): - self.position = position - self.color = color - self.intensity = intensity - - -class Scene: - def __init__(self, objects, lights, background_color=Vec3(0.1, 0.1, 0.3)): - self.objects = objects - self.lights = lights - self.background_color = background_color - - -class Camera: - def __init__(self, look_from, look_at, vup, vfov_degrees, aspect_ratio): - self.origin = look_from - self.vfov_rad = math.radians(vfov_degrees) - self.aspect_ratio = aspect_ratio - w = (look_from - look_at).normalize() - u = vup.cross(w).normalize() - v = w.cross(u).normalize() - viewport_height = 2.0 * math.tan(self.vfov_rad / 2.0) - viewport_width = self.aspect_ratio * viewport_height - self.horizontal = u * viewport_width - self.vertical = v * viewport_height - viewport_center = self.origin - w - self.lower_left_corner = viewport_center - (self.horizontal * 0.5) - (self.vertical * 0.5) - - def get_ray(self, u_norm, v_norm): - direction = self.lower_left_corner + (self.horizontal * u_norm) + (self.vertical * v_norm) - self.origin - return Ray(self.origin, direction.normalize()) - -# --- Core Raytracing Logic (executed by Pymonik tasks) --- - -def trace_ray_for_pixel_color(ray, scene): - min_t = float('inf') - hit_object = None - for obj in scene.objects: - t = obj.intersect(ray) - if t is not None and t < min_t: - min_t = t - hit_object = obj - - if hit_object: - hit_point = ray.point_at_parameter(min_t) - normal = (hit_point - hit_object.center).normalize() - final_color = Vec3(0, 0, 0) - ambient_light_contribution = hit_object.material.color * hit_object.material.ambient - final_color += ambient_light_contribution - - for light in scene.lights: - light_dir = (light.position - hit_point).normalize() - diffuse_intensity_factor = max(0.0, normal.dot(light_dir)) - material_x_light_color = hit_object.material.color * light.color - diffuse_color_contribution = material_x_light_color * diffuse_intensity_factor * hit_object.material.diffuse * light.intensity - final_color += diffuse_color_contribution - - final_color.x = max(0.0, min(1.0, final_color.x)) - final_color.y = max(0.0, min(1.0, final_color.y)) - final_color.z = max(0.0, min(1.0, final_color.z)) - return final_color - else: - return scene.background_color - -# --- Pymonik Task --- - -@task -def render_tile_task(tile_y_start, tile_y_end, image_width, image_height, camera_obj, scene_obj): - """ - Renders a horizontal strip (tile) of the image. - Accepts scene and camera objects directly. - """ - tile_pixel_data = [] - - for y in range(tile_y_start, tile_y_end): - for x in range(image_width): - u_norm = (x + 0.5) / image_width - v_norm = (image_height - 1 - y + 0.5) / image_height # Flipped y - - # Use the get_ray method from the camera object - ray = camera_obj.get_ray(u_norm, v_norm) - - pixel_color_vec3 = trace_ray_for_pixel_color(ray, scene_obj) - tile_pixel_data.append(pixel_color_vec3.to_color()) - - return tile_y_start, tile_pixel_data - - -if __name__ == "__main__": - img_width = 8000 - img_height = 8000 - - material_red = Material(color=Vec3(1.0, 0.2, 0.2), ambient=0.1, diffuse=0.9) - material_green = Material(color=Vec3(0.2, 1.0, 0.2), ambient=0.1, diffuse=0.8) - material_blue = Material(color=Vec3(0.2, 0.2, 1.0), ambient=0.1, diffuse=0.7) - material_grey_floor = Material(color=Vec3(0.5, 0.5, 0.5), ambient=0.2, diffuse=0.9) - - scene_objects = [ - Sphere(center=Vec3(0, 0, -1), radius=0.5, material=material_red), - Sphere(center=Vec3(1.0, 0.2, -1.5), radius=0.7, material=material_green), - Sphere(center=Vec3(-1.2, -0.1, -2.0), radius=0.4, material=material_blue), - Sphere(center=Vec3(0, -100.5, -1), radius=100, material=material_grey_floor) - ] - scene_lights = [ - PointLight(position=Vec3(-2, 2, 1), color=Vec3(1.0, 1.0, 1.0), intensity=0.8), - PointLight(position=Vec3(2, 1, 0), color=Vec3(0.8, 0.8, 1.0), intensity=0.6) - ] - scene_background = Vec3(0.7, 0.8, 1.0) - - main_scene = Scene(scene_objects, scene_lights, scene_background) - - look_from = Vec3(0, 0.5, 1.5) - look_at = Vec3(0, 0, -1) - vup = Vec3(0, 1, 0) - vfov = 60.0 - aspect_ratio = img_width / img_height - main_camera = Camera(look_from, look_at, vup, vfov, aspect_ratio) - - with Pymonik(endpoint="localhost:5001"): - print("Successfully connected to Pymonik.") - - num_tasks = int(os.getenv("NUM_RAYTRACING_TASKS", "200")) - num_tasks = max(1, min(num_tasks, img_height)) - rows_per_task = math.ceil(img_height / num_tasks) - - task_args_list = [] - for i in range(num_tasks): - y_start = i * rows_per_task - y_end = min((i + 1) * rows_per_task, img_height) - if y_start >= y_end: - continue - # Pass main_camera and main_scene objects directly - task_args_list.append( - (y_start, y_end, img_width, img_height, main_camera, main_scene) - ) - - if not task_args_list: - print("Error: No tasks generated. Check image dimensions and num_tasks.") - if __name__ == '__main__': - exit() - - print(f"Submitting {len(task_args_list)} raytracing tasks to Pymonik...") - results_handle = render_tile_task.map_invoke(task_args_list) - - print("Waiting for tasks to complete...") - results_handle.wait() - print("All tasks completed. Fetching results...") - - final_image = Image.new("RGB", (img_width, img_height)) - rendered_tiles_data = {} - for task_idx in range(len(task_args_list)): - try: - tile_y_start, tile_pixel_data = results_handle[task_idx].get() - rendered_tiles_data[tile_y_start] = tile_pixel_data - expected_rows_for_tile = task_args_list[task_idx][1] - task_args_list[task_idx][0] - print(f" Retrieved tile starting at row {tile_y_start} ({len(tile_pixel_data)} pixels for {expected_rows_for_tile} rows)") - except Exception as e: - print(f" Error retrieving result for task {task_idx} (expected y_start {task_args_list[task_idx][0]}): {e}") - - print("Assembling final image...") - default_missing_tile_color = (255, 0, 255) - flat_pixel_list = [ default_missing_tile_color ] * (img_width * img_height) - - for original_task_args in task_args_list: - y_start_key = original_task_args[0] - expected_y_end = original_task_args[1] - - if y_start_key in rendered_tiles_data: - tile_pixels = rendered_tiles_data[y_start_key] - current_pixel_idx_in_tile = 0 - for y_offset in range(expected_y_end - y_start_key): - current_y = y_start_key + y_offset - if current_y >= img_height: continue - for x in range(img_width): - if current_pixel_idx_in_tile < len(tile_pixels): - flat_idx = current_y * img_width + x - if flat_idx < len(flat_pixel_list): - flat_pixel_list[flat_idx] = tile_pixels[current_pixel_idx_in_tile] - else: - print(f"Error: flat_idx {flat_idx} out of bounds for flat_pixel_list (len {len(flat_pixel_list)})") - else: - print(f"Warning: Missing pixel data in tile starting {y_start_key} at relative pixel {current_pixel_idx_in_tile} (x:{x}, y_offset:{y_offset})") - current_pixel_idx_in_tile +=1 - else: - print(f"Warning: Missing data for entire tile starting at row {y_start_key}. Will be filled with default color.") - - final_image.putdata(flat_pixel_list) - print(f"Final image assembled. Total pixels processed: {img_width * img_height}") - - output_filename = "pymonik_raytraced_image.png" - try: - final_image.save(output_filename) - print(f"Image saved as {output_filename}") - except Exception as e: - print(f"Error saving or showing image: {e}") - - print("Raytracing finished.") \ No newline at end of file diff --git a/examples/raytracing/uv.lock b/examples/raytracing/uv.lock deleted file mode 100644 index 64542ef..0000000 --- a/examples/raytracing/uv.lock +++ /dev/null @@ -1,545 +0,0 @@ -version = 1 -revision = 2 -requires-python = ">=3.10.12" - -[[package]] -name = "ale-py" -version = "0.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/e1/91489adc45e9979f090077893012d7115f57e3e5b088801a47401aaf5105/ale_py-0.11.0-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:f9c54aeac2cce17cb3907793d4da610bd8a0aff0d60e6aafb3a2b87aeea017c0", size = 2331915, upload-time = "2025-04-26T13:18:57.398Z" }, - { url = "https://files.pythonhosted.org/packages/b5/75/add33c5e6ae0216f31a3c0c6a4919c69a91a22869e7c45733358adfcc7cc/ale_py-0.11.0-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:815343c042f7e7249b724bdc3a2047186e2a3295c17b4a232ed784e7e942fad1", size = 2462115, upload-time = "2025-04-26T13:19:01.012Z" }, - { url = "https://files.pythonhosted.org/packages/84/20/0799138c4cb6f7ef5a25c5e635da9e8aa705b20e6f048659b39f15548eb1/ale_py-0.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5510d0db836656961bb3b9c0388334b285e1e37de9eaa8a01248e125b9f5c876", size = 4605870, upload-time = "2025-04-26T13:19:05.285Z" }, - { url = "https://files.pythonhosted.org/packages/05/55/bc10fd0ce4b4d55338e071c7558c170310a76f29b4d4f0d69c4714b3a098/ale_py-0.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:8f46bbed02dbd69536ec29ebff0520ade268290f08645d57ae7338461aa6a559", size = 3433750, upload-time = "2025-04-26T13:19:08.41Z" }, - { url = "https://files.pythonhosted.org/packages/75/80/ba1ebca77d7c3c12b046289a872797fedbba66ecf24fa71ccc9408fd9617/ale_py-0.11.0-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:7f080ed7e25fb423ad37814ad5881362af7c70f55fb8241a582200ce658c3234", size = 2332870, upload-time = "2025-04-26T13:19:11.067Z" }, - { url = "https://files.pythonhosted.org/packages/06/c8/5f147bd01258bd578ae6475b002356fdf18529e8aa8f2546bba53d7a4291/ale_py-0.11.0-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:3c6d45cca81b6cdafd61386e1e6f22a8d3d2fc0802c18fda13a5dcc6544e004c", size = 2463476, upload-time = "2025-04-26T13:19:13.415Z" }, - { url = "https://files.pythonhosted.org/packages/42/8b/ec7780f919985f546957c573d8303f7fc39f977ba596725b52960acd499c/ale_py-0.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42738511a759e1ef3658faff221ad0b53b212384e5225ec290ce5370695968fd", size = 4608692, upload-time = "2025-04-26T13:19:17.115Z" }, - { url = "https://files.pythonhosted.org/packages/15/f6/b443164051fefd52ed428ad8442cc7bc3541e87d83ee9797dee9370b1da1/ale_py-0.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:64843b4e8769dd26246606b30078a99a1d571d257fa05c638ebb7f9a3c8be421", size = 3435289, upload-time = "2025-04-26T13:19:20.638Z" }, - { url = "https://files.pythonhosted.org/packages/d2/71/b225d7bb2740ac4d5146da00b953d039d2be551ad368cbae746143a2cbbb/ale_py-0.11.0-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:95b2d483c0b0325c455183ec352e167f27986a20836af6ab8a0f986aa9f52d36", size = 2332867, upload-time = "2025-04-26T13:19:23.042Z" }, - { url = "https://files.pythonhosted.org/packages/f4/db/9f9766b297449ed572cfe3d962ca35d855aa0899cd2dd6721a407070ec94/ale_py-0.11.0-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:443c671452d864232e6c00159cb693ec2dfd1bed454dc450f05f0b2352d0d6ca", size = 2466161, upload-time = "2025-04-26T13:19:25.799Z" }, - { url = "https://files.pythonhosted.org/packages/25/f7/b27c3ded23c47ce9eef4e1ea5447d3032377b84d3a719a4a6becf979e6f2/ale_py-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efc7549d785b6901f749396b7b2f78e34e4d346652f7958559424efb61989ad3", size = 4608506, upload-time = "2025-04-26T13:19:29.064Z" }, - { url = "https://files.pythonhosted.org/packages/8c/82/c443ad9b77519eeb0a833bc96833514d7081061c520abc737d48711e1358/ale_py-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:d694b0292b6780a1e21e2fc640f465cd27e1619e787e1a683aa18ec11aa8a750", size = 3435460, upload-time = "2025-04-26T13:19:32.002Z" }, - { url = "https://files.pythonhosted.org/packages/08/96/577e13e11d6ced5f75bd25ea0a4906fc5926f4fb0f21f9adede3f88eff97/ale_py-0.11.0-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:7418731017070295ba02507522c3aa0c9e5ce202d4f73d03514c05989f6df67c", size = 2332892, upload-time = "2025-04-26T13:19:34.661Z" }, - { url = "https://files.pythonhosted.org/packages/90/b4/d84438c886565cf70b6c5564cb000e50939995e6432096a53095a4ddc544/ale_py-0.11.0-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:c3db4e0d97f6b048948595e19604bb533ac7a0fd2d2bdaa09e1c2b108bc5a4ce", size = 2466172, upload-time = "2025-04-26T13:19:36.972Z" }, - { url = "https://files.pythonhosted.org/packages/c7/e8/37629f955447561be6207acf9287e9b877b3c97170e472b0ee3a54e0830d/ale_py-0.11.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bd29dad12a683de80fea6c6d92871c985d39170741c40a0dd66cd2ff42fdae7", size = 4608094, upload-time = "2025-04-26T13:19:40.141Z" }, - { url = "https://files.pythonhosted.org/packages/d3/d4/2a8e1351a1ab5adec66247a9a4d7f0f354f8d8716e94c78a5165430b8b91/ale_py-0.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:85abda2b28ecf5d3b014ed176422e25e7a91e469e1f231aaaf60ba655fb7e24f", size = 3435567, upload-time = "2025-04-26T13:19:43.125Z" }, -] - -[[package]] -name = "armonik" -version = "3.25.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "deprecation" }, - { name = "grpcio" }, - { name = "grpcio-tools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8e/22/6299b9be5da13e6a909a3dc285e1b4b1b4990da337e598d4863acdc723aa/armonik-3.25.0.tar.gz", hash = "sha256:5fd5114da313b279d28fccdcb4cd1e6e1d0d302f5cf01b83d2ebd888ef1982c9", size = 89599, upload-time = "2025-03-06T14:39:21.779Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/2d/03ba8cad0f8bbe0850cdbee37bc904a70a005552c0091f904a0244223dc4/armonik-3.25.0-py3-none-any.whl", hash = "sha256:2c4f4428264bcf0b4016599441eeaad13f8947cfd3e3398b5a75ef4fe4d70483", size = 148256, upload-time = "2025-03-06T14:39:20.077Z" }, -] - -[[package]] -name = "cffi" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/07/f44ca684db4e4f08a3fdc6eeb9a0d15dc6883efc7b8c90357fdbf74e186c/cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14", size = 182191, upload-time = "2024-09-04T20:43:30.027Z" }, - { url = "https://files.pythonhosted.org/packages/08/fd/cc2fedbd887223f9f5d170c96e57cbf655df9831a6546c1727ae13fa977a/cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67", size = 178592, upload-time = "2024-09-04T20:43:32.108Z" }, - { url = "https://files.pythonhosted.org/packages/de/cc/4635c320081c78d6ffc2cab0a76025b691a91204f4aa317d568ff9280a2d/cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382", size = 426024, upload-time = "2024-09-04T20:43:34.186Z" }, - { url = "https://files.pythonhosted.org/packages/b6/7b/3b2b250f3aab91abe5f8a51ada1b717935fdaec53f790ad4100fe2ec64d1/cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702", size = 448188, upload-time = "2024-09-04T20:43:36.286Z" }, - { url = "https://files.pythonhosted.org/packages/d3/48/1b9283ebbf0ec065148d8de05d647a986c5f22586b18120020452fff8f5d/cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3", size = 455571, upload-time = "2024-09-04T20:43:38.586Z" }, - { url = "https://files.pythonhosted.org/packages/40/87/3b8452525437b40f39ca7ff70276679772ee7e8b394934ff60e63b7b090c/cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6", size = 436687, upload-time = "2024-09-04T20:43:40.084Z" }, - { url = "https://files.pythonhosted.org/packages/8d/fb/4da72871d177d63649ac449aec2e8a29efe0274035880c7af59101ca2232/cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17", size = 446211, upload-time = "2024-09-04T20:43:41.526Z" }, - { url = "https://files.pythonhosted.org/packages/ab/a0/62f00bcb411332106c02b663b26f3545a9ef136f80d5df746c05878f8c4b/cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8", size = 461325, upload-time = "2024-09-04T20:43:43.117Z" }, - { url = "https://files.pythonhosted.org/packages/36/83/76127035ed2e7e27b0787604d99da630ac3123bfb02d8e80c633f218a11d/cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e", size = 438784, upload-time = "2024-09-04T20:43:45.256Z" }, - { url = "https://files.pythonhosted.org/packages/21/81/a6cd025db2f08ac88b901b745c163d884641909641f9b826e8cb87645942/cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be", size = 461564, upload-time = "2024-09-04T20:43:46.779Z" }, - { url = "https://files.pythonhosted.org/packages/f8/fe/4d41c2f200c4a457933dbd98d3cf4e911870877bd94d9656cc0fcb390681/cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c", size = 171804, upload-time = "2024-09-04T20:43:48.186Z" }, - { url = "https://files.pythonhosted.org/packages/d1/b6/0b0f5ab93b0df4acc49cae758c81fe4e5ef26c3ae2e10cc69249dfd8b3ab/cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15", size = 181299, upload-time = "2024-09-04T20:43:49.812Z" }, - { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264, upload-time = "2024-09-04T20:43:51.124Z" }, - { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651, upload-time = "2024-09-04T20:43:52.872Z" }, - { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259, upload-time = "2024-09-04T20:43:56.123Z" }, - { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200, upload-time = "2024-09-04T20:43:57.891Z" }, - { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235, upload-time = "2024-09-04T20:44:00.18Z" }, - { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721, upload-time = "2024-09-04T20:44:01.585Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242, upload-time = "2024-09-04T20:44:03.467Z" }, - { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999, upload-time = "2024-09-04T20:44:05.023Z" }, - { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242, upload-time = "2024-09-04T20:44:06.444Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604, upload-time = "2024-09-04T20:44:08.206Z" }, - { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727, upload-time = "2024-09-04T20:44:09.481Z" }, - { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400, upload-time = "2024-09-04T20:44:10.873Z" }, - { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, - { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, - { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, - { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, - { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, - { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, - { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, - { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, - { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, - { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989, upload-time = "2024-09-04T20:44:28.956Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802, upload-time = "2024-09-04T20:44:30.289Z" }, - { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792, upload-time = "2024-09-04T20:44:32.01Z" }, - { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893, upload-time = "2024-09-04T20:44:33.606Z" }, - { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810, upload-time = "2024-09-04T20:44:35.191Z" }, - { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200, upload-time = "2024-09-04T20:44:36.743Z" }, - { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447, upload-time = "2024-09-04T20:44:38.492Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358, upload-time = "2024-09-04T20:44:40.046Z" }, - { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469, upload-time = "2024-09-04T20:44:41.616Z" }, - { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475, upload-time = "2024-09-04T20:44:43.733Z" }, - { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, -] - -[[package]] -name = "cloudpickle" -version = "3.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/39/069100b84d7418bc358d81669d5748efb14b9cceacd2f9c75f550424132f/cloudpickle-3.1.1.tar.gz", hash = "sha256:b216fa8ae4019d5482a8ac3c95d8f6346115d8835911fd4aefd1a445e4242c64", size = 22113, upload-time = "2025-01-14T17:02:05.085Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/e8/64c37fadfc2816a7701fa8a6ed8d87327c7d54eacfbfb6edab14a2f2be75/cloudpickle-3.1.1-py3-none-any.whl", hash = "sha256:c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e", size = 20992, upload-time = "2025-01-14T17:02:02.417Z" }, -] - -[[package]] -name = "cryptography" -version = "44.0.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/53/d6/1411ab4d6108ab167d06254c5be517681f1e331f90edf1379895bcb87020/cryptography-44.0.3.tar.gz", hash = "sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053", size = 711096, upload-time = "2025-05-02T19:36:04.667Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/08/53/c776d80e9d26441bb3868457909b4e74dd9ccabd182e10b2b0ae7a07e265/cryptography-44.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:962bc30480a08d133e631e8dfd4783ab71cc9e33d5d7c1e192f0b7c06397bb88", size = 6670281, upload-time = "2025-05-02T19:34:50.665Z" }, - { url = "https://files.pythonhosted.org/packages/6a/06/af2cf8d56ef87c77319e9086601bef621bedf40f6f59069e1b6d1ec498c5/cryptography-44.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffc61e8f3bf5b60346d89cd3d37231019c17a081208dfbbd6e1605ba03fa137", size = 3959305, upload-time = "2025-05-02T19:34:53.042Z" }, - { url = "https://files.pythonhosted.org/packages/ae/01/80de3bec64627207d030f47bf3536889efee8913cd363e78ca9a09b13c8e/cryptography-44.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58968d331425a6f9eedcee087f77fd3c927c88f55368f43ff7e0a19891f2642c", size = 4171040, upload-time = "2025-05-02T19:34:54.675Z" }, - { url = "https://files.pythonhosted.org/packages/bd/48/bb16b7541d207a19d9ae8b541c70037a05e473ddc72ccb1386524d4f023c/cryptography-44.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e28d62e59a4dbd1d22e747f57d4f00c459af22181f0b2f787ea83f5a876d7c76", size = 3963411, upload-time = "2025-05-02T19:34:56.61Z" }, - { url = "https://files.pythonhosted.org/packages/42/b2/7d31f2af5591d217d71d37d044ef5412945a8a8e98d5a2a8ae4fd9cd4489/cryptography-44.0.3-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:af653022a0c25ef2e3ffb2c673a50e5a0d02fecc41608f4954176f1933b12359", size = 3689263, upload-time = "2025-05-02T19:34:58.591Z" }, - { url = "https://files.pythonhosted.org/packages/25/50/c0dfb9d87ae88ccc01aad8eb93e23cfbcea6a6a106a9b63a7b14c1f93c75/cryptography-44.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:157f1f3b8d941c2bd8f3ffee0af9b049c9665c39d3da9db2dc338feca5e98a43", size = 4196198, upload-time = "2025-05-02T19:35:00.988Z" }, - { url = "https://files.pythonhosted.org/packages/66/c9/55c6b8794a74da652690c898cb43906310a3e4e4f6ee0b5f8b3b3e70c441/cryptography-44.0.3-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:c6cd67722619e4d55fdb42ead64ed8843d64638e9c07f4011163e46bc512cf01", size = 3966502, upload-time = "2025-05-02T19:35:03.091Z" }, - { url = "https://files.pythonhosted.org/packages/b6/f7/7cb5488c682ca59a02a32ec5f975074084db4c983f849d47b7b67cc8697a/cryptography-44.0.3-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b424563394c369a804ecbee9b06dfb34997f19d00b3518e39f83a5642618397d", size = 4196173, upload-time = "2025-05-02T19:35:05.018Z" }, - { url = "https://files.pythonhosted.org/packages/d2/0b/2f789a8403ae089b0b121f8f54f4a3e5228df756e2146efdf4a09a3d5083/cryptography-44.0.3-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c91fc8e8fd78af553f98bc7f2a1d8db977334e4eea302a4bfd75b9461c2d8904", size = 4087713, upload-time = "2025-05-02T19:35:07.187Z" }, - { url = "https://files.pythonhosted.org/packages/1d/aa/330c13655f1af398fc154089295cf259252f0ba5df93b4bc9d9c7d7f843e/cryptography-44.0.3-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:25cd194c39fa5a0aa4169125ee27d1172097857b27109a45fadc59653ec06f44", size = 4299064, upload-time = "2025-05-02T19:35:08.879Z" }, - { url = "https://files.pythonhosted.org/packages/10/a8/8c540a421b44fd267a7d58a1fd5f072a552d72204a3f08194f98889de76d/cryptography-44.0.3-cp37-abi3-win32.whl", hash = "sha256:3be3f649d91cb182c3a6bd336de8b61a0a71965bd13d1a04a0e15b39c3d5809d", size = 2773887, upload-time = "2025-05-02T19:35:10.41Z" }, - { url = "https://files.pythonhosted.org/packages/b9/0d/c4b1657c39ead18d76bbd122da86bd95bdc4095413460d09544000a17d56/cryptography-44.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:3883076d5c4cc56dbef0b898a74eb6992fdac29a7b9013870b34efe4ddb39a0d", size = 3209737, upload-time = "2025-05-02T19:35:12.12Z" }, - { url = "https://files.pythonhosted.org/packages/34/a3/ad08e0bcc34ad436013458d7528e83ac29910943cea42ad7dd4141a27bbb/cryptography-44.0.3-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:5639c2b16764c6f76eedf722dbad9a0914960d3489c0cc38694ddf9464f1bb2f", size = 6673501, upload-time = "2025-05-02T19:35:13.775Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f0/7491d44bba8d28b464a5bc8cc709f25a51e3eac54c0a4444cf2473a57c37/cryptography-44.0.3-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759", size = 3960307, upload-time = "2025-05-02T19:35:15.917Z" }, - { url = "https://files.pythonhosted.org/packages/f7/c8/e5c5d0e1364d3346a5747cdcd7ecbb23ca87e6dea4f942a44e88be349f06/cryptography-44.0.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:192ed30fac1728f7587c6f4613c29c584abdc565d7417c13904708db10206645", size = 4170876, upload-time = "2025-05-02T19:35:18.138Z" }, - { url = "https://files.pythonhosted.org/packages/73/96/025cb26fc351d8c7d3a1c44e20cf9a01e9f7cf740353c9c7a17072e4b264/cryptography-44.0.3-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7d5fe7195c27c32a64955740b949070f21cba664604291c298518d2e255931d2", size = 3964127, upload-time = "2025-05-02T19:35:19.864Z" }, - { url = "https://files.pythonhosted.org/packages/01/44/eb6522db7d9f84e8833ba3bf63313f8e257729cf3a8917379473fcfd6601/cryptography-44.0.3-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3f07943aa4d7dad689e3bb1638ddc4944cc5e0921e3c227486daae0e31a05e54", size = 3689164, upload-time = "2025-05-02T19:35:21.449Z" }, - { url = "https://files.pythonhosted.org/packages/68/fb/d61a4defd0d6cee20b1b8a1ea8f5e25007e26aeb413ca53835f0cae2bcd1/cryptography-44.0.3-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cb90f60e03d563ca2445099edf605c16ed1d5b15182d21831f58460c48bffb93", size = 4198081, upload-time = "2025-05-02T19:35:23.187Z" }, - { url = "https://files.pythonhosted.org/packages/1b/50/457f6911d36432a8811c3ab8bd5a6090e8d18ce655c22820994913dd06ea/cryptography-44.0.3-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:ab0b005721cc0039e885ac3503825661bd9810b15d4f374e473f8c89b7d5460c", size = 3967716, upload-time = "2025-05-02T19:35:25.426Z" }, - { url = "https://files.pythonhosted.org/packages/35/6e/dca39d553075980ccb631955c47b93d87d27f3596da8d48b1ae81463d915/cryptography-44.0.3-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:3bb0847e6363c037df8f6ede57d88eaf3410ca2267fb12275370a76f85786a6f", size = 4197398, upload-time = "2025-05-02T19:35:27.678Z" }, - { url = "https://files.pythonhosted.org/packages/9b/9d/d1f2fe681eabc682067c66a74addd46c887ebacf39038ba01f8860338d3d/cryptography-44.0.3-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b0cc66c74c797e1db750aaa842ad5b8b78e14805a9b5d1348dc603612d3e3ff5", size = 4087900, upload-time = "2025-05-02T19:35:29.312Z" }, - { url = "https://files.pythonhosted.org/packages/c4/f5/3599e48c5464580b73b236aafb20973b953cd2e7b44c7c2533de1d888446/cryptography-44.0.3-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6866df152b581f9429020320e5eb9794c8780e90f7ccb021940d7f50ee00ae0b", size = 4301067, upload-time = "2025-05-02T19:35:31.547Z" }, - { url = "https://files.pythonhosted.org/packages/a7/6c/d2c48c8137eb39d0c193274db5c04a75dab20d2f7c3f81a7dcc3a8897701/cryptography-44.0.3-cp39-abi3-win32.whl", hash = "sha256:c138abae3a12a94c75c10499f1cbae81294a6f983b3af066390adee73f433028", size = 2775467, upload-time = "2025-05-02T19:35:33.805Z" }, - { url = "https://files.pythonhosted.org/packages/c9/ad/51f212198681ea7b0deaaf8846ee10af99fba4e894f67b353524eab2bbe5/cryptography-44.0.3-cp39-abi3-win_amd64.whl", hash = "sha256:5d186f32e52e66994dce4f766884bcb9c68b8da62d61d9d215bfe5fb56d21334", size = 3210375, upload-time = "2025-05-02T19:35:35.369Z" }, - { url = "https://files.pythonhosted.org/packages/7f/10/abcf7418536df1eaba70e2cfc5c8a0ab07aa7aa02a5cbc6a78b9d8b4f121/cryptography-44.0.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:cad399780053fb383dc067475135e41c9fe7d901a97dd5d9c5dfb5611afc0d7d", size = 3393192, upload-time = "2025-05-02T19:35:37.468Z" }, - { url = "https://files.pythonhosted.org/packages/06/59/ecb3ef380f5891978f92a7f9120e2852b1df6f0a849c277b8ea45b865db2/cryptography-44.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:21a83f6f35b9cc656d71b5de8d519f566df01e660ac2578805ab245ffd8523f8", size = 3898419, upload-time = "2025-05-02T19:35:39.065Z" }, - { url = "https://files.pythonhosted.org/packages/bb/d0/35e2313dbb38cf793aa242182ad5bc5ef5c8fd4e5dbdc380b936c7d51169/cryptography-44.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4", size = 4117892, upload-time = "2025-05-02T19:35:40.839Z" }, - { url = "https://files.pythonhosted.org/packages/dc/c8/31fb6e33b56c2c2100d76de3fd820afaa9d4d0b6aea1ccaf9aaf35dc7ce3/cryptography-44.0.3-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:e909df4053064a97f1e6565153ff8bb389af12c5c8d29c343308760890560aff", size = 3900855, upload-time = "2025-05-02T19:35:42.599Z" }, - { url = "https://files.pythonhosted.org/packages/43/2a/08cc2ec19e77f2a3cfa2337b429676406d4bb78ddd130a05c458e7b91d73/cryptography-44.0.3-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:dad80b45c22e05b259e33ddd458e9e2ba099c86ccf4e88db7bbab4b747b18d06", size = 4117619, upload-time = "2025-05-02T19:35:44.774Z" }, - { url = "https://files.pythonhosted.org/packages/02/68/fc3d3f84022a75f2ac4b1a1c0e5d6a0c2ea259e14cd4aae3e0e68e56483c/cryptography-44.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:479d92908277bed6e1a1c69b277734a7771c2b78633c224445b5c60a9f4bc1d9", size = 3136570, upload-time = "2025-05-02T19:35:46.94Z" }, - { url = "https://files.pythonhosted.org/packages/8d/4b/c11ad0b6c061902de5223892d680e89c06c7c4d606305eb8de56c5427ae6/cryptography-44.0.3-pp311-pypy311_pp73-macosx_10_9_x86_64.whl", hash = "sha256:896530bc9107b226f265effa7ef3f21270f18a2026bc09fed1ebd7b66ddf6375", size = 3390230, upload-time = "2025-05-02T19:35:49.062Z" }, - { url = "https://files.pythonhosted.org/packages/58/11/0a6bf45d53b9b2290ea3cec30e78b78e6ca29dc101e2e296872a0ffe1335/cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9b4d4a5dbee05a2c390bf212e78b99434efec37b17a4bff42f50285c5c8c9647", size = 3895216, upload-time = "2025-05-02T19:35:51.351Z" }, - { url = "https://files.pythonhosted.org/packages/0a/27/b28cdeb7270e957f0077a2c2bfad1b38f72f1f6d699679f97b816ca33642/cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259", size = 4115044, upload-time = "2025-05-02T19:35:53.044Z" }, - { url = "https://files.pythonhosted.org/packages/35/b0/ec4082d3793f03cb248881fecefc26015813199b88f33e3e990a43f79835/cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:dd3db61b8fe5be220eee484a17233287d0be6932d056cf5738225b9c05ef4fff", size = 3898034, upload-time = "2025-05-02T19:35:54.72Z" }, - { url = "https://files.pythonhosted.org/packages/0b/7f/adf62e0b8e8d04d50c9a91282a57628c00c54d4ae75e2b02a223bd1f2613/cryptography-44.0.3-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:978631ec51a6bbc0b7e58f23b68a8ce9e5f09721940933e9c217068388789fe5", size = 4114449, upload-time = "2025-05-02T19:35:57.139Z" }, - { url = "https://files.pythonhosted.org/packages/87/62/d69eb4a8ee231f4bf733a92caf9da13f1c81a44e874b1d4080c25ecbb723/cryptography-44.0.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:5d20cc348cca3a8aa7312f42ab953a56e15323800ca3ab0706b8cd452a3a056c", size = 3134369, upload-time = "2025-05-02T19:35:58.907Z" }, -] - -[[package]] -name = "deprecation" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788, upload-time = "2020-04-20T14:23:38.738Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178, upload-time = "2020-04-20T14:23:36.581Z" }, -] - -[[package]] -name = "farama-notifications" -version = "0.0.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2e/2c/8384832b7a6b1fd6ba95bbdcae26e7137bb3eedc955c42fd5cdcc086cfbf/Farama-Notifications-0.0.4.tar.gz", hash = "sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18", size = 2131, upload-time = "2023-02-27T18:28:41.047Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/2c/ffc08c54c05cdce6fbed2aeebc46348dbe180c6d2c541c7af7ba0aa5f5f8/Farama_Notifications-0.0.4-py3-none-any.whl", hash = "sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae", size = 2511, upload-time = "2023-02-27T18:28:39.447Z" }, -] - -[[package]] -name = "grpcio" -version = "1.62.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/2e/1e2cd0edeaaeaae0ab9df2615492725d0f2f689d3f9aa3b088356af7a584/grpcio-1.62.3.tar.gz", hash = "sha256:4439bbd759636e37b66841117a66444b454937e27f0125205d2d117d7827c643", size = 26297933, upload-time = "2024-08-06T00:32:51.086Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/86/f1aa615ee551e8b4f59b0d1189a09e16eefb3d243487115ab7be56eecbec/grpcio-1.62.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:13571a5b868dcc308a55d36669a2d17d9dcd6ec8335213f6c49cc68da7305abe", size = 4766342, upload-time = "2024-08-06T00:20:32.775Z" }, - { url = "https://files.pythonhosted.org/packages/c5/63/ee244c4b64f0e71cef5314f9fa1d120c072e33c2e4c545dc75bd1af2a5c5/grpcio-1.62.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:f5def814c5a4c90c8fe389c526ab881f4a28b7e239b23ed8e02dd02934dfaa1a", size = 9991627, upload-time = "2024-08-06T00:20:35.662Z" }, - { url = "https://files.pythonhosted.org/packages/70/69/23bd58a27c472221fc340dd08eee2becf1a2c9d27d00e279c78a6b6f53cc/grpcio-1.62.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:7349cd7445ac65fbe1b744dcab9cc1ec02dae2256941a2e67895926cbf7422b4", size = 5290817, upload-time = "2024-08-06T00:20:38.718Z" }, - { url = "https://files.pythonhosted.org/packages/74/3a/875be32d9d1516398049ebc39cc6e7620d50d807093ce624f0469cee5e51/grpcio-1.62.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:646c14e9f3356d3f34a65b58b0f8d08daa741ba1d4fcd4966b79407543332154", size = 5829374, upload-time = "2024-08-06T00:20:41.631Z" }, - { url = "https://files.pythonhosted.org/packages/58/2f/f3fc773270cf17e7ca076c1f6435278f58641d475a25cdeea5b2d8d4845b/grpcio-1.62.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:807176971c504c598976f5a9ea62363cffbbbb6c7509d9808c2342b020880fa2", size = 5549649, upload-time = "2024-08-06T00:20:44.562Z" }, - { url = "https://files.pythonhosted.org/packages/b4/dc/deb8b1da1fa6111c3f44253433faf977678dea7dd381ce397ee33a1b4d8c/grpcio-1.62.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:43670a25b752b7ed960fcec3db50ae5886dc0df897269b3f5119cde9b731745f", size = 6113730, upload-time = "2024-08-06T00:20:47.107Z" }, - { url = "https://files.pythonhosted.org/packages/9d/e4/d3556f073563cea4aabfa340b08f462e8a748c7190f34a3467442d72ac48/grpcio-1.62.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:668211f3699bbee4deaf1d6e6b8df59328bf63f077bf2dc9b8bfa4a17df4a279", size = 5779201, upload-time = "2024-08-06T00:20:50.606Z" }, - { url = "https://files.pythonhosted.org/packages/15/c3/bf758db22525e1e3cd541f9bbfd33b248cf6866678a1285127cf5b6ec6a0/grpcio-1.62.3-cp310-cp310-win32.whl", hash = "sha256:216740723fc5971429550c374a0c039723b9d4dcaf7ba05227b7e0a500b06417", size = 3170437, upload-time = "2024-08-06T00:20:53.245Z" }, - { url = "https://files.pythonhosted.org/packages/d4/2f/1a4710440cc9e94a8d38af6dce0e670803a029ebc0f904929079a1c7ba58/grpcio-1.62.3-cp310-cp310-win_amd64.whl", hash = "sha256:b708401ede2c4cb8943e8a713988fcfe6cbea105b07cd7fa7c8a9f137c22bddb", size = 3730887, upload-time = "2024-08-06T00:20:56.018Z" }, - { url = "https://files.pythonhosted.org/packages/f5/3a/c3da1df7d55cfe481b02221d8e22e603f43fdf1646f2c02e7d69370d5e4b/grpcio-1.62.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:c8bb1a7aa82af6c7713cdf9dcb8f4ea1024ac7ce82bb0a0a82a49aea5237da34", size = 4774831, upload-time = "2024-08-06T00:20:58.985Z" }, - { url = "https://files.pythonhosted.org/packages/c8/12/c769a65437081cce5c7aece3fcc0a0e9e5293d85455cbdc9dd4edc9c56b9/grpcio-1.62.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:57823dc7299c4f258ae9c32fd327d29f729d359c34d7612b36e48ed45b3ab8d0", size = 10019810, upload-time = "2024-08-06T00:21:02.31Z" }, - { url = "https://files.pythonhosted.org/packages/18/08/b2a2c66f183240e55ca99a0dd85c2c2ad1cb0846e7ad628900843a49a155/grpcio-1.62.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:1de3d04d9a4ec31ebe848ae1fe61e4cbc367fb9495cbf6c54368e60609a998d9", size = 5294405, upload-time = "2024-08-06T00:21:05.586Z" }, - { url = "https://files.pythonhosted.org/packages/fe/ac/6e523ccaf068dc022de3cc798f539bd070fd45e4db953241cff23fd867a6/grpcio-1.62.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:325c56ce94d738c31059cf91376f625d3effdff8f85c96660a5fd6395d5a707f", size = 5830738, upload-time = "2024-08-06T00:21:08.512Z" }, - { url = "https://files.pythonhosted.org/packages/78/a0/3a1c81854f76d8c1462533e18cc754b9d3e434234678cb2273b1bff5885c/grpcio-1.62.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c175b252d063af388523a397dbe8edbc4319761f5ee892a8a0f5890acc067362", size = 5547176, upload-time = "2024-08-06T00:21:11.374Z" }, - { url = "https://files.pythonhosted.org/packages/7b/1a/9462e3c81429c4b299a7222df8cd3c1f84625eecc353967d5ddfec43f8f2/grpcio-1.62.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:25cd75dc73c5269932413e517be778640402f18cf9a81147e68645bd8af18ab0", size = 6116809, upload-time = "2024-08-06T00:21:13.99Z" }, - { url = "https://files.pythonhosted.org/packages/40/6c/99f922adeb6ca65b6f84f6bf1032f5b94c042eef65ab9b13d438819e9205/grpcio-1.62.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1b85d35a7d9638c03321dfe466645b87e23c30df1266f9e04bbb5f44e7579a9", size = 5779755, upload-time = "2024-08-06T00:21:17.029Z" }, - { url = "https://files.pythonhosted.org/packages/ae/b7/d48ff1a5f6ad59748df7717a3f101679e0e1b9004f5b6efa96831c2b847c/grpcio-1.62.3-cp311-cp311-win32.whl", hash = "sha256:6be243f3954b0ca709f56f9cae926c84ac96e1cce19844711e647a1f1db88b99", size = 3167821, upload-time = "2024-08-06T00:21:20.522Z" }, - { url = "https://files.pythonhosted.org/packages/9f/90/32ba95836adb03dd04a393f1b9a8bde7919e9f08ee5fd94dc77351f9305f/grpcio-1.62.3-cp311-cp311-win_amd64.whl", hash = "sha256:e9ffdb7bc9ccd56ec201aec3eab3432e1e820335b5a16ad2b37e094218dcd7a6", size = 3729044, upload-time = "2024-08-06T00:21:23.542Z" }, - { url = "https://files.pythonhosted.org/packages/33/3f/23748407c2fd739e983c366b805aeb86ed57c718f2619aa3a5856594ed67/grpcio-1.62.3-cp312-cp312-linux_armv7l.whl", hash = "sha256:4c9c1502c76cadbf2e145061b63af077b08d5677afcef91970d6db87b30e2f8b", size = 4733041, upload-time = "2024-08-06T00:21:26.788Z" }, - { url = "https://files.pythonhosted.org/packages/de/27/0f85db1ad84569c8c2f82c0d473b84dd09f8fe5e053298b1f35935b92d62/grpcio-1.62.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:abfe64811177e681edc81d9d9d1bd23edc5f599bd9846650864769264ace30cd", size = 9978073, upload-time = "2024-08-06T00:21:29.381Z" }, - { url = "https://files.pythonhosted.org/packages/4d/d1/df712e7f5cdd2676fde7a3459783f18dd8b6b8c6a201774551d431cfa50c/grpcio-1.62.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:3737e5ef0aa0fcdfeaf3b4ecc1a6be78b494549b28aec4b7f61b5dc357f7d8be", size = 5233910, upload-time = "2024-08-06T00:21:32.391Z" }, - { url = "https://files.pythonhosted.org/packages/12/b1/9e28e35382a4f29def23f7cbf5414a667d2249ce83eaf7024d31f88b0399/grpcio-1.62.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:940459d81685549afdfe13a6de102c52ea4cdda093477baa53056884aadf7c48", size = 5766972, upload-time = "2024-08-06T00:21:35.743Z" }, - { url = "https://files.pythonhosted.org/packages/82/2e/43218874d1852af1ea9801a2be62cc596ddd45984e7adba0fb9f66393c81/grpcio-1.62.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac9783d5679c8da612465168c820fd0b916e70ec5496c840bddba0be7f2d124c", size = 5492246, upload-time = "2024-08-06T00:21:38.978Z" }, - { url = "https://files.pythonhosted.org/packages/7a/59/8d83b5b52cf9a655633e36e7953899901fc93aefd15d3e1ff8129a7ef30e/grpcio-1.62.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:c95a0b76a44c548e6bd8c5f7dbecf89c77e2e16d3965be817b57769c4a30bea2", size = 6062547, upload-time = "2024-08-06T00:21:44.445Z" }, - { url = "https://files.pythonhosted.org/packages/99/8c/cf726cbee9a3e636adecc94a55136c72da8c36422c8c0173e0e3be535665/grpcio-1.62.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b097347441b86a8c3ad9579abaf5e5f7f82b1d74a898f47360433b2bca0e4536", size = 5729178, upload-time = "2024-08-06T00:21:48.757Z" }, - { url = "https://files.pythonhosted.org/packages/ea/a5/3a24d6d05da642e9d94902aa5c681ce9afd6f6af079d05a1d6d3aaa20cd6/grpcio-1.62.3-cp312-cp312-win32.whl", hash = "sha256:3fb7d966a976d762a31346353a19fce4afcffbeda3027dd563bc8cb521fcf799", size = 3156979, upload-time = "2024-08-06T00:21:51.846Z" }, - { url = "https://files.pythonhosted.org/packages/d7/9d/dc29922afbd0bb2616a14241508e6ee871b35f783a6b2e7104b44f82a2c6/grpcio-1.62.3-cp312-cp312-win_amd64.whl", hash = "sha256:454a6aed4ebd56198d37e1f3be6f1c70838e33dd62d1e2cea12f2bcb08efecc5", size = 3711573, upload-time = "2024-08-06T00:21:55.032Z" }, -] - -[[package]] -name = "grpcio-tools" -version = "1.62.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "grpcio" }, - { name = "protobuf" }, - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/54/fa/b69bd8040eafc09b88bb0ec0fea59e8aacd1a801e688af087cead213b0d0/grpcio-tools-1.62.3.tar.gz", hash = "sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833", size = 4538520, upload-time = "2024-08-06T00:37:11.035Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/eb/eb0a3aa9480c3689d31fd2ad536df6a828e97a60f667c8a93d05bdf07150/grpcio_tools-1.62.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2f968b049c2849540751ec2100ab05e8086c24bead769ca734fdab58698408c1", size = 5117556, upload-time = "2024-08-06T00:30:32.892Z" }, - { url = "https://files.pythonhosted.org/packages/f3/fb/8be3dda485f7fab906bfa02db321c3ecef953a87cdb5f6572ca08b187bcb/grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:0a8c0c4724ae9c2181b7dbc9b186df46e4f62cb18dc184e46d06c0ebeccf569e", size = 2719330, upload-time = "2024-08-06T00:30:36.113Z" }, - { url = "https://files.pythonhosted.org/packages/63/de/6978f8d10066e240141cd63d1fbfc92818d96bb53427074f47a8eda921e1/grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5782883a27d3fae8c425b29a9d3dcf5f47d992848a1b76970da3b5a28d424b26", size = 3070818, upload-time = "2024-08-06T00:30:38.797Z" }, - { url = "https://files.pythonhosted.org/packages/74/34/bb8f816893fc73fd6d830e895e8638d65d13642bb7a434f9175c5ca7da11/grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3d812daffd0c2d2794756bd45a353f89e55dc8f91eb2fc840c51b9f6be62667", size = 2804993, upload-time = "2024-08-06T00:30:41.72Z" }, - { url = "https://files.pythonhosted.org/packages/78/60/b2198d7db83293cdb9760fc083f077c73e4c182da06433b3b157a1567d06/grpcio_tools-1.62.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b47d0dda1bdb0a0ba7a9a6de88e5a1ed61f07fad613964879954961e36d49193", size = 3684915, upload-time = "2024-08-06T00:30:43.996Z" }, - { url = "https://files.pythonhosted.org/packages/61/20/56dbdc4ecb14d42a03cd164ff45e6e84572bbe61ee59c50c39f4d556a8d5/grpcio_tools-1.62.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ca246dffeca0498be9b4e1ee169b62e64694b0f92e6d0be2573e65522f39eea9", size = 3297482, upload-time = "2024-08-06T00:30:46.451Z" }, - { url = "https://files.pythonhosted.org/packages/4a/dc/e417a313c905744ce8cedf1e1edd81c41dc45ff400ae1c45080e18f26712/grpcio_tools-1.62.3-cp310-cp310-win32.whl", hash = "sha256:6a56d344b0bab30bf342a67e33d386b0b3c4e65868ffe93c341c51e1a8853ca5", size = 909793, upload-time = "2024-08-06T00:30:49.465Z" }, - { url = "https://files.pythonhosted.org/packages/d9/69/75e7ebfd8d755d3e7be5c6d1aa6d13220f5bba3a98965e4b50c329046777/grpcio_tools-1.62.3-cp310-cp310-win_amd64.whl", hash = "sha256:710fecf6a171dcbfa263a0a3e7070e0df65ba73158d4c539cec50978f11dad5d", size = 1052459, upload-time = "2024-08-06T00:30:51.98Z" }, - { url = "https://files.pythonhosted.org/packages/23/52/2dfe0a46b63f5ebcd976570aa5fc62f793d5a8b169e211c6a5aede72b7ae/grpcio_tools-1.62.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:703f46e0012af83a36082b5f30341113474ed0d91e36640da713355cd0ea5d23", size = 5147623, upload-time = "2024-08-06T00:30:54.894Z" }, - { url = "https://files.pythonhosted.org/packages/f0/2e/29fdc6c034e058482e054b4a3c2432f84ff2e2765c1342d4f0aa8a5c5b9a/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:7cc83023acd8bc72cf74c2edbe85b52098501d5b74d8377bfa06f3e929803492", size = 2719538, upload-time = "2024-08-06T00:30:57.928Z" }, - { url = "https://files.pythonhosted.org/packages/f9/60/abe5deba32d9ec2c76cdf1a2f34e404c50787074a2fee6169568986273f1/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ff7d58a45b75df67d25f8f144936a3e44aabd91afec833ee06826bd02b7fbe7", size = 3070964, upload-time = "2024-08-06T00:31:00.267Z" }, - { url = "https://files.pythonhosted.org/packages/bc/ad/e2b066684c75f8d9a48508cde080a3a36618064b9cadac16d019ca511444/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f2483ea232bd72d98a6dc6d7aefd97e5bc80b15cd909b9e356d6f3e326b6e43", size = 2805003, upload-time = "2024-08-06T00:31:02.565Z" }, - { url = "https://files.pythonhosted.org/packages/9c/3f/59bf7af786eae3f9d24ee05ce75318b87f541d0950190ecb5ffb776a1a58/grpcio_tools-1.62.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:962c84b4da0f3b14b3cdb10bc3837ebc5f136b67d919aea8d7bb3fd3df39528a", size = 3685154, upload-time = "2024-08-06T00:31:05.339Z" }, - { url = "https://files.pythonhosted.org/packages/f1/79/4dd62478b91e27084c67b35a2316ce8a967bd8b6cb8d6ed6c86c3a0df7cb/grpcio_tools-1.62.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8ad0473af5544f89fc5a1ece8676dd03bdf160fb3230f967e05d0f4bf89620e3", size = 3297942, upload-time = "2024-08-06T00:31:08.456Z" }, - { url = "https://files.pythonhosted.org/packages/b8/cb/86449ecc58bea056b52c0b891f26977afc8c4464d88c738f9648da941a75/grpcio_tools-1.62.3-cp311-cp311-win32.whl", hash = "sha256:db3bc9fa39afc5e4e2767da4459df82b095ef0cab2f257707be06c44a1c2c3e5", size = 910231, upload-time = "2024-08-06T00:31:11.464Z" }, - { url = "https://files.pythonhosted.org/packages/45/a4/9736215e3945c30ab6843280b0c6e1bff502910156ea2414cd77fbf1738c/grpcio_tools-1.62.3-cp311-cp311-win_amd64.whl", hash = "sha256:e0898d412a434e768a0c7e365acabe13ff1558b767e400936e26b5b6ed1ee51f", size = 1052496, upload-time = "2024-08-06T00:31:13.665Z" }, - { url = "https://files.pythonhosted.org/packages/2a/a5/d6887eba415ce318ae5005e8dfac3fa74892400b54b6d37b79e8b4f14f5e/grpcio_tools-1.62.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:d102b9b21c4e1e40af9a2ab3c6d41afba6bd29c0aa50ca013bf85c99cdc44ac5", size = 5147690, upload-time = "2024-08-06T00:31:16.436Z" }, - { url = "https://files.pythonhosted.org/packages/8a/7c/3cde447a045e83ceb4b570af8afe67ffc86896a2fe7f59594dc8e5d0a645/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:0a52cc9444df978438b8d2332c0ca99000521895229934a59f94f37ed896b133", size = 2720538, upload-time = "2024-08-06T00:31:18.905Z" }, - { url = "https://files.pythonhosted.org/packages/88/07/f83f2750d44ac4f06c07c37395b9c1383ef5c994745f73c6bfaf767f0944/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141d028bf5762d4a97f981c501da873589df3f7e02f4c1260e1921e565b376fa", size = 3071571, upload-time = "2024-08-06T00:31:21.684Z" }, - { url = "https://files.pythonhosted.org/packages/37/74/40175897deb61e54aca716bc2e8919155b48f33aafec8043dda9592d8768/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47a5c093ab256dec5714a7a345f8cc89315cb57c298b276fa244f37a0ba507f0", size = 2806207, upload-time = "2024-08-06T00:31:24.208Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ee/d8de915105a217cbcb9084d684abdc032030dcd887277f2ef167372287fe/grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d", size = 3685815, upload-time = "2024-08-06T00:31:26.917Z" }, - { url = "https://files.pythonhosted.org/packages/fd/d9/4360a6c12be3d7521b0b8c39e5d3801d622fbb81cc2721dbd3eee31e28c8/grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e02d7c1a02e3814c94ba0cfe43d93e872c758bd8fd5c2797f894d0c49b4a1dfc", size = 3298378, upload-time = "2024-08-06T00:31:30.401Z" }, - { url = "https://files.pythonhosted.org/packages/29/3b/7cdf4a9e5a3e0a35a528b48b111355cd14da601413a4f887aa99b6da468f/grpcio_tools-1.62.3-cp312-cp312-win32.whl", hash = "sha256:b881fd9505a84457e9f7e99362eeedd86497b659030cf57c6f0070df6d9c2b9b", size = 910416, upload-time = "2024-08-06T00:31:33.118Z" }, - { url = "https://files.pythonhosted.org/packages/6c/66/dd3ec249e44c1cc15e902e783747819ed41ead1336fcba72bf841f72c6e9/grpcio_tools-1.62.3-cp312-cp312-win_amd64.whl", hash = "sha256:11c625eebefd1fd40a228fc8bae385e448c7e32a6ae134e43cf13bbc23f902b7", size = 1052856, upload-time = "2024-08-06T00:31:36.519Z" }, -] - -[[package]] -name = "gymnasium" -version = "1.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cloudpickle" }, - { name = "farama-notifications" }, - { name = "numpy" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/90/69/70cd29e9fc4953d013b15981ee71d4c9ef4d8b2183e6ef2fe89756746dce/gymnasium-1.1.1.tar.gz", hash = "sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d", size = 829326, upload-time = "2025-03-06T16:30:36.428Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/68/2bdc7b46b5f543dd865575f9d19716866bdb76e50dd33b71ed1a3dd8bb42/gymnasium-1.1.1-py3-none-any.whl", hash = "sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a", size = 965410, upload-time = "2025-03-06T16:30:34.443Z" }, -] - -[package.optional-dependencies] -atari = [ - { name = "ale-py" }, -] - -[[package]] -name = "numpy" -version = "2.2.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/b2/ce4b867d8cd9c0ee84938ae1e6a6f7926ebf928c9090d036fc3c6a04f946/numpy-2.2.5.tar.gz", hash = "sha256:a9c0d994680cd991b1cb772e8b297340085466a6fe964bc9d4e80f5e2f43c291", size = 20273920, upload-time = "2025-04-19T23:27:42.561Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/4e/3d9e6d16237c2aa5485695f0626cbba82f6481efca2e9132368dea3b885e/numpy-2.2.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f4a922da1729f4c40932b2af4fe84909c7a6e167e6e99f71838ce3a29f3fe26", size = 21252117, upload-time = "2025-04-19T22:31:01.142Z" }, - { url = "https://files.pythonhosted.org/packages/38/e4/db91349d4079cd15c02ff3b4b8882a529991d6aca077db198a2f2a670406/numpy-2.2.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b6f91524d31b34f4a5fee24f5bc16dcd1491b668798b6d85585d836c1e633a6a", size = 14424615, upload-time = "2025-04-19T22:31:24.873Z" }, - { url = "https://files.pythonhosted.org/packages/f8/59/6e5b011f553c37b008bd115c7ba7106a18f372588fbb1b430b7a5d2c41ce/numpy-2.2.5-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:19f4718c9012e3baea91a7dba661dcab2451cda2550678dc30d53acb91a7290f", size = 5428691, upload-time = "2025-04-19T22:31:33.998Z" }, - { url = "https://files.pythonhosted.org/packages/a2/58/d5d70ebdac82b3a6ddf409b3749ca5786636e50fd64d60edb46442af6838/numpy-2.2.5-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:eb7fd5b184e5d277afa9ec0ad5e4eb562ecff541e7f60e69ee69c8d59e9aeaba", size = 6965010, upload-time = "2025-04-19T22:31:45.281Z" }, - { url = "https://files.pythonhosted.org/packages/dc/a8/c290394be346d4e7b48a40baf292626fd96ec56a6398ace4c25d9079bc6a/numpy-2.2.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6413d48a9be53e183eb06495d8e3b006ef8f87c324af68241bbe7a39e8ff54c3", size = 14369885, upload-time = "2025-04-19T22:32:06.557Z" }, - { url = "https://files.pythonhosted.org/packages/c2/70/fed13c70aabe7049368553e81d7ca40f305f305800a007a956d7cd2e5476/numpy-2.2.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7451f92eddf8503c9b8aa4fe6aa7e87fd51a29c2cfc5f7dbd72efde6c65acf57", size = 16418372, upload-time = "2025-04-19T22:32:31.716Z" }, - { url = "https://files.pythonhosted.org/packages/04/ab/c3c14f25ddaecd6fc58a34858f6a93a21eea6c266ba162fa99f3d0de12ac/numpy-2.2.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0bcb1d057b7571334139129b7f941588f69ce7c4ed15a9d6162b2ea54ded700c", size = 15883173, upload-time = "2025-04-19T22:32:55.106Z" }, - { url = "https://files.pythonhosted.org/packages/50/18/f53710a19042911c7aca824afe97c203728a34b8cf123e2d94621a12edc3/numpy-2.2.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:36ab5b23915887543441efd0417e6a3baa08634308894316f446027611b53bf1", size = 18206881, upload-time = "2025-04-19T22:33:22.08Z" }, - { url = "https://files.pythonhosted.org/packages/6b/ec/5b407bab82f10c65af5a5fe754728df03f960fd44d27c036b61f7b3ef255/numpy-2.2.5-cp310-cp310-win32.whl", hash = "sha256:422cc684f17bc963da5f59a31530b3936f57c95a29743056ef7a7903a5dbdf88", size = 6609852, upload-time = "2025-04-19T22:33:33.357Z" }, - { url = "https://files.pythonhosted.org/packages/b6/f5/467ca8675c7e6c567f571d8db942cc10a87588bd9e20a909d8af4171edda/numpy-2.2.5-cp310-cp310-win_amd64.whl", hash = "sha256:e4f0b035d9d0ed519c813ee23e0a733db81ec37d2e9503afbb6e54ccfdee0fa7", size = 12944922, upload-time = "2025-04-19T22:33:53.192Z" }, - { url = "https://files.pythonhosted.org/packages/f5/fb/e4e4c254ba40e8f0c78218f9e86304628c75b6900509b601c8433bdb5da7/numpy-2.2.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c42365005c7a6c42436a54d28c43fe0e01ca11eb2ac3cefe796c25a5f98e5e9b", size = 21256475, upload-time = "2025-04-19T22:34:24.174Z" }, - { url = "https://files.pythonhosted.org/packages/81/32/dd1f7084f5c10b2caad778258fdaeedd7fbd8afcd2510672811e6138dfac/numpy-2.2.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:498815b96f67dc347e03b719ef49c772589fb74b8ee9ea2c37feae915ad6ebda", size = 14461474, upload-time = "2025-04-19T22:34:46.578Z" }, - { url = "https://files.pythonhosted.org/packages/0e/65/937cdf238ef6ac54ff749c0f66d9ee2b03646034c205cea9b6c51f2f3ad1/numpy-2.2.5-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:6411f744f7f20081b1b4e7112e0f4c9c5b08f94b9f086e6f0adf3645f85d3a4d", size = 5426875, upload-time = "2025-04-19T22:34:56.281Z" }, - { url = "https://files.pythonhosted.org/packages/25/17/814515fdd545b07306eaee552b65c765035ea302d17de1b9cb50852d2452/numpy-2.2.5-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:9de6832228f617c9ef45d948ec1cd8949c482238d68b2477e6f642c33a7b0a54", size = 6969176, upload-time = "2025-04-19T22:35:07.518Z" }, - { url = "https://files.pythonhosted.org/packages/e5/32/a66db7a5c8b5301ec329ab36d0ecca23f5e18907f43dbd593c8ec326d57c/numpy-2.2.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:369e0d4647c17c9363244f3468f2227d557a74b6781cb62ce57cf3ef5cc7c610", size = 14374850, upload-time = "2025-04-19T22:35:31.347Z" }, - { url = "https://files.pythonhosted.org/packages/ad/c9/1bf6ada582eebcbe8978f5feb26584cd2b39f94ededeea034ca8f84af8c8/numpy-2.2.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:262d23f383170f99cd9191a7c85b9a50970fe9069b2f8ab5d786eca8a675d60b", size = 16430306, upload-time = "2025-04-19T22:35:57.573Z" }, - { url = "https://files.pythonhosted.org/packages/6a/f0/3f741863f29e128f4fcfdb99253cc971406b402b4584663710ee07f5f7eb/numpy-2.2.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aa70fdbdc3b169d69e8c59e65c07a1c9351ceb438e627f0fdcd471015cd956be", size = 15884767, upload-time = "2025-04-19T22:36:22.245Z" }, - { url = "https://files.pythonhosted.org/packages/98/d9/4ccd8fd6410f7bf2d312cbc98892e0e43c2fcdd1deae293aeb0a93b18071/numpy-2.2.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37e32e985f03c06206582a7323ef926b4e78bdaa6915095ef08070471865b906", size = 18219515, upload-time = "2025-04-19T22:36:49.822Z" }, - { url = "https://files.pythonhosted.org/packages/b1/56/783237243d4395c6dd741cf16eeb1a9035ee3d4310900e6b17e875d1b201/numpy-2.2.5-cp311-cp311-win32.whl", hash = "sha256:f5045039100ed58fa817a6227a356240ea1b9a1bc141018864c306c1a16d4175", size = 6607842, upload-time = "2025-04-19T22:37:01.624Z" }, - { url = "https://files.pythonhosted.org/packages/98/89/0c93baaf0094bdaaaa0536fe61a27b1dce8a505fa262a865ec142208cfe9/numpy-2.2.5-cp311-cp311-win_amd64.whl", hash = "sha256:b13f04968b46ad705f7c8a80122a42ae8f620536ea38cf4bdd374302926424dd", size = 12949071, upload-time = "2025-04-19T22:37:21.098Z" }, - { url = "https://files.pythonhosted.org/packages/e2/f7/1fd4ff108cd9d7ef929b8882692e23665dc9c23feecafbb9c6b80f4ec583/numpy-2.2.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ee461a4eaab4f165b68780a6a1af95fb23a29932be7569b9fab666c407969051", size = 20948633, upload-time = "2025-04-19T22:37:52.4Z" }, - { url = "https://files.pythonhosted.org/packages/12/03/d443c278348371b20d830af155ff2079acad6a9e60279fac2b41dbbb73d8/numpy-2.2.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ec31367fd6a255dc8de4772bd1658c3e926d8e860a0b6e922b615e532d320ddc", size = 14176123, upload-time = "2025-04-19T22:38:15.058Z" }, - { url = "https://files.pythonhosted.org/packages/2b/0b/5ca264641d0e7b14393313304da48b225d15d471250376f3fbdb1a2be603/numpy-2.2.5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:47834cde750d3c9f4e52c6ca28a7361859fcaf52695c7dc3cc1a720b8922683e", size = 5163817, upload-time = "2025-04-19T22:38:24.885Z" }, - { url = "https://files.pythonhosted.org/packages/04/b3/d522672b9e3d28e26e1613de7675b441bbd1eaca75db95680635dd158c67/numpy-2.2.5-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2c1a1c6ccce4022383583a6ded7bbcda22fc635eb4eb1e0a053336425ed36dfa", size = 6698066, upload-time = "2025-04-19T22:38:35.782Z" }, - { url = "https://files.pythonhosted.org/packages/a0/93/0f7a75c1ff02d4b76df35079676b3b2719fcdfb39abdf44c8b33f43ef37d/numpy-2.2.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d75f338f5f79ee23548b03d801d28a505198297534f62416391857ea0479571", size = 14087277, upload-time = "2025-04-19T22:38:57.697Z" }, - { url = "https://files.pythonhosted.org/packages/b0/d9/7c338b923c53d431bc837b5b787052fef9ae68a56fe91e325aac0d48226e/numpy-2.2.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a801fef99668f309b88640e28d261991bfad9617c27beda4a3aec4f217ea073", size = 16135742, upload-time = "2025-04-19T22:39:22.689Z" }, - { url = "https://files.pythonhosted.org/packages/2d/10/4dec9184a5d74ba9867c6f7d1e9f2e0fb5fe96ff2bf50bb6f342d64f2003/numpy-2.2.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:abe38cd8381245a7f49967a6010e77dbf3680bd3627c0fe4362dd693b404c7f8", size = 15581825, upload-time = "2025-04-19T22:39:45.794Z" }, - { url = "https://files.pythonhosted.org/packages/80/1f/2b6fcd636e848053f5b57712a7d1880b1565eec35a637fdfd0a30d5e738d/numpy-2.2.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5a0ac90e46fdb5649ab6369d1ab6104bfe5854ab19b645bf5cda0127a13034ae", size = 17899600, upload-time = "2025-04-19T22:40:13.427Z" }, - { url = "https://files.pythonhosted.org/packages/ec/87/36801f4dc2623d76a0a3835975524a84bd2b18fe0f8835d45c8eae2f9ff2/numpy-2.2.5-cp312-cp312-win32.whl", hash = "sha256:0cd48122a6b7eab8f06404805b1bd5856200e3ed6f8a1b9a194f9d9054631beb", size = 6312626, upload-time = "2025-04-19T22:40:25.223Z" }, - { url = "https://files.pythonhosted.org/packages/8b/09/4ffb4d6cfe7ca6707336187951992bd8a8b9142cf345d87ab858d2d7636a/numpy-2.2.5-cp312-cp312-win_amd64.whl", hash = "sha256:ced69262a8278547e63409b2653b372bf4baff0870c57efa76c5703fd6543282", size = 12645715, upload-time = "2025-04-19T22:40:44.528Z" }, - { url = "https://files.pythonhosted.org/packages/e2/a0/0aa7f0f4509a2e07bd7a509042967c2fab635690d4f48c6c7b3afd4f448c/numpy-2.2.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:059b51b658f4414fff78c6d7b1b4e18283ab5fa56d270ff212d5ba0c561846f4", size = 20935102, upload-time = "2025-04-19T22:41:16.234Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e4/a6a9f4537542912ec513185396fce52cdd45bdcf3e9d921ab02a93ca5aa9/numpy-2.2.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:47f9ed103af0bc63182609044b0490747e03bd20a67e391192dde119bf43d52f", size = 14191709, upload-time = "2025-04-19T22:41:38.472Z" }, - { url = "https://files.pythonhosted.org/packages/be/65/72f3186b6050bbfe9c43cb81f9df59ae63603491d36179cf7a7c8d216758/numpy-2.2.5-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:261a1ef047751bb02f29dfe337230b5882b54521ca121fc7f62668133cb119c9", size = 5149173, upload-time = "2025-04-19T22:41:47.823Z" }, - { url = "https://files.pythonhosted.org/packages/e5/e9/83e7a9432378dde5802651307ae5e9ea07bb72b416728202218cd4da2801/numpy-2.2.5-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4520caa3807c1ceb005d125a75e715567806fed67e315cea619d5ec6e75a4191", size = 6684502, upload-time = "2025-04-19T22:41:58.689Z" }, - { url = "https://files.pythonhosted.org/packages/ea/27/b80da6c762394c8ee516b74c1f686fcd16c8f23b14de57ba0cad7349d1d2/numpy-2.2.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d14b17b9be5f9c9301f43d2e2a4886a33b53f4e6fdf9ca2f4cc60aeeee76372", size = 14084417, upload-time = "2025-04-19T22:42:19.897Z" }, - { url = "https://files.pythonhosted.org/packages/aa/fc/ebfd32c3e124e6a1043e19c0ab0769818aa69050ce5589b63d05ff185526/numpy-2.2.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ba321813a00e508d5421104464510cc962a6f791aa2fca1c97b1e65027da80d", size = 16133807, upload-time = "2025-04-19T22:42:44.433Z" }, - { url = "https://files.pythonhosted.org/packages/bf/9b/4cc171a0acbe4666f7775cfd21d4eb6bb1d36d3a0431f48a73e9212d2278/numpy-2.2.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4cbdef3ddf777423060c6f81b5694bad2dc9675f110c4b2a60dc0181543fac7", size = 15575611, upload-time = "2025-04-19T22:43:09.928Z" }, - { url = "https://files.pythonhosted.org/packages/a3/45/40f4135341850df48f8edcf949cf47b523c404b712774f8855a64c96ef29/numpy-2.2.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:54088a5a147ab71a8e7fdfd8c3601972751ded0739c6b696ad9cb0343e21ab73", size = 17895747, upload-time = "2025-04-19T22:43:36.983Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4c/b32a17a46f0ffbde8cc82df6d3daeaf4f552e346df143e1b188a701a8f09/numpy-2.2.5-cp313-cp313-win32.whl", hash = "sha256:c8b82a55ef86a2d8e81b63da85e55f5537d2157165be1cb2ce7cfa57b6aef38b", size = 6309594, upload-time = "2025-04-19T22:47:10.523Z" }, - { url = "https://files.pythonhosted.org/packages/13/ae/72e6276feb9ef06787365b05915bfdb057d01fceb4a43cb80978e518d79b/numpy-2.2.5-cp313-cp313-win_amd64.whl", hash = "sha256:d8882a829fd779f0f43998e931c466802a77ca1ee0fe25a3abe50278616b1471", size = 12638356, upload-time = "2025-04-19T22:47:30.253Z" }, - { url = "https://files.pythonhosted.org/packages/79/56/be8b85a9f2adb688e7ded6324e20149a03541d2b3297c3ffc1a73f46dedb/numpy-2.2.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:e8b025c351b9f0e8b5436cf28a07fa4ac0204d67b38f01433ac7f9b870fa38c6", size = 20963778, upload-time = "2025-04-19T22:44:09.251Z" }, - { url = "https://files.pythonhosted.org/packages/ff/77/19c5e62d55bff507a18c3cdff82e94fe174957bad25860a991cac719d3ab/numpy-2.2.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8dfa94b6a4374e7851bbb6f35e6ded2120b752b063e6acdd3157e4d2bb922eba", size = 14207279, upload-time = "2025-04-19T22:44:31.383Z" }, - { url = "https://files.pythonhosted.org/packages/75/22/aa11f22dc11ff4ffe4e849d9b63bbe8d4ac6d5fae85ddaa67dfe43be3e76/numpy-2.2.5-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:97c8425d4e26437e65e1d189d22dff4a079b747ff9c2788057bfb8114ce1e133", size = 5199247, upload-time = "2025-04-19T22:44:40.361Z" }, - { url = "https://files.pythonhosted.org/packages/4f/6c/12d5e760fc62c08eded0394f62039f5a9857f758312bf01632a81d841459/numpy-2.2.5-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:352d330048c055ea6db701130abc48a21bec690a8d38f8284e00fab256dc1376", size = 6711087, upload-time = "2025-04-19T22:44:51.188Z" }, - { url = "https://files.pythonhosted.org/packages/ef/94/ece8280cf4218b2bee5cec9567629e61e51b4be501e5c6840ceb593db945/numpy-2.2.5-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b4c0773b6ada798f51f0f8e30c054d32304ccc6e9c5d93d46cb26f3d385ab19", size = 14059964, upload-time = "2025-04-19T22:45:12.451Z" }, - { url = "https://files.pythonhosted.org/packages/39/41/c5377dac0514aaeec69115830a39d905b1882819c8e65d97fc60e177e19e/numpy-2.2.5-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55f09e00d4dccd76b179c0f18a44f041e5332fd0e022886ba1c0bbf3ea4a18d0", size = 16121214, upload-time = "2025-04-19T22:45:37.734Z" }, - { url = "https://files.pythonhosted.org/packages/db/54/3b9f89a943257bc8e187145c6bc0eb8e3d615655f7b14e9b490b053e8149/numpy-2.2.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:02f226baeefa68f7d579e213d0f3493496397d8f1cff5e2b222af274c86a552a", size = 15575788, upload-time = "2025-04-19T22:46:01.908Z" }, - { url = "https://files.pythonhosted.org/packages/b1/c4/2e407e85df35b29f79945751b8f8e671057a13a376497d7fb2151ba0d290/numpy-2.2.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c26843fd58f65da9491165072da2cccc372530681de481ef670dcc8e27cfb066", size = 17893672, upload-time = "2025-04-19T22:46:28.585Z" }, - { url = "https://files.pythonhosted.org/packages/29/7e/d0b44e129d038dba453f00d0e29ebd6eaf2f06055d72b95b9947998aca14/numpy-2.2.5-cp313-cp313t-win32.whl", hash = "sha256:1a161c2c79ab30fe4501d5a2bbfe8b162490757cf90b7f05be8b80bc02f7bb8e", size = 6377102, upload-time = "2025-04-19T22:46:39.949Z" }, - { url = "https://files.pythonhosted.org/packages/63/be/b85e4aa4bf42c6502851b971f1c326d583fcc68227385f92089cf50a7b45/numpy-2.2.5-cp313-cp313t-win_amd64.whl", hash = "sha256:d403c84991b5ad291d3809bace5e85f4bbf44a04bdc9a88ed2bb1807b3360bb8", size = 12750096, upload-time = "2025-04-19T22:47:00.147Z" }, - { url = "https://files.pythonhosted.org/packages/35/e4/5ef5ef1d4308f96961198b2323bfc7c7afb0ccc0d623b01c79bc87ab496d/numpy-2.2.5-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b4ea7e1cff6784e58fe281ce7e7f05036b3e1c89c6f922a6bfbc0a7e8768adbe", size = 21083404, upload-time = "2025-04-19T22:48:01.605Z" }, - { url = "https://files.pythonhosted.org/packages/a3/5f/bde9238e8e977652a16a4b114ed8aa8bb093d718c706eeecb5f7bfa59572/numpy-2.2.5-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:d7543263084a85fbc09c704b515395398d31d6395518446237eac219eab9e55e", size = 6828578, upload-time = "2025-04-19T22:48:13.118Z" }, - { url = "https://files.pythonhosted.org/packages/ef/7f/813f51ed86e559ab2afb6a6f33aa6baf8a560097e25e4882a938986c76c2/numpy-2.2.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0255732338c4fdd00996c0421884ea8a3651eea555c3a56b84892b66f696eb70", size = 16234796, upload-time = "2025-04-19T22:48:37.102Z" }, - { url = "https://files.pythonhosted.org/packages/68/67/1175790323026d3337cc285cc9c50eca637d70472b5e622529df74bb8f37/numpy-2.2.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d2e3bdadaba0e040d1e7ab39db73e0afe2c74ae277f5614dad53eadbecbbb169", size = 12859001, upload-time = "2025-04-19T22:48:57.665Z" }, -] - -[[package]] -name = "packaging" -version = "25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, -] - -[[package]] -name = "pillow" -version = "11.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/af/cb/bb5c01fcd2a69335b86c22142b2bccfc3464087efb7fd382eee5ffc7fdf7/pillow-11.2.1.tar.gz", hash = "sha256:a64dd61998416367b7ef979b73d3a85853ba9bec4c2925f74e588879a58716b6", size = 47026707, upload-time = "2025-04-12T17:50:03.289Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/8b/b158ad57ed44d3cc54db8d68ad7c0a58b8fc0e4c7a3f995f9d62d5b464a1/pillow-11.2.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:d57a75d53922fc20c165016a20d9c44f73305e67c351bbc60d1adaf662e74047", size = 3198442, upload-time = "2025-04-12T17:47:10.666Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f8/bb5d956142f86c2d6cc36704943fa761f2d2e4c48b7436fd0a85c20f1713/pillow-11.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:127bf6ac4a5b58b3d32fc8289656f77f80567d65660bc46f72c0d77e6600cc95", size = 3030553, upload-time = "2025-04-12T17:47:13.153Z" }, - { url = "https://files.pythonhosted.org/packages/22/7f/0e413bb3e2aa797b9ca2c5c38cb2e2e45d88654e5b12da91ad446964cfae/pillow-11.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4ba4be812c7a40280629e55ae0b14a0aafa150dd6451297562e1764808bbe61", size = 4405503, upload-time = "2025-04-12T17:47:15.36Z" }, - { url = "https://files.pythonhosted.org/packages/f3/b4/cc647f4d13f3eb837d3065824aa58b9bcf10821f029dc79955ee43f793bd/pillow-11.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8bd62331e5032bc396a93609982a9ab6b411c05078a52f5fe3cc59234a3abd1", size = 4490648, upload-time = "2025-04-12T17:47:17.37Z" }, - { url = "https://files.pythonhosted.org/packages/c2/6f/240b772a3b35cdd7384166461567aa6713799b4e78d180c555bd284844ea/pillow-11.2.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:562d11134c97a62fe3af29581f083033179f7ff435f78392565a1ad2d1c2c45c", size = 4508937, upload-time = "2025-04-12T17:47:19.066Z" }, - { url = "https://files.pythonhosted.org/packages/f3/5e/7ca9c815ade5fdca18853db86d812f2f188212792780208bdb37a0a6aef4/pillow-11.2.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c97209e85b5be259994eb5b69ff50c5d20cca0f458ef9abd835e262d9d88b39d", size = 4599802, upload-time = "2025-04-12T17:47:21.404Z" }, - { url = "https://files.pythonhosted.org/packages/02/81/c3d9d38ce0c4878a77245d4cf2c46d45a4ad0f93000227910a46caff52f3/pillow-11.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0c3e6d0f59171dfa2e25d7116217543310908dfa2770aa64b8f87605f8cacc97", size = 4576717, upload-time = "2025-04-12T17:47:23.571Z" }, - { url = "https://files.pythonhosted.org/packages/42/49/52b719b89ac7da3185b8d29c94d0e6aec8140059e3d8adcaa46da3751180/pillow-11.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cc1c3bc53befb6096b84165956e886b1729634a799e9d6329a0c512ab651e579", size = 4654874, upload-time = "2025-04-12T17:47:25.783Z" }, - { url = "https://files.pythonhosted.org/packages/5b/0b/ede75063ba6023798267023dc0d0401f13695d228194d2242d5a7ba2f964/pillow-11.2.1-cp310-cp310-win32.whl", hash = "sha256:312c77b7f07ab2139924d2639860e084ec2a13e72af54d4f08ac843a5fc9c79d", size = 2331717, upload-time = "2025-04-12T17:47:28.922Z" }, - { url = "https://files.pythonhosted.org/packages/ed/3c/9831da3edea527c2ed9a09f31a2c04e77cd705847f13b69ca60269eec370/pillow-11.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:9bc7ae48b8057a611e5fe9f853baa88093b9a76303937449397899385da06fad", size = 2676204, upload-time = "2025-04-12T17:47:31.283Z" }, - { url = "https://files.pythonhosted.org/packages/01/97/1f66ff8a1503d8cbfc5bae4dc99d54c6ec1e22ad2b946241365320caabc2/pillow-11.2.1-cp310-cp310-win_arm64.whl", hash = "sha256:2728567e249cdd939f6cc3d1f049595c66e4187f3c34078cbc0a7d21c47482d2", size = 2414767, upload-time = "2025-04-12T17:47:34.655Z" }, - { url = "https://files.pythonhosted.org/packages/68/08/3fbf4b98924c73037a8e8b4c2c774784805e0fb4ebca6c5bb60795c40125/pillow-11.2.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:35ca289f712ccfc699508c4658a1d14652e8033e9b69839edf83cbdd0ba39e70", size = 3198450, upload-time = "2025-04-12T17:47:37.135Z" }, - { url = "https://files.pythonhosted.org/packages/84/92/6505b1af3d2849d5e714fc75ba9e69b7255c05ee42383a35a4d58f576b16/pillow-11.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0409af9f829f87a2dfb7e259f78f317a5351f2045158be321fd135973fff7bf", size = 3030550, upload-time = "2025-04-12T17:47:39.345Z" }, - { url = "https://files.pythonhosted.org/packages/3c/8c/ac2f99d2a70ff966bc7eb13dacacfaab57c0549b2ffb351b6537c7840b12/pillow-11.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4e5c5edee874dce4f653dbe59db7c73a600119fbea8d31f53423586ee2aafd7", size = 4415018, upload-time = "2025-04-12T17:47:41.128Z" }, - { url = "https://files.pythonhosted.org/packages/1f/e3/0a58b5d838687f40891fff9cbaf8669f90c96b64dc8f91f87894413856c6/pillow-11.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b93a07e76d13bff9444f1a029e0af2964e654bfc2e2c2d46bfd080df5ad5f3d8", size = 4498006, upload-time = "2025-04-12T17:47:42.912Z" }, - { url = "https://files.pythonhosted.org/packages/21/f5/6ba14718135f08fbfa33308efe027dd02b781d3f1d5c471444a395933aac/pillow-11.2.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:e6def7eed9e7fa90fde255afaf08060dc4b343bbe524a8f69bdd2a2f0018f600", size = 4517773, upload-time = "2025-04-12T17:47:44.611Z" }, - { url = "https://files.pythonhosted.org/packages/20/f2/805ad600fc59ebe4f1ba6129cd3a75fb0da126975c8579b8f57abeb61e80/pillow-11.2.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:8f4f3724c068be008c08257207210c138d5f3731af6c155a81c2b09a9eb3a788", size = 4607069, upload-time = "2025-04-12T17:47:46.46Z" }, - { url = "https://files.pythonhosted.org/packages/71/6b/4ef8a288b4bb2e0180cba13ca0a519fa27aa982875882392b65131401099/pillow-11.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a0a6709b47019dff32e678bc12c63008311b82b9327613f534e496dacaefb71e", size = 4583460, upload-time = "2025-04-12T17:47:49.255Z" }, - { url = "https://files.pythonhosted.org/packages/62/ae/f29c705a09cbc9e2a456590816e5c234382ae5d32584f451c3eb41a62062/pillow-11.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f6b0c664ccb879109ee3ca702a9272d877f4fcd21e5eb63c26422fd6e415365e", size = 4661304, upload-time = "2025-04-12T17:47:51.067Z" }, - { url = "https://files.pythonhosted.org/packages/6e/1a/c8217b6f2f73794a5e219fbad087701f412337ae6dbb956db37d69a9bc43/pillow-11.2.1-cp311-cp311-win32.whl", hash = "sha256:cc5d875d56e49f112b6def6813c4e3d3036d269c008bf8aef72cd08d20ca6df6", size = 2331809, upload-time = "2025-04-12T17:47:54.425Z" }, - { url = "https://files.pythonhosted.org/packages/e2/72/25a8f40170dc262e86e90f37cb72cb3de5e307f75bf4b02535a61afcd519/pillow-11.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:0f5c7eda47bf8e3c8a283762cab94e496ba977a420868cb819159980b6709193", size = 2676338, upload-time = "2025-04-12T17:47:56.535Z" }, - { url = "https://files.pythonhosted.org/packages/06/9e/76825e39efee61efea258b479391ca77d64dbd9e5804e4ad0fa453b4ba55/pillow-11.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:4d375eb838755f2528ac8cbc926c3e31cc49ca4ad0cf79cff48b20e30634a4a7", size = 2414918, upload-time = "2025-04-12T17:47:58.217Z" }, - { url = "https://files.pythonhosted.org/packages/c7/40/052610b15a1b8961f52537cc8326ca6a881408bc2bdad0d852edeb6ed33b/pillow-11.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:78afba22027b4accef10dbd5eed84425930ba41b3ea0a86fa8d20baaf19d807f", size = 3190185, upload-time = "2025-04-12T17:48:00.417Z" }, - { url = "https://files.pythonhosted.org/packages/e5/7e/b86dbd35a5f938632093dc40d1682874c33dcfe832558fc80ca56bfcb774/pillow-11.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78092232a4ab376a35d68c4e6d5e00dfd73454bd12b230420025fbe178ee3b0b", size = 3030306, upload-time = "2025-04-12T17:48:02.391Z" }, - { url = "https://files.pythonhosted.org/packages/a4/5c/467a161f9ed53e5eab51a42923c33051bf8d1a2af4626ac04f5166e58e0c/pillow-11.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25a5f306095c6780c52e6bbb6109624b95c5b18e40aab1c3041da3e9e0cd3e2d", size = 4416121, upload-time = "2025-04-12T17:48:04.554Z" }, - { url = "https://files.pythonhosted.org/packages/62/73/972b7742e38ae0e2ac76ab137ca6005dcf877480da0d9d61d93b613065b4/pillow-11.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c7b29dbd4281923a2bfe562acb734cee96bbb129e96e6972d315ed9f232bef4", size = 4501707, upload-time = "2025-04-12T17:48:06.831Z" }, - { url = "https://files.pythonhosted.org/packages/e4/3a/427e4cb0b9e177efbc1a84798ed20498c4f233abde003c06d2650a6d60cb/pillow-11.2.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3e645b020f3209a0181a418bffe7b4a93171eef6c4ef6cc20980b30bebf17b7d", size = 4522921, upload-time = "2025-04-12T17:48:09.229Z" }, - { url = "https://files.pythonhosted.org/packages/fe/7c/d8b1330458e4d2f3f45d9508796d7caf0c0d3764c00c823d10f6f1a3b76d/pillow-11.2.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b2dbea1012ccb784a65349f57bbc93730b96e85b42e9bf7b01ef40443db720b4", size = 4612523, upload-time = "2025-04-12T17:48:11.631Z" }, - { url = "https://files.pythonhosted.org/packages/b3/2f/65738384e0b1acf451de5a573d8153fe84103772d139e1e0bdf1596be2ea/pillow-11.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:da3104c57bbd72948d75f6a9389e6727d2ab6333c3617f0a89d72d4940aa0443", size = 4587836, upload-time = "2025-04-12T17:48:13.592Z" }, - { url = "https://files.pythonhosted.org/packages/6a/c5/e795c9f2ddf3debb2dedd0df889f2fe4b053308bb59a3cc02a0cd144d641/pillow-11.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:598174aef4589af795f66f9caab87ba4ff860ce08cd5bb447c6fc553ffee603c", size = 4669390, upload-time = "2025-04-12T17:48:15.938Z" }, - { url = "https://files.pythonhosted.org/packages/96/ae/ca0099a3995976a9fce2f423166f7bff9b12244afdc7520f6ed38911539a/pillow-11.2.1-cp312-cp312-win32.whl", hash = "sha256:1d535df14716e7f8776b9e7fee118576d65572b4aad3ed639be9e4fa88a1cad3", size = 2332309, upload-time = "2025-04-12T17:48:17.885Z" }, - { url = "https://files.pythonhosted.org/packages/7c/18/24bff2ad716257fc03da964c5e8f05d9790a779a8895d6566e493ccf0189/pillow-11.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:14e33b28bf17c7a38eede290f77db7c664e4eb01f7869e37fa98a5aa95978941", size = 2676768, upload-time = "2025-04-12T17:48:19.655Z" }, - { url = "https://files.pythonhosted.org/packages/da/bb/e8d656c9543276517ee40184aaa39dcb41e683bca121022f9323ae11b39d/pillow-11.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:21e1470ac9e5739ff880c211fc3af01e3ae505859392bf65458c224d0bf283eb", size = 2415087, upload-time = "2025-04-12T17:48:21.991Z" }, - { url = "https://files.pythonhosted.org/packages/36/9c/447528ee3776e7ab8897fe33697a7ff3f0475bb490c5ac1456a03dc57956/pillow-11.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fdec757fea0b793056419bca3e9932eb2b0ceec90ef4813ea4c1e072c389eb28", size = 3190098, upload-time = "2025-04-12T17:48:23.915Z" }, - { url = "https://files.pythonhosted.org/packages/b5/09/29d5cd052f7566a63e5b506fac9c60526e9ecc553825551333e1e18a4858/pillow-11.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b0e130705d568e2f43a17bcbe74d90958e8a16263868a12c3e0d9c8162690830", size = 3030166, upload-time = "2025-04-12T17:48:25.738Z" }, - { url = "https://files.pythonhosted.org/packages/71/5d/446ee132ad35e7600652133f9c2840b4799bbd8e4adba881284860da0a36/pillow-11.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bdb5e09068332578214cadd9c05e3d64d99e0e87591be22a324bdbc18925be0", size = 4408674, upload-time = "2025-04-12T17:48:27.908Z" }, - { url = "https://files.pythonhosted.org/packages/69/5f/cbe509c0ddf91cc3a03bbacf40e5c2339c4912d16458fcb797bb47bcb269/pillow-11.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d189ba1bebfbc0c0e529159631ec72bb9e9bc041f01ec6d3233d6d82eb823bc1", size = 4496005, upload-time = "2025-04-12T17:48:29.888Z" }, - { url = "https://files.pythonhosted.org/packages/f9/b3/dd4338d8fb8a5f312021f2977fb8198a1184893f9b00b02b75d565c33b51/pillow-11.2.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:191955c55d8a712fab8934a42bfefbf99dd0b5875078240943f913bb66d46d9f", size = 4518707, upload-time = "2025-04-12T17:48:31.874Z" }, - { url = "https://files.pythonhosted.org/packages/13/eb/2552ecebc0b887f539111c2cd241f538b8ff5891b8903dfe672e997529be/pillow-11.2.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:ad275964d52e2243430472fc5d2c2334b4fc3ff9c16cb0a19254e25efa03a155", size = 4610008, upload-time = "2025-04-12T17:48:34.422Z" }, - { url = "https://files.pythonhosted.org/packages/72/d1/924ce51bea494cb6e7959522d69d7b1c7e74f6821d84c63c3dc430cbbf3b/pillow-11.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:750f96efe0597382660d8b53e90dd1dd44568a8edb51cb7f9d5d918b80d4de14", size = 4585420, upload-time = "2025-04-12T17:48:37.641Z" }, - { url = "https://files.pythonhosted.org/packages/43/ab/8f81312d255d713b99ca37479a4cb4b0f48195e530cdc1611990eb8fd04b/pillow-11.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fe15238d3798788d00716637b3d4e7bb6bde18b26e5d08335a96e88564a36b6b", size = 4667655, upload-time = "2025-04-12T17:48:39.652Z" }, - { url = "https://files.pythonhosted.org/packages/94/86/8f2e9d2dc3d308dfd137a07fe1cc478df0a23d42a6c4093b087e738e4827/pillow-11.2.1-cp313-cp313-win32.whl", hash = "sha256:3fe735ced9a607fee4f481423a9c36701a39719252a9bb251679635f99d0f7d2", size = 2332329, upload-time = "2025-04-12T17:48:41.765Z" }, - { url = "https://files.pythonhosted.org/packages/6d/ec/1179083b8d6067a613e4d595359b5fdea65d0a3b7ad623fee906e1b3c4d2/pillow-11.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:74ee3d7ecb3f3c05459ba95eed5efa28d6092d751ce9bf20e3e253a4e497e691", size = 2676388, upload-time = "2025-04-12T17:48:43.625Z" }, - { url = "https://files.pythonhosted.org/packages/23/f1/2fc1e1e294de897df39fa8622d829b8828ddad938b0eaea256d65b84dd72/pillow-11.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:5119225c622403afb4b44bad4c1ca6c1f98eed79db8d3bc6e4e160fc6339d66c", size = 2414950, upload-time = "2025-04-12T17:48:45.475Z" }, - { url = "https://files.pythonhosted.org/packages/c4/3e/c328c48b3f0ead7bab765a84b4977acb29f101d10e4ef57a5e3400447c03/pillow-11.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:8ce2e8411c7aaef53e6bb29fe98f28cd4fbd9a1d9be2eeea434331aac0536b22", size = 3192759, upload-time = "2025-04-12T17:48:47.866Z" }, - { url = "https://files.pythonhosted.org/packages/18/0e/1c68532d833fc8b9f404d3a642991441d9058eccd5606eab31617f29b6d4/pillow-11.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9ee66787e095127116d91dea2143db65c7bb1e232f617aa5957c0d9d2a3f23a7", size = 3033284, upload-time = "2025-04-12T17:48:50.189Z" }, - { url = "https://files.pythonhosted.org/packages/b7/cb/6faf3fb1e7705fd2db74e070f3bf6f88693601b0ed8e81049a8266de4754/pillow-11.2.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9622e3b6c1d8b551b6e6f21873bdcc55762b4b2126633014cea1803368a9aa16", size = 4445826, upload-time = "2025-04-12T17:48:52.346Z" }, - { url = "https://files.pythonhosted.org/packages/07/94/8be03d50b70ca47fb434a358919d6a8d6580f282bbb7af7e4aa40103461d/pillow-11.2.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63b5dff3a68f371ea06025a1a6966c9a1e1ee452fc8020c2cd0ea41b83e9037b", size = 4527329, upload-time = "2025-04-12T17:48:54.403Z" }, - { url = "https://files.pythonhosted.org/packages/fd/a4/bfe78777076dc405e3bd2080bc32da5ab3945b5a25dc5d8acaa9de64a162/pillow-11.2.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:31df6e2d3d8fc99f993fd253e97fae451a8db2e7207acf97859732273e108406", size = 4549049, upload-time = "2025-04-12T17:48:56.383Z" }, - { url = "https://files.pythonhosted.org/packages/65/4d/eaf9068dc687c24979e977ce5677e253624bd8b616b286f543f0c1b91662/pillow-11.2.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:062b7a42d672c45a70fa1f8b43d1d38ff76b63421cbbe7f88146b39e8a558d91", size = 4635408, upload-time = "2025-04-12T17:48:58.782Z" }, - { url = "https://files.pythonhosted.org/packages/1d/26/0fd443365d9c63bc79feb219f97d935cd4b93af28353cba78d8e77b61719/pillow-11.2.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4eb92eca2711ef8be42fd3f67533765d9fd043b8c80db204f16c8ea62ee1a751", size = 4614863, upload-time = "2025-04-12T17:49:00.709Z" }, - { url = "https://files.pythonhosted.org/packages/49/65/dca4d2506be482c2c6641cacdba5c602bc76d8ceb618fd37de855653a419/pillow-11.2.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f91ebf30830a48c825590aede79376cb40f110b387c17ee9bd59932c961044f9", size = 4692938, upload-time = "2025-04-12T17:49:02.946Z" }, - { url = "https://files.pythonhosted.org/packages/b3/92/1ca0c3f09233bd7decf8f7105a1c4e3162fb9142128c74adad0fb361b7eb/pillow-11.2.1-cp313-cp313t-win32.whl", hash = "sha256:e0b55f27f584ed623221cfe995c912c61606be8513bfa0e07d2c674b4516d9dd", size = 2335774, upload-time = "2025-04-12T17:49:04.889Z" }, - { url = "https://files.pythonhosted.org/packages/a5/ac/77525347cb43b83ae905ffe257bbe2cc6fd23acb9796639a1f56aa59d191/pillow-11.2.1-cp313-cp313t-win_amd64.whl", hash = "sha256:36d6b82164c39ce5482f649b437382c0fb2395eabc1e2b1702a6deb8ad647d6e", size = 2681895, upload-time = "2025-04-12T17:49:06.635Z" }, - { url = "https://files.pythonhosted.org/packages/67/32/32dc030cfa91ca0fc52baebbba2e009bb001122a1daa8b6a79ad830b38d3/pillow-11.2.1-cp313-cp313t-win_arm64.whl", hash = "sha256:225c832a13326e34f212d2072982bb1adb210e0cc0b153e688743018c94a2681", size = 2417234, upload-time = "2025-04-12T17:49:08.399Z" }, - { url = "https://files.pythonhosted.org/packages/33/49/c8c21e4255b4f4a2c0c68ac18125d7f5460b109acc6dfdef1a24f9b960ef/pillow-11.2.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:9b7b0d4fd2635f54ad82785d56bc0d94f147096493a79985d0ab57aedd563156", size = 3181727, upload-time = "2025-04-12T17:49:31.898Z" }, - { url = "https://files.pythonhosted.org/packages/6d/f1/f7255c0838f8c1ef6d55b625cfb286835c17e8136ce4351c5577d02c443b/pillow-11.2.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:aa442755e31c64037aa7c1cb186e0b369f8416c567381852c63444dd666fb772", size = 2999833, upload-time = "2025-04-12T17:49:34.2Z" }, - { url = "https://files.pythonhosted.org/packages/e2/57/9968114457bd131063da98d87790d080366218f64fa2943b65ac6739abb3/pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0d3348c95b766f54b76116d53d4cb171b52992a1027e7ca50c81b43b9d9e363", size = 3437472, upload-time = "2025-04-12T17:49:36.294Z" }, - { url = "https://files.pythonhosted.org/packages/b2/1b/e35d8a158e21372ecc48aac9c453518cfe23907bb82f950d6e1c72811eb0/pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85d27ea4c889342f7e35f6d56e7e1cb345632ad592e8c51b693d7b7556043ce0", size = 3459976, upload-time = "2025-04-12T17:49:38.988Z" }, - { url = "https://files.pythonhosted.org/packages/26/da/2c11d03b765efff0ccc473f1c4186dc2770110464f2177efaed9cf6fae01/pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:bf2c33d6791c598142f00c9c4c7d47f6476731c31081331664eb26d6ab583e01", size = 3527133, upload-time = "2025-04-12T17:49:40.985Z" }, - { url = "https://files.pythonhosted.org/packages/79/1a/4e85bd7cadf78412c2a3069249a09c32ef3323650fd3005c97cca7aa21df/pillow-11.2.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e616e7154c37669fc1dfc14584f11e284e05d1c650e1c0f972f281c4ccc53193", size = 3571555, upload-time = "2025-04-12T17:49:42.964Z" }, - { url = "https://files.pythonhosted.org/packages/69/03/239939915216de1e95e0ce2334bf17a7870ae185eb390fab6d706aadbfc0/pillow-11.2.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:39ad2e0f424394e3aebc40168845fee52df1394a4673a6ee512d840d14ab3013", size = 2674713, upload-time = "2025-04-12T17:49:44.944Z" }, - { url = "https://files.pythonhosted.org/packages/a4/ad/2613c04633c7257d9481ab21d6b5364b59fc5d75faafd7cb8693523945a3/pillow-11.2.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:80f1df8dbe9572b4b7abdfa17eb5d78dd620b1d55d9e25f834efdbee872d3aed", size = 3181734, upload-time = "2025-04-12T17:49:46.789Z" }, - { url = "https://files.pythonhosted.org/packages/a4/fd/dcdda4471ed667de57bb5405bb42d751e6cfdd4011a12c248b455c778e03/pillow-11.2.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ea926cfbc3957090becbcbbb65ad177161a2ff2ad578b5a6ec9bb1e1cd78753c", size = 2999841, upload-time = "2025-04-12T17:49:48.812Z" }, - { url = "https://files.pythonhosted.org/packages/ac/89/8a2536e95e77432833f0db6fd72a8d310c8e4272a04461fb833eb021bf94/pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:738db0e0941ca0376804d4de6a782c005245264edaa253ffce24e5a15cbdc7bd", size = 3437470, upload-time = "2025-04-12T17:49:50.831Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8f/abd47b73c60712f88e9eda32baced7bfc3e9bd6a7619bb64b93acff28c3e/pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9db98ab6565c69082ec9b0d4e40dd9f6181dab0dd236d26f7a50b8b9bfbd5076", size = 3460013, upload-time = "2025-04-12T17:49:53.278Z" }, - { url = "https://files.pythonhosted.org/packages/f6/20/5c0a0aa83b213b7a07ec01e71a3d6ea2cf4ad1d2c686cc0168173b6089e7/pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:036e53f4170e270ddb8797d4c590e6dd14d28e15c7da375c18978045f7e6c37b", size = 3527165, upload-time = "2025-04-12T17:49:55.164Z" }, - { url = "https://files.pythonhosted.org/packages/58/0e/2abab98a72202d91146abc839e10c14f7cf36166f12838ea0c4db3ca6ecb/pillow-11.2.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:14f73f7c291279bd65fda51ee87affd7c1e097709f7fdd0188957a16c264601f", size = 3571586, upload-time = "2025-04-12T17:49:57.171Z" }, - { url = "https://files.pythonhosted.org/packages/21/2c/5e05f58658cf49b6667762cca03d6e7d85cededde2caf2ab37b81f80e574/pillow-11.2.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:208653868d5c9ecc2b327f9b9ef34e0e42a4cdd172c2988fd81d62d2bc9bc044", size = 2674751, upload-time = "2025-04-12T17:49:59.628Z" }, -] - -[[package]] -name = "pong-ml" -version = "0.1.0" -source = { virtual = "." } -dependencies = [ - { name = "gymnasium", extra = ["atari"] }, - { name = "numpy" }, - { name = "pillow" }, - { name = "pymonik" }, -] - -[package.metadata] -requires-dist = [ - { name = "gymnasium", extras = ["atari"], specifier = ">=1.1.1" }, - { name = "numpy", specifier = ">=2.2.5" }, - { name = "pillow", specifier = ">=11.2.1" }, - { name = "pymonik", editable = "../../pymonik" }, -] - -[[package]] -name = "protobuf" -version = "4.25.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/63/84fdeac1f03864c2b8b9f0b7fe711c4af5f95759ee281d2026530086b2f5/protobuf-4.25.7.tar.gz", hash = "sha256:28f65ae8c14523cc2c76c1e91680958700d3eac69f45c96512c12c63d9a38807", size = 380612, upload-time = "2025-04-24T02:56:58.685Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/ed/9a58076cfb8edc237c92617f1d3744660e9b4457d54f3c2fdf1a4bbae5c7/protobuf-4.25.7-cp310-abi3-win32.whl", hash = "sha256:dc582cf1a73a6b40aa8e7704389b8d8352da616bc8ed5c6cc614bdd0b5ce3f7a", size = 392457, upload-time = "2025-04-24T02:56:40.798Z" }, - { url = "https://files.pythonhosted.org/packages/28/b3/e00870528029fe252cf3bd6fa535821c276db3753b44a4691aee0d52ff9e/protobuf-4.25.7-cp310-abi3-win_amd64.whl", hash = "sha256:cd873dbddb28460d1706ff4da2e7fac175f62f2a0bebc7b33141f7523c5a2399", size = 413446, upload-time = "2025-04-24T02:56:44.199Z" }, - { url = "https://files.pythonhosted.org/packages/60/1d/f450a193f875a20099d4492d2c1cb23091d65d512956fb1e167ee61b4bf0/protobuf-4.25.7-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4c899f09b0502eb39174c717ccf005b844ea93e31137c167ddcacf3e09e49610", size = 394248, upload-time = "2025-04-24T02:56:45.75Z" }, - { url = "https://files.pythonhosted.org/packages/c8/b8/ea88e9857484a0618c74121618b9e620fc50042de43cdabbebe1b93a83e0/protobuf-4.25.7-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:6d2f5dede3d112e573f0e5f9778c0c19d9f9e209727abecae1d39db789f522c6", size = 293717, upload-time = "2025-04-24T02:56:47.427Z" }, - { url = "https://files.pythonhosted.org/packages/a7/81/d0b68e9a9a76804113b6dedc6fffed868b97048bbe6f1bedc675bdb8523c/protobuf-4.25.7-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:d41fb7ae72a25fcb79b2d71e4247f0547a02e8185ed51587c22827a87e5736ed", size = 294636, upload-time = "2025-04-24T02:56:48.976Z" }, - { url = "https://files.pythonhosted.org/packages/17/d7/1e7c80cb2ea2880cfe38580dcfbb22b78b746640c9c13fc3337a6967dc4c/protobuf-4.25.7-py3-none-any.whl", hash = "sha256:e9d969f5154eaeab41404def5dcf04e62162178f4b9de98b2d3c1c70f5f84810", size = 156468, upload-time = "2025-04-24T02:56:56.957Z" }, -] - -[[package]] -name = "pycparser" -version = "2.22" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload-time = "2024-03-30T13:22:22.564Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, -] - -[[package]] -name = "pymonik" -version = "0.1.6" -source = { editable = "../../pymonik" } -dependencies = [ - { name = "armonik" }, - { name = "cloudpickle" }, - { name = "grpcio" }, - { name = "pyyaml" }, -] - -[package.metadata] -requires-dist = [ - { name = "armonik", specifier = ">=3.25.0" }, - { name = "cloudpickle", specifier = ">=3.1.1" }, - { name = "grpcio" }, - { name = "pyyaml", specifier = ">=6.0.2" }, -] - -[package.metadata.requires-dev] -dev = [{ name = "ruff", specifier = ">=0.11.6" }] - -[[package]] -name = "pyyaml" -version = "6.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" }, - { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" }, - { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" }, - { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" }, - { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" }, - { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" }, - { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" }, - { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" }, - { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" }, - { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" }, - { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" }, - { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" }, - { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" }, - { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" }, - { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" }, - { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" }, - { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" }, - { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, - { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, - { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, - { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" }, - { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" }, - { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" }, - { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" }, - { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" }, - { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, - { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" }, - { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" }, - { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" }, - { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" }, - { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" }, - { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" }, - { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" }, - { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" }, - { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, -] - -[[package]] -name = "setuptools" -version = "80.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/32/0cc40fe41fd2adb80a2f388987f4f8db3c866c69e33e0b4c8b093fdf700e/setuptools-80.4.0.tar.gz", hash = "sha256:5a78f61820bc088c8e4add52932ae6b8cf423da2aff268c23f813cfbb13b4006", size = 1315008, upload-time = "2025-05-09T20:42:27.972Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/93/dba5ed08c2e31ec7cdc2ce75705a484ef0be1a2fecac8a58272489349de8/setuptools-80.4.0-py3-none-any.whl", hash = "sha256:6cdc8cb9a7d590b237dbe4493614a9b75d0559b888047c1f67d49ba50fc3edb2", size = 1200812, upload-time = "2025-05-09T20:42:25.325Z" }, -] - -[[package]] -name = "typing-extensions" -version = "4.13.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/37/23083fcd6e35492953e8d2aaaa68b860eb422b34627b13f2ce3eb6106061/typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef", size = 106967, upload-time = "2025-04-10T14:19:05.416Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/54/b1ae86c0973cc6f0210b53d508ca3641fb6d0c56823f288d108bc7ab3cc8/typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c", size = 45806, upload-time = "2025-04-10T14:19:03.967Z" }, -] diff --git a/examples/retries.py b/examples/retries.py new file mode 100644 index 0000000..f3cdd9a --- /dev/null +++ b/examples/retries.py @@ -0,0 +1,85 @@ +"""Decorator-level retries with exception-type filter and backoff. + +Shows two flavours: + +1. Cluster-side blanket retries via ``@task(retries=N)`` — ArmoniK retries + N times for any failure (infra or user-code). + +2. Client-side filtered retries via ``@task(retries=N, retry_on=(...,), + retry_backoff=...)`` — the SDK observes the failure type, sleeps the + configured backoff, and re-spawns. Only matching exceptions retry; + anything else surfaces immediately. + +The ``flaky`` task fails its first 2 attempts and then succeeds, so we +can see the retry loop without flapping. + + uv run python examples/retries.py --partition pymonikv1 +""" + +from __future__ import annotations + +import argparse +import time + +from pymonik import PymonikClient, TaskFailed, current, task +import pymonik + + +# ---- a deterministic flaky function ---- +# Uses ``pymonik.current().attempt`` (threaded through the envelope on +# every retry) so the function's success condition is independent of where +# it lands — works the same on the cluster and under LocalCluster. +@task(retries=4, retry_on=(TaskFailed,), retry_backoff="exponential") +def flaky_with_filter(label: str) -> str: + ctx = current() + n = ctx.attempt + ctx.log.info("flaky attempt", label=label, attempt=n) + if n < 3: + raise RuntimeError(f"intentional failure on attempt {n}") + return f"{label} succeeded on attempt {n}" + + +# ---- a never-retried failure shape ---- +# `retry_on=(KeyError,)` means a RuntimeError will surface immediately +# without retrying — we expect this to fail on the first attempt. +@task(retries=4, retry_on=(KeyError,), retry_backoff="constant") +def fails_with_unmatched_type(label: str) -> str: + raise RuntimeError(f"{label}: deliberately unmatched failure") + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument("--partition", default="pymonikv1") + args = ap.parse_args() + + with PymonikClient() as client: + with client.session(partition=args.partition) as s: + # 1) Filtered retries actually retry until success. + t0 = time.monotonic() + label = f"run-{int(time.time())}" + try: + result = flaky_with_filter.spawn(label).result(timeout=120) + print(f"flaky_with_filter: {result} ({time.monotonic() - t0:.1f}s)") + except Exception as e: + # If the worker pod's /tmp is shared across our retries (same + # pod handled both attempts), the third attempt succeeds. + # If retries land on fresh pods, the counter resets and we'd + # exhaust the budget — that's a property of how pods scale + # rather than a retry-logic bug. + print(f"flaky_with_filter: exhausted -> {e}") + + # 2) Unmatched exception type — no retry, raises immediately. + t0 = time.monotonic() + try: + fails_with_unmatched_type.spawn("immediate").result(timeout=60) + print("UNEXPECTED success on fails_with_unmatched_type") + except TaskFailed as e: + print( + f"fails_with_unmatched_type: surfaced after " + f"{time.monotonic() - t0:.1f}s as expected; head={str(e)[:80]}…" + ) + + +if __name__ == "__main__": + main() diff --git a/examples/subtasking.py b/examples/subtasking.py new file mode 100644 index 0000000..e9efc19 --- /dev/null +++ b/examples/subtasking.py @@ -0,0 +1,44 @@ +"""Sub-tasking via ``task.tail()``. + +Decreasing-counter recursion: each task either terminates or spawns a +single child with the parent's expected output slot. ArmoniK stitches the +chain together; the client only sees the final answer. + + uv run python examples/subtasking.py --partition pymonikv1 --depth 5 +""" + +from __future__ import annotations + +import argparse + +from pymonik import PymonikClient, current, task +import pymonik + + +@task +def increment_chain(n: int, acc: int) -> int: + """Base case returns acc; recursive step delegates its output to a child.""" + current().log.info("step", n=n, acc=acc) + if n <= 0: + return acc + # `tail()` submits the child with this task's expected_output_id, so + # ArmoniK marks our output as done when the child completes. The + # returned TailPromise tells the dispatcher we handed off. + return increment_chain.tail(n - 1, acc + 1) # type: ignore[return-value] + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument("--partition", default="pymonikv1") + ap.add_argument("--depth", type=int, default=5) + args = ap.parse_args() + + with PymonikClient() as client: + with client.session(partition=args.partition) as s: + result = increment_chain.spawn(args.depth, 0).result(timeout=300) + print(f"chain({args.depth}) -> {result} (expected {args.depth})") + + +if __name__ == "__main__": + main() diff --git a/examples/task_options.py b/examples/task_options.py new file mode 100644 index 0000000..cae1920 --- /dev/null +++ b/examples/task_options.py @@ -0,0 +1,65 @@ +"""Per-task and per-call option overrides. + +Demonstrates decorator options, ``.with_options(...)``, and the +session-default → @task → .with_options precedence. + +In a multi-partition deployment you'd use ``.with_options(partition=...)`` +to route a task to a specific partition. The quick-deploy here only has +``pymonikv1``, so the partition switch is shown in code but would +otherwise target e.g. ``pymonik_gpu``. + + uv run python examples/task_options.py --partition pymonikv1 +""" + +from __future__ import annotations + +import argparse +from datetime import timedelta + +from pymonik import PymonikClient, TaskOpts, current, task +import pymonik + + +# Decorator-level options. Merged with session default; overridable per-call. +@task(retries=3, timeout=timedelta(seconds=30), priority=5) +def heavy_compute(n: int) -> int: + ctx = current() + ctx.log.info("heavy_compute", attempt=ctx.attempt, n=n) + total = 0 + for i in range(n): + total += i * i + return total + + +@task +def echo(value: str) -> str: + return f"echo: {value}" + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument("--partition", default="pymonikv1") + args = ap.parse_args() + + # Session default: applies to every task unless overridden. + session_default = TaskOpts(priority=1, timeout=120) + + with PymonikClient() as client: + with client.session(partition=args.partition, default_options=session_default) as s: + # Uses @task(retries=3, timeout=30s, priority=5) merged over session default. + f1 = heavy_compute.spawn(10_000) + print("heavy_compute ->", f1.result(timeout=60)) + + # .with_options overrides per-call: a shorter timeout just for this spawn. + f2 = heavy_compute.with_options(timeout=5, priority=10).spawn(1_000) + print("heavy_compute (per-call override) ->", f2.result(timeout=60)) + + # .with_options returns a new Task; the original is untouched. + assert heavy_compute.opts.priority == 5 + f3 = echo.spawn("plain decorator, no options") + print("echo ->", f3.result(timeout=60)) + + +if __name__ == "__main__": + main() diff --git a/examples/with_deps.py b/examples/with_deps.py new file mode 100644 index 0000000..4381bc3 --- /dev/null +++ b/examples/with_deps.py @@ -0,0 +1,76 @@ +"""Runtime pip deps via ``client.session(deps=[...])``. + +The worker image only has pymonik; numpy and polars get installed into +a per-deps venv on first use, and reused across every task in this +session (and across sessions/clients that pick the same deps list). + +Imports are at module level. cloudpickle ships the function with a +by-name reference to the imported module — on the worker, the +``deps`` venv (spliced into ``sys.path``) makes that import resolve. +The client side needs the dep too: it has to import the module to +build the function in the first place. (To submit deps your client +doesn't have, drop the import inside the task body.) + +Run: + + export AKCONFIG=/path/to/generated/armonik-cli.yaml + uv run python examples/with_deps.py +""" + +from __future__ import annotations + +import argparse +import time + +import numpy as np +import polars as pl + +import pymonik +from pymonik import PymonikClient, task + + +@task +def numpy_stats(n: int) -> dict[str, float]: + rng = np.random.default_rng(seed=n) + arr = rng.standard_normal(size=10_000) + return { + "mean": float(arr.mean()), + "std": float(arr.std()), + "min": float(arr.min()), + "max": float(arr.max()), + } + + +@task +def polars_demo() -> str: + df = pl.DataFrame({"x": [1, 2, 3, 4, 5], "y": [10, 20, 30, 40, 50]}) + return f"polars {pl.__version__}: rows={df.height}, sum_y={df['y'].sum()}" + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument("--endpoint", default=None) + ap.add_argument("--partition", default="pymonik") + args = ap.parse_args() + + with PymonikClient(endpoint=args.endpoint) as client: + with client.session( + partition=args.partition, + deps=["numpy", "polars"], + ) as s: + print("first task pays the install (~tens of seconds)") + t0 = time.monotonic() + stats = numpy_stats.spawn(42).result(timeout=600) + print(f" numpy_stats(seed=42) -> {stats}") + print(f" elapsed: {time.monotonic() - t0:.1f}s") + + print("subsequent tasks: in-process, ~ms per call") + t1 = time.monotonic() + polars_msg = polars_demo.spawn().result(timeout=120) + print(f" polars_demo() -> {polars_msg}") + print(f" elapsed: {time.monotonic() - t1:.3f}s") + + +if __name__ == "__main__": + main() diff --git a/examples/with_deps_isolated.py b/examples/with_deps_isolated.py new file mode 100644 index 0000000..90402a1 --- /dev/null +++ b/examples/with_deps_isolated.py @@ -0,0 +1,58 @@ +"""``isolate=True`` — opt into per-task subprocess isolation. + +Default deps mode is **in-process splice**: the worker adds the venv's +site-packages to ``sys.path`` and calls the function inline. ~1 ms per +task once warm, but module-level state (and the *single* installed +version of every package) is shared across every task running on that +worker pod. + +If you need stronger isolation — concurrent sessions on the same pod +with conflicting deps, or tasks that mutate global module state — pass +``isolate=True`` to spawn a fresh Python interpreter per task. The +trade-off is wall-clock: numpy alone adds ~400-500 ms to every task +(Python startup + numpy import). Heavier stacks (torch, scipy) hurt +more. + +Run: + + uv run python examples/with_deps_isolated.py +""" + +from __future__ import annotations + +import argparse +import time + +import numpy as np + +import pymonik +from pymonik import PymonikClient, task + + +@task +def quick_numpy(n: int) -> int: + return int(np.arange(n).sum()) + + +def main() -> None: + pymonik.enable_logging() + ap = argparse.ArgumentParser() + ap.add_argument("--endpoint", default=None) + ap.add_argument("--partition", default="pymonik") + args = ap.parse_args() + + with PymonikClient(endpoint=args.endpoint) as client: + with client.session( + partition=args.partition, + deps=["numpy"], + isolate=True, + ) as s: + futures = [quick_numpy.spawn(i * 1000) for i in range(1, 6)] + for fut in futures: + t = time.monotonic() + v = fut.result(timeout=300) + print(f"quick_numpy -> {v} ({time.monotonic() - t:.2f}s)") + + +if __name__ == "__main__": + main() diff --git a/examples/with_deps_local.py b/examples/with_deps_local.py new file mode 100644 index 0000000..070ebaf --- /dev/null +++ b/examples/with_deps_local.py @@ -0,0 +1,77 @@ +"""Runtime deps under ``LocalCluster`` — same code path as the cluster. + +The local backend exercises the same ``ensure_env`` + dispatcher that +runs on real workers, so this is also a self-contained integration +test for the Path A pipeline. Useful for debugging install failures +(PyPI lookups, conflicts) without round-tripping a real cluster. + +``isolate=False`` (the default) is shown first: install once, then +every subsequent task is essentially free. The ``isolate=True`` +section shows the cost of the subprocess path for comparison. + +Run: + + uv run python examples/with_deps_local.py +""" + +from __future__ import annotations + +import time + +import numpy as np + +import pymonik +from pymonik import task +from pymonik.testing import LocalCluster + + +@task +def numpy_sum(n: int) -> int: + return int(np.arange(n).sum()) + + +@task(deps=["numpy"]) +def numpy_sum_per_task(n: int) -> int: + """Same payload, but deps come from the @task decorator rather than + the session — useful for "this one task needs numpy, the rest + of the session doesn't". + """ + return int(np.arange(n).sum()) + + +def main() -> None: + pymonik.enable_logging() + + print("default isolate=False (in-process splice)") + with LocalCluster() as client: + with client.session(deps=["numpy"]) as s: + t0 = time.monotonic() + v = numpy_sum.spawn(1_000).result(timeout=600) + print( + f" numpy_sum(1000) -> {v} ({time.monotonic() - t0:.1f}s, first call pays install)" + ) + t1 = time.monotonic() + v = numpy_sum.spawn(10_000).result(timeout=60) + print( + f" numpy_sum(10000) -> {v} ({time.monotonic() - t1:.4f}s, in-process — ~free)" + ) + + print("\nopt-in isolate=True (subprocess per task)") + with LocalCluster() as client: + with client.session(deps=["numpy"], isolate=True) as s: + t0 = time.monotonic() + v = numpy_sum.spawn(1_000).result(timeout=600) + print(f" numpy_sum(1000) -> {v} ({time.monotonic() - t0:.2f}s, subprocess startup + numpy import)") + t1 = time.monotonic() + v = numpy_sum.spawn(10_000).result(timeout=60) + print(f" numpy_sum(10000) -> {v} ({time.monotonic() - t1:.2f}s, subprocess startup + numpy import)") + + print("\nper-task deps via @task(deps=[...])") + with LocalCluster() as client: + with client.session() as s: + v = numpy_sum_per_task.spawn(500).result(timeout=600) + print(f" numpy_sum_per_task(500) -> {v}") + + +if __name__ == "__main__": + main() diff --git a/examples/with_otel.py b/examples/with_otel.py new file mode 100644 index 0000000..b4aeeee --- /dev/null +++ b/examples/with_otel.py @@ -0,0 +1,80 @@ +"""OTel tracing — opt-in, end-to-end, with a one-container UI. + +The minimum-infra path for visualising spans: + +1. Run Jaeger all-in-one (UI + OTLP collector + storage in one container):: + + docker run --rm -d --name jaeger \\ + -p 16686:16686 \\ + -p 4317:4317 \\ + -e COLLECTOR_OTLP_ENABLED=true \\ + jaegertracing/all-in-one:latest + + - 16686 — Jaeger UI (http://localhost:16686) + - 4317 — OTLP/gRPC ingress (where pymonik exports to) + +2. Point both client and worker at it via env vars:: + + export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 + export OTEL_SERVICE_NAME=pymonik-client # optional + export OTEL_EXPORTER_OTLP_INSECURE=true # for plain http + + For a real ArmoniK cluster, set the same vars on the worker pods (bake + into the worker image or set in the partition's pod template). For + ``LocalCluster``, exporting on the client side is enough — the worker + ``LocalSession`` runs in the same process. + +3. Run this script:: + + uv pip install 'pymonik[otel]' + OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \\ + uv run python examples/with_otel.py + +4. Open http://localhost:16686, pick "pymonik" service, hit "Find traces". + +You should see a tree like:: + + pymonik.session.open + └── pymonik.submit (count=8, func=square) + ├── pymonik.task.run (task_id=..., attempt=1) + ├── pymonik.task.run + └── ... (8 leaves) + pymonik.future.wait +""" + +from __future__ import annotations + +import time + +import pymonik +from pymonik import task +from pymonik.testing import LocalCluster + + +@task +def square(x: int) -> int: + return x * x + + +@task +def total(xs: list[int]) -> int: + return sum(xs) + + +def main() -> None: + pymonik.enable_logging() + + # otel=None auto-detects from OTEL_EXPORTER_OTLP_ENDPOINT etc. Pass + # otel=True to force-enable (handy if you've configured a TracerProvider + # yourself and just want pymonik to emit spans). + with LocalCluster() as client: + with client.session(partition="local") as s: + t0 = time.monotonic() + partials = square.map(range(8)) + answer = total.spawn(partials).result(timeout=30) + elapsed = time.monotonic() - t0 + print(f"sum of squares 0..7 = {answer} (elapsed {elapsed:.2f}s)") + + +if __name__ == "__main__": + main() diff --git a/pymonik/.python-version b/pymonik/.python-version deleted file mode 100644 index 56d91d3..0000000 --- a/pymonik/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.10.12 diff --git a/pymonik/pyproject.toml b/pymonik/pyproject.toml deleted file mode 100644 index 97f2dc3..0000000 --- a/pymonik/pyproject.toml +++ /dev/null @@ -1,29 +0,0 @@ -[build-system] -requires = ["hatchling", "uv-dynamic-versioning"] -build-backend = "hatchling.build" - -[tool.hatch.version] -source = "uv-dynamic-versioning" -variable = "PYMONIK_BUILD_VERSION" - -[project] -name = "pymonik" -dynamic = ["version"] -description = "Lightweight Distributed Computing Framework" -readme = "README.md" -requires-python = ">=3.10" -dependencies = [ - "armonik>=3.25.0", - "cloudpickle>=3.1.1", - "grpcio", - "pyyaml>=6.0.2", -] - -[tool.setuptools] -py-modules = [] - -[dependency-groups] -dev = [ - "ruff>=0.11.6", -] - diff --git a/pymonik/src/pymonik/__init__.py b/pymonik/src/pymonik/__init__.py deleted file mode 100644 index 7957512..0000000 --- a/pymonik/src/pymonik/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -import importlib.metadata - -from .core import Pymonik, Task, task -from .context import PymonikContext -from .results import ResultHandle, MultiResultHandle -from .worker import run_pymonik_worker -from .materialize import Materialize, materialize -from armonik.common import TaskOptions - -try: - __version__ = importlib.metadata.version(__name__) -except importlib.metadata.PackageNotFoundError: - __version__ = "0.0.0" # Fallback for development mode - -__all__ = [ - "Pymonik", - "task", - "PymonikContext", - "run_pymonik_worker", - "Task", - "ResultHandle", - "MultiResultHandle", - "TaskOptions", - "Materialize", - "materialize" -] diff --git a/pymonik/src/pymonik/context.py b/pymonik/src/pymonik/context.py deleted file mode 100644 index 25a894a..0000000 --- a/pymonik/src/pymonik/context.py +++ /dev/null @@ -1,240 +0,0 @@ -import io -import zipfile -import cloudpickle as pickle - -from logging import Logger -from pathlib import Path -from typing import Any, Optional, Union - - -from .materialize import Materialize, _calculate_directory_hash, _calculate_file_hash -from .environment import RuntimeEnvironment -from armonik.worker import TaskHandler -from armonik.protogen.common.agent_common_pb2 import (DataRequest, DataResponse) - - -class PymonikContext: - """ - Context for PymoniK execution. - This class is used to manage the execution environment and logging for PymoniK tasks. - When running in a local environment, it uses the provided logger. - """ - def __init__(self, task_handler: TaskHandler, logger: Logger): - self.task_handler = task_handler - self.logger = logger - self.environment = RuntimeEnvironment(logger) - self.is_local = task_handler is None - - @staticmethod - def from_local(logger: Optional[Logger] = None) -> "PymonikContext": - """ - Create a PymonikContext for local execution. - """ - if logger is None: - logger = Logger("PymonikLocalExecution") - return PymonikContext(task_handler=None, logger=logger) - - def retrieve_object( - self, - result_id: str, - auto_unpickle: bool = True, - check_exists: bool = True, - force_retrieve: bool = False - ) -> Union[bool, Any, bytes, None]: - """ - Retrieves an object from ArmoniK storage to the local worker cache. - - Args: - result_id (str): The ID of the result/object to retrieve - auto_unpickle (bool): If True, automatically unpickle and return the object. - If False, just retrieve the file and return the bytes. - Defaults to True. - check_exists (bool): If True, check if the object already exists locally - before attempting to retrieve. Defaults to True. - force_retrieve (bool): If True, retrieve the object even if it already exists - locally. Only used when check_exists=True. Defaults to False. - - Returns: - - If auto_unpickle=True: The unpickled object if successful, None if failed - - If auto_unpickle=False: The raw bytes if successful, None if failed - - Raises: - RuntimeError: If called in local context (no task handler available) - """ - if self.is_local: - raise RuntimeError("retrieve_object can only be called in worker context") - - object_path = self.get_object_path(result_id) - - # Check if object already exists locally - if check_exists and object_path.exists(): - self.logger.info(f"=== DEBUG RETRIEVE: Object {result_id} already exists locally at {object_path} ===") - - if not force_retrieve: - if auto_unpickle: - try: - with open(object_path, "rb") as fh: - return pickle.load(fh) - except Exception as e: - self.logger.error(f"Failed to unpickle existing object {result_id}: {e}") - return None - else: - # Return the bytes from the existing file - try: - with open(object_path, "rb") as fh: - return fh.read() - except Exception as e: - self.logger.error(f"Failed to read existing object {result_id}: {e}") - return None - else: - self.logger.info(f"force_retrieve=True, retrieving {result_id} anyway") - - self.logger.info(f"=== DEBUG RETRIEVE: {result_id} not in data_dependencies, trying GetResourceData ===") - try: - # Ensure the parent directory exists - object_path.parent.mkdir(parents=True, exist_ok=True) - - data_request = DataRequest( - communication_token=self.task_handler.token, - result_id=result_id - ) - - # GetResourceData downloads the file directly to object_path - data_response: DataResponse = self.task_handler._client.GetResourceData(data_request) - - if data_response.result_id != result_id: - self.logger.error(f"Retrieved object ID mismatch: expected {result_id}, got {data_response.result_id}") - return None - - # The file should now exist at object_path - if not object_path.exists(): - self.logger.error(f"GetResourceData completed but file doesn't exist at {object_path}") - return None - - self.logger.info(f"Successfully retrieved object {result_id} via GetResourceData to {object_path}") - - if auto_unpickle: - try: - with open(object_path, "rb") as fh: - unpickled_obj = pickle.load(fh) - self.logger.debug(f"Successfully unpickled object {result_id}") - return unpickled_obj - except Exception as e: - self.logger.error(f"Failed to unpickle object {result_id}: {e}") - return None - else: - # Return the raw bytes from the downloaded file - try: - with open(object_path, "rb") as fh: - return fh.read() - except Exception as e: - self.logger.error(f"Failed to read downloaded file {object_path}: {e}") - return None - - except Exception as e: - self.logger.error(f"Failed to retrieve object {result_id} via GetResourceData: {e}") - import traceback - self.logger.error(f"=== DEBUG RETRIEVE: Traceback: {traceback.format_exc()} ===") - return None - - def get_object_path(self, result_id: str) -> Path: - """ - Get the local file path where an object would be stored. - - Args: - result_id (str): The ID of the result/object - - Returns: - Path: The local path where the object is/would be stored - """ - return Path("/cache/shared/") / Path(self.task_handler.token) / Path(result_id) - - def object_exists_locally(self, result_id: str) -> bool: - """ - Check if an object exists locally in the worker cache. - - Args: - result_id (str): The ID of the result/object to check - - Returns: - bool: True if the object exists locally, False otherwise - """ - return self.get_object_path(result_id).exists() - - def materialize_file(self, mat: Materialize) -> bool: - """ - Materialize a file/directory in the worker if needed. - - Args: - mat: Materialize object describing what to materialize - - Returns: - bool: True if materialization was successful, False otherwise - """ - if self.is_local: - self.logger.warning("materialize_file called in local context, skipping") - return True - - worker_path = Path(mat.worker_path) - - # Check if file/directory already exists and has correct hash - if worker_path.exists(): - try: - if mat.is_directory and worker_path.is_dir(): - existing_hash = _calculate_directory_hash(worker_path) - elif not mat.is_directory and worker_path.is_file(): - existing_hash = _calculate_file_hash(worker_path) - else: - # Type mismatch (file vs directory), need to re-materialize - existing_hash = None - - if existing_hash == mat.content_hash: - self.logger.info(f"Materialize content already exists with correct hash: {worker_path}") - return True - else: - self.logger.info(f"Materialize content exists but hash mismatch, re-materializing: {worker_path}") - except Exception as e: - self.logger.warning(f"Error checking existing materialize content: {e}") - - # Need to retrieve and materialize - if not mat.result_id: - self.logger.error(f"Materialize object has no result_id: {mat}") - return False - - try: - # Retrieve the content - content_bytes = self.retrieve_object(mat.result_id, auto_unpickle=False, check_exists=False) - if not content_bytes: - self.logger.error(f"Failed to retrieve materialize content: {mat.result_id}") - return False - - # Create parent directories - worker_path.parent.mkdir(parents=True, exist_ok=True) - - if mat.is_directory: - # Extract zip to target directory - with zipfile.ZipFile(io.BytesIO(content_bytes), 'r') as zipf: - zipf.extractall(worker_path) - self.logger.info(f"Extracted directory to: {worker_path}") - else: - # Write file directly - with open(worker_path, 'wb') as f: - f.write(content_bytes) - self.logger.info(f"Wrote file to: {worker_path}") - - # Verify hash after materialization - if mat.is_directory: - final_hash = _calculate_directory_hash(worker_path) - else: - final_hash = _calculate_file_hash(worker_path) - - if final_hash != mat.content_hash: - self.logger.error(f"Hash mismatch after materialization: expected {mat.content_hash}, got {final_hash}") - return False - - self.logger.info(f"Successfully materialized: {mat.source_path} -> {worker_path}") - return True - - except Exception as e: - self.logger.error(f"Error materializing content: {e}") - return False diff --git a/pymonik/src/pymonik/core.py b/pymonik/src/pymonik/core.py deleted file mode 100644 index 96eb782..0000000 --- a/pymonik/src/pymonik/core.py +++ /dev/null @@ -1,1011 +0,0 @@ -import contextvars -import io -import os -import sys -import zipfile -import signal - -import uuid -import yaml -import cloudpickle as pickle - -from datetime import timedelta -from typing import Any, Callable, Dict, Generic, List, Optional, ParamSpec, Tuple, TypeVar, Union -from .utils import LazyArgs, _poll_batch_for_results, create_grpc_channel -from .results import ResultHandle, MultiResultHandle -from .materialize import Materialize, _create_zip_from_directory - -from armonik.client import ArmoniKTasks, ArmoniKResults, ArmoniKSessions, ArmoniKEvents -from armonik.common import TaskOptions, TaskDefinition, Result, batched -from armonik.worker import TaskHandler - -_CURRENT_PYMONIK: contextvars.ContextVar[Optional["Pymonik"]] = contextvars.ContextVar( - "_CURRENT_PYMONIK", default=None -) - -P_Args = ParamSpec("P_Args") -R_Type = TypeVar("R_Type") - -U_Obj = TypeVar("U_Obj") # For single object in put -V_Obj = TypeVar("V_Obj") # For type of objects in a list for put_many - -class Task(Generic[P_Args, R_Type]): - """A wrapper for a function that can be executed as an ArmoniK task.""" - - def __init__( - self, func: Callable, require_context: bool = False, func_name: str = None, task_options: Optional[TaskOptions] = None - - ): - self.func: Callable[P_Args, R_Type] = func - self.func_name = func_name or func.__name__ - self.require_context = require_context - self.task_options = task_options - - - def _merge_task_options( - self, - pymonik_instance: "Pymonik", - task_options: Optional[TaskOptions] = None, - pmk_kwargs: Dict[str, Any] = None - ) -> TaskOptions: - """Merge task options from different sources with proper precedence.""" - pmk_kwargs = pmk_kwargs or {} - # Start with Pymonik instance defaults - base_options = pymonik_instance.task_options - - # Create a dictionary to build the merged options - merged_attrs = { - 'max_duration': base_options.max_duration, - 'priority': base_options.priority, - 'max_retries': base_options.max_retries, - 'partition_id': base_options.partition_id, - 'options' : base_options.options - } - - # Apply task decorator options if they exist - if self.task_options: - if self.task_options.max_duration is not None: - merged_attrs['max_duration'] = self.task_options.max_duration - if self.task_options.priority is not None: - merged_attrs['priority'] = self.task_options.priority - if self.task_options.max_retries is not None: - merged_attrs['max_retries'] = self.task_options.max_retries - if self.task_options.partition_id is not None: - merged_attrs['partition_id'] = self.task_options.partition_id - if self.task_options.options is not None: - merged_attrs['options'] = self.task_options.options - - # Apply invocation-specific task options - if task_options: - if task_options.max_duration is not None: - merged_attrs['max_duration'] = task_options.max_duration - if task_options.priority is not None: - merged_attrs['priority'] = task_options.priority - if task_options.max_retries is not None: - merged_attrs['max_retries'] = task_options.max_retries - if task_options.partition_id is not None: - merged_attrs['partition_id'] = task_options.partition_id - if task_options.options is not None: - merged_attrs['options'] = task_options.options - - # Apply pmk_ prefixed options - for key, value in pmk_kwargs.items(): - if key.startswith('pmk_'): - option_name = key[4:] # Remove 'pmk_' prefix - if option_name == 'max_duration': - # Handle duration conversion if needed - if isinstance(value, (int, float)): - merged_attrs['max_duration'] = timedelta(seconds=value) - elif isinstance(value, timedelta): - merged_attrs['max_duration'] = value - else: - merged_attrs[option_name] = value - - return TaskOptions( - max_duration=merged_attrs['max_duration'], - priority=merged_attrs['priority'], - max_retries=merged_attrs['max_retries'], - partition_id=merged_attrs['partition_id'], - options=merged_attrs['options'] - ) - - # TODO: repeat invocations for parameter-less functions my_function.invoke(repeat=5) - def invoke( - self, *args, pymonik: Optional["Pymonik"] = None, delegate=False, task_options: Optional[TaskOptions] = None, **kwargs - ) -> ResultHandle[R_Type]: - """Invoke the task with the given arguments.""" - - pmk_kwargs = {k: v for k, v in kwargs.items() if k.startswith('pmk_')} - regular_kwargs = {k: v for k, v in kwargs.items() if not k.startswith('pmk_')} - - - # Handle the case of a single task - if pymonik is None: - pymonik = _CURRENT_PYMONIK.get(None) - if pymonik is None: - raise RuntimeError( - "No active PymoniK instance found. Please create one and pass it in or use the context manager." - ) - - # I'm using the 'pmk_' prefix to avoid potential naming conflicts. - merged_task_options = self._merge_task_options(pymonik, task_options, pmk_kwargs) - - if len(args) == 0: - results = self._invoke_multiple([(Pymonik.NoInput,)], pymonik, delegate, merged_task_options) - return results[0] - results: List[ResultHandle[R_Type]] = self._invoke_multiple([args], pymonik, delegate, merged_task_options, additional_kwargs=regular_kwargs if regular_kwargs != {} else None) - return results[0] - - def map_invoke( - self, - args_list: List[Tuple], - pymonik: Optional["Pymonik"] = None, - delegate=False, - task_options: Optional[TaskOptions] = None, - **kwargs - ) -> MultiResultHandle: - """Invoke the task with the given arguments and return a MultiResultHandle.""" - - pmk_kwargs = {k: v for k, v in kwargs.items() if k.startswith('pmk_')} - - if pymonik is None: - pymonik = _CURRENT_PYMONIK.get(None) - if pymonik is None: - raise RuntimeError( - "No active PymoniK instance found. Please create one and pass it in or use the context manager." - ) - - merged_task_options = self._merge_task_options(pymonik, task_options, pmk_kwargs) - - # Handle the case of multiple tasks - result_handles: List[ResultHandle[R_Type]] = self._invoke_multiple(args_list, pymonik, delegate, merged_task_options) - return MultiResultHandle(result_handles) - - def __call__(self, *args, **kwds): - return self.func(*args, **kwds) - - def _invoke_multiple( - self, args_list: List[Tuple], pymonik_instance: "Pymonik", delegate: bool, task_options: TaskOptions, additional_kwargs: Optional[Dict[str, Any]] = None - ) -> List[ResultHandle]: - """Invoke a multiple tasks with the given arguments.""" - # Ensure we have an active connection and session - - if delegate and not pymonik_instance.is_worker(): - raise RuntimeError( - "Delegation is only supported in worker mode. Please use the worker context." - ) - - if delegate and len(args_list) > 1: - raise RuntimeError( - "Delegation is only supported for a single task with a single result handle. Please use the invoke method, or combine the results into a single result." - ) - - if not pymonik_instance._connected: - pymonik_instance.create() - - # Process arguments to extract ResultHandles for data dependencies - if not pymonik_instance._session_created: - raise RuntimeError( - "No existing session to link the invocation to, create one first (hint: call create or use the context manager)" - ) - - # - function_instance_remote_name = ( - pymonik_instance._session_id + "__function__" + self.func_name - ) - - if function_instance_remote_name not in pymonik_instance.remote_functions: - pymonik_instance.register_tasks([self]) - - function_id = pymonik_instance.remote_functions[ - pymonik_instance._session_id + "__function__" + self.func_name - ].result_id - - all_function_invocation_info = [] - all_result_names = [] - all_payloads = {} - for args in args_list: - payload_name = f"{pymonik_instance._session_id}__payload__{self.func_name}__{uuid.uuid4()}" - result_name = f"{pymonik_instance._session_id}__output__{self.func_name}__{uuid.uuid4()}" - function_invocation_info = { - "data_dependencies": [function_id], - "payload_name": payload_name, - "result_name": result_name, - } - processed_args = [] - # Prepare function call args description - for arg in args: - if arg is pymonik_instance.NoInput: - processed_args.append("__no_input__") - elif isinstance(arg, ResultHandle): - function_invocation_info["data_dependencies"].append(arg.result_id) - processed_args.append(f"__result_handle__{arg.result_id}") - elif isinstance(arg, MultiResultHandle): - # If it's a MultiResultHandle, add all result IDs as dependencies - for handle in arg.result_handles: - function_invocation_info["data_dependencies"].append( - handle.result_id - ) - processed_args.append( - f"__multi_result_handle__" - + ",".join([handle.result_id for handle in arg.result_handles]) - ) - elif isinstance(arg, Materialize): - if not arg.result_id: - raise ValueError(f"Materialize object must be uploaded first: {arg}") - # Add the materialized content as a dependency - function_invocation_info["data_dependencies"].append(arg.result_id) - # Pass the Materialize object directly (it will be pickled) - processed_args.append(arg) - else: - processed_args.append(arg) - - # Serialize the function call information - payload = pickle.dumps( - { - "func_name": self.func_name, - "func_id": function_id, - "require_context": self.require_context, - "environment": pymonik_instance.environment, - "args": LazyArgs(processed_args), - } - ) - - all_payloads[payload_name] = payload - all_result_names.append(result_name) - all_function_invocation_info.append(function_invocation_info) - # Create result metadata for output - - if delegate: - results_created = { - all_function_invocation_info[0]["result_name"]: Result( - result_id=pymonik_instance.parent_task_result_id - ) - } - else: - results_created = pymonik_instance._dispatch_create_metadata( - all_result_names, - ) - # Create the payloads for all the tasks to submit - payload_results = pymonik_instance._dispatch_create_payloads(all_payloads) - - # Submit all the tasks: - task_definitions = [] - for invocation_info in all_function_invocation_info: - # Create the task definition - # TODO: Wrap this for into a TaskInvocation object that can be manipulated for subtasking on the worker side of things - task_definitions.append( - TaskDefinition( - payload_id=payload_results[ - invocation_info["payload_name"] - ].result_id, - expected_output_ids=[ - results_created[invocation_info["result_name"]].result_id - ], - data_dependencies=invocation_info["data_dependencies"], - ) - ) - - # Submit the task - pymonik_instance._dispatch_submit_tasks( - task_definitions, # TODO: use different batch size for tasks/results - task_options - ) - - # Return a handle to the result - result_handles = [ - ResultHandle( - result.result_id, pymonik_instance._session_id, pymonik_instance - ) - for result in results_created.values() - ] - return result_handles - -class Pymonik: - """A wrapper around ArmoniK for task-based distributed computing.""" - - # A singleton to indicate that a task takes no input - NoInput = object() - - def __init__( - self, - endpoint: Optional[str] = None, - partition: Optional[Union[str, List[str]]] = "pymonik", - environment: Dict[str, Any] = {}, - is_worker: bool = False, - batch_size: int = 32, - task_options: Optional[TaskOptions] = None, - disable_events_client: bool = False, - polling_interval: int = 1, - polling_batch_size: int = 10, - local_session: bool = False - ): - """Initializes a PymoniK client instance. - - This constructor sets up the configuration for interacting with an ArmoniK - cluster. It can be configured to run as a client (submitting tasks) - or on the worker (subtasking). - - Args: - endpoint: The gRPC endpoint of the ArmoniK control plane - (e.g., "localhost:5001"). If None, PymoniK might attempt - to discover it from environment variables (e.g., AKCONFIG) - during the `create()` call. - partition: The ArmoniK partition ID where tasks will be - submitted or processed. Defaults to "pymonik". - environment: A dictionary specifying the execution environment - for tasks. This can include configurations for dependencies, - file mounts, or environment variables for the task runtime. - Defaults to an empty dictionary. - is_worker: If True, this instance operates in worker mode. - Worker mode instances are typically managed by the ArmoniK agent - and receive tasks to execute. They do not create sessions or - submit new top-level tasks themselves but can submit sub-tasks. - Defaults to False (client mode). - batch_size: Default batch size for certain ArmoniK operations, - such as creating multiple results or submitting tasks in bulk. - Defaults to 32. - task_options: Default `TaskOptions` to be used for tasks submitted - by this client. These options include settings like maximum - duration, priority, and retry attempts for tasks. If None, - a default set of `TaskOptions` will be generated. - disable_events_client: A flag to disable the use of the events client. This switches - to a polling based approach for waiting for results. - polling_interval: When using the polling based approach, polling interval in seconds. - polling_batch_size: Batch size to use when polling for results. - local_session: A flag intended to control session behavior, - for local testing, it makes it so your function invokes execute locally. - Note: This parameter is not actively used in the current - `__init__` body's logic but is stored for potential future use. - Defaults to False. - """ - self._endpoint = endpoint - self._partition = partition - self.task_options = task_options if task_options is not None else TaskOptions( - max_duration=timedelta(seconds=300), - priority=1, - max_retries=5, - partition_id=self._partition if isinstance(self._partition, str) else self._partition[0], # if using multiple partitions and no task options we just use the first partition specified, - ) - self._connected = False - self._session_created = False - self.remote_functions = {} # TODO: I should probably delete all these results when a session is closed. - self.environment = environment - self._token: Optional[contextvars.Token] = None - self._is_worker_mode = is_worker - self.disable_events_client = disable_events_client - self.polling_interval = polling_interval - self.polling_batch_size = polling_batch_size - self.batch_size = batch_size - self.task_handler: Optional[TaskHandler] = None - self._original_sigint_handler = None - self._sigint_handler_set = False - - def _handle_ctrl_c(self, signum, frame): - """Custom SIGINT handler to cancel the session.""" - print(f"\nCtrl+C detected! Cancelling PymoniK session {self._session_id}...", file=sys.stderr) - # It's important that cancel() is somewhat idempotent or handles being called multiple times - self.cancel() - - # Restore the original handler before raising KeyboardInterrupt - # This is important if the program has its own KeyboardInterrupt handling - # or to ensure default behavior if this handler is somehow invoked again. - if self._original_sigint_handler is not None: - try: - signal.signal(signal.SIGINT, self._original_sigint_handler) - except (ValueError, OSError): # pragma: no cover (e.g. if not in main thread) - pass # Ignore if we can't restore (e.g., not in main thread) - self._original_sigint_handler = None # Clear it after restoring - - self._sigint_handler_set = False # Mark that our handler is no longer active - - # Re-raise KeyboardInterrupt to ensure the script terminates as expected - # or allows for further user-defined KeyboardInterrupt handling. - raise KeyboardInterrupt - - def _dispatch_create_metadata(self, names: List[str]) -> Dict[str, Result]: - """Internal method to create result metadata, dispatching to worker/client.""" - if self.is_worker(): - if not self.task_handler: - raise RuntimeError("Task handler not available in worker mode.") - # TaskHandler uses batch_size internally in the method call - return self.task_handler.create_results_metadata( - names, batch_size=self.batch_size - ) - else: - if not self._results_client: - raise RuntimeError("Results client not initialized.") - # ArmoniKResults client takes session_id and batch_size explicitly - return self._results_client.create_results_metadata( - names, self._session_id, batch_size=self.batch_size - ) - - def _dispatch_upload_payload(self, name: str, payload: bytes | bytearray) -> Dict[str, Result]: - """Internal method to create upload data, dispatching to worker/client.""" - if self.is_worker(): - if not self.task_handler: - raise RuntimeError("Task handler not available in worker mode.") - # TaskHandler uses batch_size internally in the method call - raise NotImplementedError( - "TaskHandler does not support upload payloads." - ) - else: - if not self._results_client: - raise RuntimeError("Results client not initialized.") - # ArmoniKResults client takes session_id and batch_size explicitly - return self._results_client.upload_result_data( - name, self._session_id, result_data=payload - ) - - - def _dispatch_create_payloads( - self, payloads: Dict[str, bytes] - ) -> Dict[str, Result]: - """Internal method to create results with data (payloads), dispatching.""" - if self.is_worker(): - if not self.task_handler: - raise RuntimeError("Task handler not available in worker mode.") - return self.task_handler.create_results( - payloads, batch_size=self.batch_size - ) - else: - if not self._results_client: - raise RuntimeError("Results client not initialized.") - return self._results_client.create_results( - payloads, self._session_id, batch_size=self.batch_size - ) - - def _dispatch_submit_tasks(self, task_definitions: List[TaskDefinition], task_options: Optional[TaskOptions] = None) -> None: - """Internal method to submit tasks, dispatching to worker/client.""" - if self.is_worker(): - if not self.task_handler: - raise RuntimeError("Task handler not available in worker mode.") - self.task_handler.submit_tasks( - task_definitions, - batch_size=self.batch_size, # NOTE: this is bad, really bad (set client side but we just use the default for worker) - default_task_options=task_options - ) - else: - if not self._tasks_client: - raise RuntimeError("Tasks client not initialized.") - self._tasks_client.submit_tasks(self._session_id, task_definitions, default_task_options=task_options) - - - def _wait_for_results_availability(self, session_id: str, result_ids: List[str]): - if self.disable_events_client: - if not result_ids: - return - - for batch_of_ids in batched(result_ids, self.polling_batch_size): - if not batch_of_ids: # This should not happen (please) - continue - - _poll_batch_for_results( - results_client=self._results_client, - result_ids_in_batch=list(batch_of_ids), - polling_interval_seconds=self.polling_interval - ) - else: - if self._events_client is None: - raise RuntimeError( - "Events client (self._events_client) is not initialized. " - "Ensure Pymonik.create() has been called or is active in the current context." - ) - return self._events_client.wait_for_result_availability( - result_ids=result_ids, - session_id=session_id, - bucket_size=self.batch_size, # Use Pymonik's configured batch_size - parallelism=1 # Sensible default for events client path here - ) - - def register_tasks(self, tasks: List[Task]): - """Register a task with the PymoniK instance.""" - pickled_functions = {} - for task in tasks: - remote_function_name = self._session_id + "__function__" + task.func_name - if remote_function_name in self.remote_functions: - # This shouldn't be a full failure, but a warning, esp. in the case where the user is trying to register stuff manually (TODO: when logging is in) - raise ValueError( - f"Task with name {task.func_name} is already registered. " - ) - - pickled_functions[remote_function_name] = pickle.dumps(task.func) - # Upload the pickled functions to the cluster - # NOTE: This is really bad for subtasking, other option would be to get results before invoke to check if task is already registered in this session and if so reuse it - upload_results = self._dispatch_create_payloads( - payloads=pickled_functions, - ) - - # Register the function - self.remote_functions.update(upload_results) - - return self - - def _zip_directory(self, dir_path: str) -> bytes: - """Zips the contents of a directory and returns the bytes.""" - if not os.path.isdir(dir_path): - raise ValueError(f"Path {dir_path} is not a valid directory.") - - zip_buffer = io.BytesIO() - with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zipf: - for root, _, files in os.walk(dir_path): - for file in files: - file_path = os.path.join(root, file) - # Create arcname relative to the directory being zipped - arcname = os.path.relpath(file_path, dir_path) - zipf.write(file_path, arcname) - - zip_buffer.seek(0) - return zip_buffer.getvalue() - - # TODO: put the task_handler and expected_output inside kwargs since they're only used internally inside the task context. - # TODO: support TaskOptions as a parameter for PymoniK - def create( - self, - task_handler: Optional[TaskHandler] = None, - expected_output: Optional[str] = None, - ) -> "Pymonik": - """Initialize client connections and create a session. - - Args: - task_handler (Optional[TaskHandler]): The task handler to use in worker mode. - Returns: - Pymonik: The current instance of Pymonik. - """ - if self._is_worker_mode: - if task_handler is None: - raise ValueError("TaskHandler must be provided in worker mode.") - self.task_handler = task_handler - self._connected = True # Mark as 'connected' in worker context - self._session_id = task_handler.session_id # Get session from handler - self._session_created = True # Mark session as 'created' in worker context - self.parent_task_result_id = expected_output # Store the expected output ID for the parent task to be used for subtasking. - return self - - if self._connected: - return self - - # TODO: Cloudpickle goes in here (maintain registrar of serialized functions, send them over during init, can also do dank thing here like with unison) - - # Initialize clients - if self._endpoint != None: - # TODO: Add parameters for TLS - self._channel = create_grpc_channel(self._endpoint) - else: - # Check if AKCONFIG is defined - akconfig_value = os.getenv("AKCONFIG") - if akconfig_value is None: - raise RuntimeError( - "No endpoint provided and AKCONFIG environment variable is not set." - ) - else: - # Load the AKCONFIG file - with open(akconfig_value, "r") as f: - config = yaml.safe_load(f) - self._endpoint = config.get("endpoint") - certificate_authority = config.get("certificate_authority") - client_certificate = config.get("client_certificate") - client_key = config.get("client_key") - self._channel = create_grpc_channel( - self._endpoint, - certificate_authority=certificate_authority, - client_certificate=client_certificate, - client_key=client_key, - ) - - self._tasks_client = ArmoniKTasks(self._channel) - self._results_client = ArmoniKResults(self._channel) - self._sessions_client = ArmoniKSessions(self._channel) - self._events_client = ArmoniKEvents(self._channel) - self._connected = True - - # Create a session - self._session_id = self._sessions_client.create_session( - default_task_options=self.task_options, - partition_ids=[self._partition] if isinstance(self._partition, str) else self._partition, - ) - self._session_created = True - print(f"Session {self._session_id} has been created") - - # Upload environment data if needed - # TODO: doesn't work as of right now - # if self.environment and "mount" in self.environment: - # mounts_to_upload = {} - # mount_name_to_target_map = {} # Maps temporary result name to mount_to path - - # original_mounts = self.environment.get("mount", []) - # if not isinstance(original_mounts, list): - # print( - # f"Warning: 'mount' in environment should be a list of tuples. Skipping mount processing." - # ) # Or raise error - # original_mounts = [] # Clear it to avoid later errors - - # for mount_info in original_mounts: - # if not isinstance(mount_info, tuple) or len(mount_info) != 2: - # print( - # f"Warning: Invalid mount entry {mount_info}. Expected (mount_from, mount_to). Skipping." - # ) - # continue - - # mount_from, mount_to = mount_info - # print( - # f"Processing mount: Zipping {mount_from} for target {mount_to}..." - # ) - # try: - # zip_bytes = self._zip_directory(mount_from) - # # Create a unique name for the result payload for this mount - # cleaned_mount_from = mount_from.replace("/", "_").replace("\\", "_") - # mount_result_name = ( - # f"{self._session_id}__mount_data__{cleaned_mount_from}" - # ) - # mounts_to_upload[mount_result_name] = zip_bytes - # mount_name_to_target_map[mount_result_name] = mount_to - # print( - # f" ... Zipped {mount_from} ({len(zip_bytes)} bytes) -> {mount_result_name}" - # ) - # except Exception as e: - # print( - # f" ... Error zipping directory {mount_from}: {e}. Skipping this mount." - # ) - # # Decide if this should be a fatal error or just skip - # # raise # Uncomment to make it fatal - - # if mounts_to_upload: - # print(f"Uploading {len(mounts_to_upload)} zipped directories...") - # # Upload the zipped directories as results - # upload_results = self._results_client.create_results( - # results_data=mounts_to_upload, - # session_id=self._session_id, - # ) - # print(" ... Upload complete.") - - # # Update self.environment["mount"] to store (result_id, mount_to) pairs - # updated_mount_info = [] - # for mount_result_name, output in upload_results.items(): - # if mount_result_name in mount_name_to_target_map: - # mount_to = mount_name_to_target_map[mount_result_name] - # result_id = output.result_id - # updated_mount_info.append((result_id, mount_to)) - # print( - # f" ... Mapped {mount_result_name} (Result ID: {result_id}) to target path {mount_to}" - # ) - # else: - # # This case should ideally not happen if logic is correct - # print( - # f"Warning: Uploaded result {mount_result_name} not found in mapping. Inconsistency detected." - # ) - - # self.environment["mount"] = updated_mount_info - # else: - # # If nothing was successfully zipped and prepared for upload - # self.environment[ - # "mount" - # ] = [] # Ensure it's an empty list if mounts were requested but failed - - - return self - - def _ensure_client_ready(self): - if self.is_worker(): - raise RuntimeError("Client operation attempted in worker mode.") - if not self._connected or not self._session_created: - self.create() - if not self._results_client: - raise RuntimeError("Results client not initialized after create().") - if not self._session_id: - raise RuntimeError("Session ID not available after create().") - - def upload_materialize(self, mat: Materialize, force_upload: bool = False) -> Materialize: - """ - Upload a Materialize object to ArmoniK if it doesn't already exist. - - Args: - mat: Materialize object to upload - - Returns: - Materialize: Updated materialize object with result_id set - """ - self._ensure_client_ready() - - # Check if result with this hash already exists - hash_result_name = f"materialize_{mat.content_hash}" - - try: - # Try to list results to see if one with this name exists - # This is a simplified approach but it should work right? - # ... surely..? - - # Query for existing results with our hash name - existing_results = self._results_client.get_results_ids( - session_id=self._session_id, - names=[hash_result_name] - ) - print(f"Existing Results = {existing_results}") - if not force_upload and hash_result_name in existing_results: - existing_result_id = existing_results[hash_result_name] - print(f"Materialize content with hash {mat.content_hash} already exists: {existing_result_id}") - mat.result_id = existing_result_id - return mat - - except Exception as e: - # If query fails, proceed with upload - print(f"Could not check for existing materialize content: {e}") - - # Prepare content for upload - if mat.is_directory: - content_bytes = _create_zip_from_directory(mat.source_path) - else: - with open(mat.source_path, 'rb') as f: - content_bytes = f.read() - # Upload to ArmoniK - upload_results = self._dispatch_create_payloads({hash_result_name: content_bytes}) - mat.result_id = upload_results[hash_result_name].result_id - - print(f"Uploaded materialize content: {mat.source_path} -> {mat.result_id} (hash: {mat.content_hash})") - return mat - - def put(self, obj: U_Obj, name: Optional[str] = None) -> ResultHandle[U_Obj]: - """ - Uploads a single Python object to ArmoniK. - - Args: - obj: The Python object to upload. - name: An optional name for this data. Used for traceability. - - Returns: - A ResultHandle for the uploaded object. - """ - self._ensure_client_ready() # Ensures create() is called if needed for client mode - - payload_bytes = pickle.dumps(obj) - - descriptive_name_part = name if name else str(uuid.uuid4()) - # This is the key used in the dictionary for _dispatch_create_payloads - internal_payload_key = f"pymonik_put_data__{descriptive_name_part}" - - payloads_to_upload = {internal_payload_key: payload_bytes} - - # _dispatch_create_payloads returns a Dict[str, Result] - # The keys in the returned dict match the keys in payloads_to_upload - created_armonik_results_map = self._dispatch_create_payloads(payloads_to_upload) - - armonik_result_obj = created_armonik_results_map[internal_payload_key] - - return ResultHandle[U_Obj]( - result_id=armonik_result_obj.result_id, - session_id=self._session_id, # type: ignore (self._session_id is confirmed by _ensure_client_ready) - pymonik_instance=self - ) - - def put_many(self, objects: List[V_Obj], names: Optional[List[str]] = None) -> List[ResultHandle[V_Obj]]: - """ - Uploads multiple Python objects to ArmoniK. - - Args: - objects: A list of Python objects to upload. - names: An optional list of names for these objects. If provided, - its length must match the length of objects. - - Returns: - A list of ResultHandles for the uploaded objects, in the same order. - """ - self._ensure_client_ready() - - if not objects: - return [] - - if names and len(objects) != len(names): - raise ValueError("Length of objects and names must match if names are provided.") - - payloads_to_upload: Dict[str, bytes] = {} - ordered_internal_keys: List[str] = [] - - for i, obj in enumerate(objects): - payload_bytes = pickle.dumps(obj) - - descriptive_name_part = names[i] if names else str(uuid.uuid4()) - internal_payload_key = f"pymonik_put_many_data__{i}__{descriptive_name_part}" # Add index for more uniqueness - - payloads_to_upload[internal_payload_key] = payload_bytes - ordered_internal_keys.append(internal_payload_key) - - created_armonik_results_map = self._dispatch_create_payloads(payloads_to_upload) - - result_handles = [] - for key in ordered_internal_keys: - armonik_result_obj = created_armonik_results_map[key] - result_handles.append( - ResultHandle[V_Obj]( - result_id=armonik_result_obj.result_id, - session_id=self._session_id, # type: ignore - pymonik_instance=self - ) - ) - - return result_handles - - def is_worker(self) -> bool: - """Returns True if running in worker mode, False if in client mode.""" - return self._is_worker_mode - - def close(self): - """Close the session and clean up resources.""" - if self._is_worker_mode: - return - - if self._session_created: - try: - self._sessions_client.close_session(self._session_id) - print(f"Session {self._session_id} has been closed") - self._session_created = False - except Exception as e: - print(f"Error closing session {self._session_id}: {e}") - - if self._connected: - self._channel.close() - self._connected = False - - def cancel(self): - """Cancel the session and clean up resources.""" - if self._is_worker_mode: - return - - if self._session_created: - try: - self._sessions_client.cancel_session(self._session_id) - print(f"Session {self._session_id} has been cancelled") - self._session_created = False - except Exception as e: - print(f"Error cancelling session {self._session_id}: {e}") - - if self._connected: - self._channel.close() - self._connected = False - - def __enter__(self): - """Context manager entry point.""" - # Workers have to create the session on their own - if not self._is_worker_mode and not self._connected: - self.create() - # Set up the SIGINT (Ctrl+C) handler - # This should only be done in the main thread. - try: - current_handler = signal.getsignal(signal.SIGINT) - # Only set our handler if it's the default one or not already our instance's handler - # This check helps prevent issues if __enter__ is called multiple times on the same instance - # without __exit__ (though that would be unusual for a context manager). - if current_handler is signal.default_int_handler or \ - (not hasattr(current_handler, '__self__') or current_handler.__self__ is not self): - self._original_sigint_handler = signal.signal(signal.SIGINT, self._handle_ctrl_c) - self._sigint_handler_set = True - # If current_handler is already self._handle_ctrl_c, _original_sigint_handler - # would point to the one set before this Pymonik instance's handler, - # or signal.default_int_handler if this is the first custom handler. - # This logic aims to correctly chain/restore if multiple Pymonik contexts were nested, - # though typically only one is active via _CURRENT_PYMONIK. - - except (ValueError, OSError, AttributeError): # pragma: no cover - # ValueError: signal only works in main thread - # OSError: can also be raised (e.g. "not in main thread") - # AttributeError: if getsignal returns something unexpected - self._original_sigint_handler = None # Indicate we couldn't set it - self._sigint_handler_set = False - print("Warning: Could not set SIGINT handler. Ctrl+C might not cancel the session gracefully.", file=sys.stderr) - - self._token = _CURRENT_PYMONIK.set(self) - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """Context manager exit point.""" - if self._token: - _CURRENT_PYMONIK.reset(self._token) - self._token = None - # Restore the original SIGINT handler if we set one - if not self._is_worker_mode and self._sigint_handler_set: - if self._original_sigint_handler is not None: - try: - signal.signal(signal.SIGINT, self._original_sigint_handler) - except (ValueError, OSError): # pragma: no cover - # In case we are in a state where it cannot be reset (e.g. thread changed) - print("Warning: Could not restore original SIGINT handler.", file=sys.stderr) - self._original_sigint_handler = None # Clear it - self._sigint_handler_set = False - self.close() - return False - -def task( - _func: Optional[Callable[P_Args,R_Type]] = None, - *, - require_context: bool = False, - function_name: Optional[str] = None, - task_options: Optional[TaskOptions] = None, - partition: Optional[str] = None, - max_duration: Optional[Union[timedelta, int, float]] = None, - priority: Optional[int] = None, - max_retries: Optional[int] = None, -) -> Union[Callable, Task]: - """Decorator to create a Task from a function. - - Args: - _func: The function to wrap (used internally by decorator syntax) - require_context: Whether the task requires a PymonikContext - function_name: Custom name for the function - task_options: Complete TaskOptions object to use as defaults - partition: Shortcut to specify partition_id - max_duration: Maximum duration for the task (timedelta, or seconds as int/float) - priority: Task priority - max_retries: Maximum number of retries - - Usage: - @task - def my_func(): - pass - - @task(partition="gpu", max_duration=600, priority=2) - def gpu_func(): - pass - - @task(task_options=TaskOptions(max_duration=timedelta(minutes=10))) - def complex_func(): - pass - """ - def decorator(func: Callable[P_Args,R_Type]) -> Task[P_Args,R_Type]: - resolved_name = function_name or func.__name__ - - # Build task options from individual parameters - decorator_task_options = None - if (task_options is not None or - partition is not None or - max_duration is not None or - priority is not None or - max_retries is not None): - - # Start with provided task_options or create new one - if task_options is not None: - # Copy the existing task options - base_max_duration = task_options.max_duration - base_priority = task_options.priority - base_max_retries = task_options.max_retries - base_partition_id = task_options.partition_id - else: - # Use None as default, will be filled by Pymonik defaults later - base_max_duration = None - base_priority = None - base_max_retries = None - base_partition_id = None - - # Override with individual parameters - final_max_duration = base_max_duration - if max_duration is not None: - if isinstance(max_duration, (int, float)): - final_max_duration = timedelta(seconds=max_duration) - elif isinstance(max_duration, timedelta): - final_max_duration = max_duration - - final_priority = priority if priority is not None else base_priority - final_max_retries = max_retries if max_retries is not None else base_max_retries - final_partition_id = partition if partition is not None else base_partition_id - - decorator_task_options = TaskOptions( - max_duration=final_max_duration, - priority=final_priority, - max_retries=final_max_retries, - partition_id=final_partition_id, - ) - - # # TODO: Remove - # print(f"Decorator Task Options {decorator_task_options}") - - return Task[P_Args,R_Type]( - func, - require_context=require_context, - func_name=resolved_name, - task_options=decorator_task_options - ) - - if _func is None: - # Case 1: Called with arguments - @task(...) - return decorator - else: - # Case 2: Called without arguments - @task - return decorator(_func) diff --git a/pymonik/src/pymonik/environment.py b/pymonik/src/pymonik/environment.py deleted file mode 100644 index 3948dfe..0000000 --- a/pymonik/src/pymonik/environment.py +++ /dev/null @@ -1,160 +0,0 @@ -from logging import Logger -import subprocess -import sys -import os -from typing import Any, Dict -import importlib - - -class RuntimeEnvironment: - """ - A class to manage the runtime environment for Python packages. - """ - - def __init__(self, logger: Logger = None): - self.logger = logger - self.python_executable = sys.executable - self.venv_path = os.path.dirname(self.python_executable) - self.pip_executable = ( - os.path.join(self.venv_path, "Scripts", "pip") - if os.name == "nt" - else os.path.join(self.venv_path, "bin", "pip") - ) - - def get_python_executable(self): - return self.python_executable - - def get_venv_path(self): - return self.venv_path - - def get_pip_executable(self): - return self.pip_executable - - def install_package(self, package_name: str, version: str = None): - """ - Installs a Python package using uv. - Args: - package_name: The name of the package to install. - version: Optional specific version string (e.g., '==1.2.3', '>=1.0'). - """ - # Check if uv is callable first (If this fails ) - try: - subprocess.run(["uv", "--version"], check=True, capture_output=True) - self.logger.info("uv command found.") - except (subprocess.CalledProcessError, FileNotFoundError): - self.logger.error( - "uv command not found on PATH. Cannot install packages dynamically this way." - ) - return False - - self.logger.info( - f"\nAttempting to install {package_name}{version or ''} using uv..." - ) - - package_spec = package_name - if version: - # Basic check - might need adjustment based on uv's exact specifier support - if not all( - c.isalnum() or c in [".", "=", "<", ">", "!", "~"] for c in version - ): # Added ~ for compatible releases - self.logger.error( - f"Error: Potentially invalid characters in version specifier for uv: {version}" - ) - # return False # Decide if you want to block or let uv handle potential errors - package_spec += version - - # --- Use uv command directly --- - # Note: uv pip install runs in the context of the current environment - # if a venv is active or detected, similar to pip. - command = ["uv", "pip", "install", package_spec] - - self.logger.info(f"Running command: {' '.join(command)}") - - try: - process = subprocess.run( - command, - capture_output=True, - text=True, - check=True, - env=os.environ.copy(), - # Consider specifying the target python/venv if needed, though uv - # usually detects the active one correctly. - # Example: command = ['uv', 'pip', 'install', package_spec, '--python', sys.executable] - ) - - self.logger.info(f"Successfully installed {package_spec} using uv") - self.logger.debug(f"Install STDOUT:\n{process.stdout}") - self.logger.debug( - f"Install STDERR:\n{process.stderr}" - ) # uv might output progress here - - # ... (rest of the importlib reload logic remains the same) ... - try: - module_name_import = package_name.replace("-", "_") - module = importlib.import_module(module_name_import) - importlib.reload(module) - self.logger.info(f"Module '{module_name_import}' reloaded/available.") - except ImportError: - self.logger.warning( - f"Could not import '{module_name_import}' immediately after install. May require script restart." - ) - except Exception as e: - self.logger.error(f"Error reloading module {module_name_import}: {e}") - - return True - - except subprocess.CalledProcessError as e: - self.logger.error( - f"Error installing {package_spec} using uv; command failed." - ) - self.logger.error(f"Return Code: {e.returncode}") - self.logger.error(f"STDOUT:\n{e.stdout}") - self.logger.error( - f"STDERR:\n{e.stderr}" - ) # This should contain the uv error - return False - except Exception as e: - self.logger.error( - f"An unexpected error occurred during uv installation setup: {e}" - ) - return False - - def construct_environment(self, environment_info: Dict[str, Any]): - """ - Constructs the runtime environment for the Python packages. - """ - self.logger.info(f"Constructing runtime environment {environment_info}...") - if "pip" in environment_info: - pip_info = environment_info["pip"] - if isinstance(pip_info, list): - for package in pip_info: - if isinstance(package, str): - self.install_package(package) - elif isinstance(package, tuple): # Tuple maybe ? - for package_name, version in package.items(): - self.install_package(package_name, version) - else: - self.logger.error(f"Invalid package specification: {package}") - else: - self.logger.error("Pip information is not a list.") - if "env_variables" in environment_info: - env_vars = environment_info["env_variables"] - if isinstance(env_vars, dict): - for key, value in env_vars.items(): - os.environ[key] = value - self.logger.info(f"Set environment variable {key} to {value}") - else: - self.logger.error( - "Environment variables information is not a dictionary." - ) - # if working directory is specified download the data (TODO: This isn't supported yet) - if "mount" in environment_info: - mount = environment_info["mount"] - if isinstance(mount, list): - for path in mount: - if os.path.exists(path): - self.logger.info(f"Mounting {path}...") - else: - self.logger.error(f"Path {path} does not exist.") - else: - self.logger.error("Mount information is not a list.") diff --git a/pymonik/src/pymonik/materialize.py b/pymonik/src/pymonik/materialize.py deleted file mode 100644 index b4e1244..0000000 --- a/pymonik/src/pymonik/materialize.py +++ /dev/null @@ -1,112 +0,0 @@ - # if pymonik is None: - # pymonik = _CURRENT_PYMONIK.get(None) - # if pymonik is None: - # raise RuntimeError( - # "No active PymoniK instance found. Please create one and pass it in or use the context manager." - # ) - - - -import hashlib -import io -import os -import zipfile -from pathlib import Path -from typing import Optional, Union -import cloudpickle as pickle -from dataclasses import dataclass - - -@dataclass -class Materialize: - """ - Represents a file or directory that should be materialized in the worker. - Files are stored content-addressably using SHA-256 hashes. - """ - source_path: str # Original local path - worker_path: str # Target path in worker - content_hash: str # SHA-256 hash of the content - is_directory: bool # Whether the source was a directory (and thus zipped) - result_id: Optional[str] = None # Set after upload to ArmoniK - - def __post_init__(self): - # Ensure paths are normalized - self.source_path = str(Path(self.source_path).resolve()) - self.worker_path = str(Path(self.worker_path)) - - -def _calculate_file_hash(file_path: Union[str, Path]) -> str: - """Calculate SHA-256 hash of a file.""" - hasher = hashlib.sha256() - with open(file_path, 'rb') as f: - for chunk in iter(lambda: f.read(8192), b""): - hasher.update(chunk) - return hasher.hexdigest() - - -def _calculate_directory_hash(dir_path: Union[str, Path]) -> str: - """Calculate SHA-256 hash of a directory by hashing its zipped contents.""" - zip_buffer = io.BytesIO() - with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zipf: - dir_path = Path(dir_path) - for file_path in sorted(dir_path.rglob('*')): - if file_path.is_file(): - arcname = file_path.relative_to(dir_path) - zipf.write(file_path, arcname) - - zip_buffer.seek(0) - hasher = hashlib.sha256() - for chunk in iter(lambda: zip_buffer.read(8192), b""): - hasher.update(chunk) - return hasher.hexdigest() - - -def _create_zip_from_directory(dir_path: Union[str, Path]) -> bytes: - """Create a zip file from a directory and return its bytes.""" - zip_buffer = io.BytesIO() - with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zipf: - dir_path = Path(dir_path) - for file_path in sorted(dir_path.rglob('*')): - if file_path.is_file(): - arcname = file_path.relative_to(dir_path) - zipf.write(file_path, arcname) - - zip_buffer.seek(0) - return zip_buffer.getvalue() - - -def materialize(source_path: Union[str, Path], worker_path: Union[str, Path]) -> Materialize: - """ - Create a Materialize object for a file or directory. - - Args: - source_path: Local file or directory path to materialize - worker_path: Target path in the worker where the file/directory should be placed - - Returns: - Materialize: Object representing the materialized content - - Raises: - FileNotFoundError: If source_path doesn't exist - ValueError: If source_path is neither a file nor directory - """ - source_path = Path(source_path) - - if not source_path.exists(): - raise FileNotFoundError(f"Source path does not exist: {source_path}") - - if source_path.is_file(): - content_hash = _calculate_file_hash(source_path) - is_directory = False - elif source_path.is_dir(): - content_hash = _calculate_directory_hash(source_path) - is_directory = True - else: - raise ValueError(f"Source path must be a file or directory: {source_path}") - - return Materialize( - source_path=str(source_path), - worker_path=str(worker_path), - content_hash=content_hash, - is_directory=is_directory - ) diff --git a/pymonik/src/pymonik/results.py b/pymonik/src/pymonik/results.py deleted file mode 100644 index 2597646..0000000 --- a/pymonik/src/pymonik/results.py +++ /dev/null @@ -1,140 +0,0 @@ -import cloudpickle as pickle -from typing import Generic, TypeVar, get_args, List - -T = TypeVar("T") - - -# TODO: Generics for better typing ... ResultHandle[str] for example.. -class ResultHandle(Generic[T]): - """A handle to a future result from an ArmoniK task.""" - - def __init__(self, result_id: str, session_id: str, pymonik_instance: "Pymonik"): - self.result_id = result_id - self.session_id: str = session_id - self._pymonik = pymonik_instance - - def wait(self) -> "ResultHandle[T]": - """Wait for the result to be available.""" - if self._pymonik.is_worker(): - raise RuntimeError( - "Cannot wait for result in worker context. Use the client context instead." - ) - try: - self._pymonik._wait_for_results_availability( - self.session_id, [self.result_id] - ) - return self - except Exception as e: - print(f"Error waiting for result {self.result_id}: {e}") - raise - - def get(self) -> T: - """Get the result value.""" - result_data = self._pymonik._results_client.download_result_data( - self.result_id, self.session_id - ) - return pickle.loads(result_data) - - def __repr__(self): - type_str = "T" # Default to the TypeVar name if not specialized - - try: - # __orig_class__ is set if the instance is created from a - # specialized generic type, e.g. ResultHandle[str] - if hasattr(self, "__orig_class__"): - type_args = get_args(self.__orig_class__) - if type_args: - actual_type_arg = type_args[0] - if isinstance(actual_type_arg, TypeVar): - type_str = actual_type_arg.__name__ # e.g., "T" - else: - # For concrete types like or typing.List[int] - type_str = str(actual_type_arg) - # Make it prettier - if type_str.startswith("typing."): - type_str = type_str[len("typing.") :] - if type_str.startswith(" -> int - type_str = type_str[len("" - - -# TODO: implement _results_as_completed for retrieving results as they're completed -# nvm maybe this is better, it'd be weird to fetch things when you iterate, implicit behavior bad.. -class MultiResultHandle: - """A handle to multiple future results from ArmoniK tasks.""" - - def __init__(self, result_handles: List[ResultHandle]): - self.result_handles = result_handles - if result_handles: - self._pymonik = result_handles[0]._pymonik - self.session_id = result_handles[0].session_id - else: - self._pymonik = None - self.session_id = None - - def wait(self): - """Wait for all results to be available.""" - if not self.result_handles: - return self - - result_ids = [handle.result_id for handle in self.result_handles] - try: - self._pymonik._wait_for_results_availability( - self.session_id, result_ids - ) - return self - except Exception as e: - print(f"Error waiting for results: {e}") - raise - - def get(self): - """Get all result values.""" - # TODO: maybe should cache the get - return [handle.get() for handle in self.result_handles] - - def append(self, other): - if isinstance(other, ResultHandle): - self.result_handles.append(other) - else: - raise TypeError(f'Cannot append a "{type(other).__name__}" type to a MultiResultHandle, append parmeter must be ResultHandle type') - - def extend(self, other): - if isinstance(other, MultiResultHandle): - self.result_handles.extend(other) - elif isinstance(other, list) and all(isinstance(x, ResultHandle) for x in other): - self.result_handles.extend(other) - else: - raise TypeError(f'Cannot extend with a "{type(other).__name__}" type, extend parmeter must be MultiResultHandle or List[ResultHandle] type') - - def __iter__(self): - return iter(self.result_handles) - - def __getitem__(self, index): - if isinstance(index, slice): - return MultiResultHandle(self.result_handles[index]) - elif isinstance(index, int): - return self.result_handles[index] - else: - raise TypeError("Index must be an integer or a slice.") - - def __len__(self): - return len(self.result_handles) - - def __repr__(self): - return f"" - -class RemoteFile: - def __init__(self) -> None: - pass diff --git a/pymonik/src/pymonik/utils.py b/pymonik/src/pymonik/utils.py deleted file mode 100644 index 925b868..0000000 --- a/pymonik/src/pymonik/utils.py +++ /dev/null @@ -1,105 +0,0 @@ -import time -import grpc -import cloudpickle as pickle - -from typing import List, Optional, Set, Union -from armonik.common import create_channel, Result, ResultStatus -from armonik.client import ArmoniKResults - -def create_grpc_channel( - endpoint: str, - certificate_authority: Optional[str] = None, - client_certificate: Optional[str] = None, - client_key: Optional[str] = None, -) -> grpc.Channel: - """ - Create a gRPC channel based on the configuration. - """ - cleaner_endpoint = endpoint - if cleaner_endpoint.startswith("http://"): - cleaner_endpoint = cleaner_endpoint[7:] - if cleaner_endpoint.endswith("/"): - cleaner_endpoint = cleaner_endpoint[:-1] - if certificate_authority: - # Create grpc channel with tls - channel = create_channel( - cleaner_endpoint, - certificate_authority=certificate_authority, - client_certificate=client_certificate, - client_key=client_key, - ) - else: - # Create insecure grpc channel - channel = grpc.insecure_channel(cleaner_endpoint) - return channel - - -class LazyArgs: - def __init__(self, args_to_pickle): - # We store the *pickled* representation of the arguments, not the arguments themselves. - self.pickled_args = pickle.dumps(args_to_pickle) # Pickle the arguments - self._args = None # Initially, the arguments are not loaded. - - def get_args(self): - # This method is responsible for actually loading (unpickling) the arguments, but *only* when they are requested. - if self._args is None: - print( - "Loading args..." - ) # Simulate the loading/unpickling process. Crucially, this happens *after* environment setup. - self._args = pickle.loads(self.pickled_args) # Unpickle only when needed - return self._args - - def __repr__(self): - return f"" if self._args is None else repr(self._args) - - -def _poll_batch_for_results( - results_client: ArmoniKResults, - result_ids_in_batch: List[str], - polling_interval_seconds: float, -) -> None: - """ - Polls for the completion or abortion of a batch of results. - """ - if not result_ids_in_batch: - return - - not_found: Set[str] = set(result_ids_in_batch) - - while not_found: - current_filter = None - for r_id in not_found: - filter_condition = (Result.result_id == r_id) - if current_filter is None: - current_filter = filter_condition - else: - current_filter = current_filter | filter_condition - - if current_filter is None: # Should not happen if not_found is populated - break - - try: - _total, fetched_results = results_client.list_results( - result_filter=current_filter, - page=0, - page_size=len(not_found), - ) - for res_summary in fetched_results: - if res_summary.result_id in not_found: - if res_summary.status == ResultStatus.COMPLETED: - not_found.remove(res_summary.result_id) - elif res_summary.status == ResultStatus.ABORTED: - raise RuntimeError(f"Result {res_summary.result_id} has been aborted.") - - if not not_found: # All results in this batch are completed - break - - except grpc.RpcError: - # Basic retry on RpcError. - pass - except RuntimeError: # Re-raise "Result ... has been aborted." - raise - except Exception as e: - raise RuntimeError(f"An unexpected error occurred while polling for results batch: {e}") - - time.sleep(polling_interval_seconds) \ No newline at end of file diff --git a/pymonik/src/pymonik/worker.py b/pymonik/src/pymonik/worker.py deleted file mode 100644 index 8916bd9..0000000 --- a/pymonik/src/pymonik/worker.py +++ /dev/null @@ -1,168 +0,0 @@ -import cloudpickle as pickle - -from .materialize import Materialize -from .core import Pymonik -from .context import PymonikContext -from .environment import RuntimeEnvironment -from .results import ResultHandle, MultiResultHandle - -from armonik.common import Output -from armonik.worker import TaskHandler, armonik_worker, ClefLogger -def _process_materialize_args(func_name, retrieved_args, task_handler, logger): - """ - Process task arguments to find and materialize any Materialize objects. - This should be called in run_pymonik_worker before executing the task. - """ - try: - logger.debug(f"Starting _process_materialize_args for {func_name}") - logger.debug(f"Retrieved args count: {len(retrieved_args)}") - - # Log each argument type - for i, arg in enumerate(retrieved_args): - logger.debug(f"Arg {i}: type={type(arg)}, value={repr(arg) if not isinstance(arg, Materialize) else f'Materialize(source={arg.source_path}, worker={arg.worker_path}, hash={arg.content_hash}, result_id={arg.result_id})'}") - - # Create context for materialization - ctx = PymonikContext(task_handler, logger) - logger.debug(f"Created PymonikContext, is_local={ctx.is_local}") - - materialize_count = 0 - for i, arg in enumerate(retrieved_args): - if isinstance(arg, Materialize): - logger.debug(f"Found Materialize argument at position {i}: {arg.source_path} -> {arg.worker_path}") - logger.debug(f"Materialize result_id: {arg.result_id}") - logger.debug(f"Materialize content_hash: {arg.content_hash}") - logger.debug(f"Materialize is_directory: {arg.is_directory}") - - success = ctx.materialize_file(arg) - if success: - materialize_count += 1 - logger.debug(f"Successfully materialized: {arg.worker_path}") - else: - logger.error(f"Failed to materialize: {arg.worker_path}") - # Note: We don't fail the task, just log the error - # The task will receive the Materialize object and can handle the failure - else: - logger.debug(f"Arg {i} is not a Materialize object (type: {type(arg)})") - - if materialize_count > 0: - logger.debug(f"Processed {materialize_count} Materialize objects for task {func_name}") - else: - logger.debug(f"No Materialize objects found in {len(retrieved_args)} arguments for task {func_name}") - - except Exception as e: - logger.error(f"Error processing Materialize arguments: {e}") - import traceback - logger.error(f"Traceback: {traceback.format_exc()}") - # Don't fail the task, just log the error - -def run_pymonik_worker(): - """Run the worker.""" - - @armonik_worker() - def processor(task_handler: TaskHandler) -> Output: - try: - logger = ClefLogger.getLogger("ArmoniKWorker") - logger.info("Starting PymoniK worker... Loading the payload") - # Deserialize the payload - payload = pickle.loads(task_handler.payload) - func_name = payload["func_name"] - func_id = payload["func_id"] - require_context = payload["require_context"] - args = payload["args"] - requested_environment = payload["environment"] - logger.info( - f"Processing task {task_handler.task_id} : {func_name} -> {func_id} with arguments {args} in session {task_handler.session_id} " - ) - # # Look up the function - # if func_name not in self._registered_tasks: - # return Output(f"Function {func_name} not found") - - env = RuntimeEnvironment(logger) - env.construct_environment(requested_environment) - - retrieved_args = args.get_args() - logger.info( - f"Retrieved args for task {task_handler.task_id} : {func_name} -> {func_id} : {args} in session {task_handler.session_id} " - ) - logger.debug(f"Retrieved args count: {len(retrieved_args)}") - - # Process arguments, retrieving results if needed - processed_args = [] - for arg in retrieved_args: - if isinstance(arg, str) and arg == "__no_input__": - # Skip NoInput arguments - continue - elif isinstance(arg, str) and arg.startswith("__result_handle__"): - # Retrieve the result data - result_id = arg[len("__result_handle__") :] - result_data = task_handler.data_dependencies[result_id] - processed_args.append(pickle.loads(result_data)) - elif isinstance(arg, str) and arg.startswith("__multi_result_handle__"): - # Retrieve multiple result data - result_ids = arg[len("__multi_result_handle__") :].split(",") - processed_args.append( - [ - pickle.loads(task_handler.data_dependencies[result_id]) - for result_id in result_ids - ] - ) - else: - processed_args.append(arg) - - # Load the function - func = pickle.loads(task_handler.data_dependencies[func_id]) - logger.info( - f"Processing task {task_handler.task_id} : Retrieved function {func_name} from data dependencies" - ) - - # Process materialization BEFORE creating context for the function - logger.info(f"About to process materialize args") - _process_materialize_args(func_name, processed_args, task_handler, logger) - logger.info(f"Finished processing materialize args") - - # Call the function with the arguments - if require_context: - # If the function requires context, pass the task handler - context = PymonikContext( - task_handler, logger - ) # TODO: create the context before and make enrich logs with task/function info - processed_args = [context] + processed_args - else: - # Otherwise, just pass the arguments - processed_args = processed_args - - pymonik_worker_client = Pymonik(is_worker=True) - # TODO: support returning multiple results (I don't have a feel for how this can be done in practice and it's something to look into) - pymonik_worker_client.create( - task_handler=task_handler, - expected_output=task_handler.expected_results[0], - ) - with pymonik_worker_client: - result = func(*processed_args) - - if isinstance(result, ResultHandle) or isinstance( - result, MultiResultHandle - ): - # If the result is a ResultHandle or MultiResultHandle, then there is a delegation going on and we should not send the result - return Output() - # Serialize the result - result_data = pickle.dumps(result) - - # Get the expected result ID - result_id = task_handler.expected_results[0] - - # Send the result - task_handler.send_results({result_id: result_data}) - - return Output() - - except Exception as e: - import traceback - - logger.error( - f"Error processing task {task_handler.task_id} : {e}\n{traceback.format_exc()}" - ) - return Output(f"Error processing task: {e}\n{traceback.format_exc()}") - - # Run the worker - processor.run() \ No newline at end of file diff --git a/pymonik/uv.lock b/pymonik/uv.lock deleted file mode 100644 index cee05f9..0000000 --- a/pymonik/uv.lock +++ /dev/null @@ -1,349 +0,0 @@ -version = 1 -revision = 2 -requires-python = ">=3.10" - -[[package]] -name = "armonik" -version = "3.25.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "deprecation" }, - { name = "grpcio" }, - { name = "grpcio-tools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8e/22/6299b9be5da13e6a909a3dc285e1b4b1b4990da337e598d4863acdc723aa/armonik-3.25.0.tar.gz", hash = "sha256:5fd5114da313b279d28fccdcb4cd1e6e1d0d302f5cf01b83d2ebd888ef1982c9", size = 89599, upload-time = "2025-03-06T14:39:21.779Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/2d/03ba8cad0f8bbe0850cdbee37bc904a70a005552c0091f904a0244223dc4/armonik-3.25.0-py3-none-any.whl", hash = "sha256:2c4f4428264bcf0b4016599441eeaad13f8947cfd3e3398b5a75ef4fe4d70483", size = 148256, upload-time = "2025-03-06T14:39:20.077Z" }, -] - -[[package]] -name = "cffi" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/07/f44ca684db4e4f08a3fdc6eeb9a0d15dc6883efc7b8c90357fdbf74e186c/cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14", size = 182191, upload-time = "2024-09-04T20:43:30.027Z" }, - { url = "https://files.pythonhosted.org/packages/08/fd/cc2fedbd887223f9f5d170c96e57cbf655df9831a6546c1727ae13fa977a/cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67", size = 178592, upload-time = "2024-09-04T20:43:32.108Z" }, - { url = "https://files.pythonhosted.org/packages/de/cc/4635c320081c78d6ffc2cab0a76025b691a91204f4aa317d568ff9280a2d/cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382", size = 426024, upload-time = "2024-09-04T20:43:34.186Z" }, - { url = "https://files.pythonhosted.org/packages/b6/7b/3b2b250f3aab91abe5f8a51ada1b717935fdaec53f790ad4100fe2ec64d1/cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702", size = 448188, upload-time = "2024-09-04T20:43:36.286Z" }, - { url = "https://files.pythonhosted.org/packages/d3/48/1b9283ebbf0ec065148d8de05d647a986c5f22586b18120020452fff8f5d/cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3", size = 455571, upload-time = "2024-09-04T20:43:38.586Z" }, - { url = "https://files.pythonhosted.org/packages/40/87/3b8452525437b40f39ca7ff70276679772ee7e8b394934ff60e63b7b090c/cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6", size = 436687, upload-time = "2024-09-04T20:43:40.084Z" }, - { url = "https://files.pythonhosted.org/packages/8d/fb/4da72871d177d63649ac449aec2e8a29efe0274035880c7af59101ca2232/cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17", size = 446211, upload-time = "2024-09-04T20:43:41.526Z" }, - { url = "https://files.pythonhosted.org/packages/ab/a0/62f00bcb411332106c02b663b26f3545a9ef136f80d5df746c05878f8c4b/cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8", size = 461325, upload-time = "2024-09-04T20:43:43.117Z" }, - { url = "https://files.pythonhosted.org/packages/36/83/76127035ed2e7e27b0787604d99da630ac3123bfb02d8e80c633f218a11d/cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e", size = 438784, upload-time = "2024-09-04T20:43:45.256Z" }, - { url = "https://files.pythonhosted.org/packages/21/81/a6cd025db2f08ac88b901b745c163d884641909641f9b826e8cb87645942/cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be", size = 461564, upload-time = "2024-09-04T20:43:46.779Z" }, - { url = "https://files.pythonhosted.org/packages/f8/fe/4d41c2f200c4a457933dbd98d3cf4e911870877bd94d9656cc0fcb390681/cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c", size = 171804, upload-time = "2024-09-04T20:43:48.186Z" }, - { url = "https://files.pythonhosted.org/packages/d1/b6/0b0f5ab93b0df4acc49cae758c81fe4e5ef26c3ae2e10cc69249dfd8b3ab/cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15", size = 181299, upload-time = "2024-09-04T20:43:49.812Z" }, - { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264, upload-time = "2024-09-04T20:43:51.124Z" }, - { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651, upload-time = "2024-09-04T20:43:52.872Z" }, - { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259, upload-time = "2024-09-04T20:43:56.123Z" }, - { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200, upload-time = "2024-09-04T20:43:57.891Z" }, - { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235, upload-time = "2024-09-04T20:44:00.18Z" }, - { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721, upload-time = "2024-09-04T20:44:01.585Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242, upload-time = "2024-09-04T20:44:03.467Z" }, - { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999, upload-time = "2024-09-04T20:44:05.023Z" }, - { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242, upload-time = "2024-09-04T20:44:06.444Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604, upload-time = "2024-09-04T20:44:08.206Z" }, - { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727, upload-time = "2024-09-04T20:44:09.481Z" }, - { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400, upload-time = "2024-09-04T20:44:10.873Z" }, - { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, - { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, - { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, - { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, - { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, - { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, - { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, - { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, - { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, - { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989, upload-time = "2024-09-04T20:44:28.956Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802, upload-time = "2024-09-04T20:44:30.289Z" }, - { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792, upload-time = "2024-09-04T20:44:32.01Z" }, - { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893, upload-time = "2024-09-04T20:44:33.606Z" }, - { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810, upload-time = "2024-09-04T20:44:35.191Z" }, - { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200, upload-time = "2024-09-04T20:44:36.743Z" }, - { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447, upload-time = "2024-09-04T20:44:38.492Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358, upload-time = "2024-09-04T20:44:40.046Z" }, - { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469, upload-time = "2024-09-04T20:44:41.616Z" }, - { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475, upload-time = "2024-09-04T20:44:43.733Z" }, - { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, -] - -[[package]] -name = "cloudpickle" -version = "3.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/39/069100b84d7418bc358d81669d5748efb14b9cceacd2f9c75f550424132f/cloudpickle-3.1.1.tar.gz", hash = "sha256:b216fa8ae4019d5482a8ac3c95d8f6346115d8835911fd4aefd1a445e4242c64", size = 22113, upload-time = "2025-01-14T17:02:05.085Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/e8/64c37fadfc2816a7701fa8a6ed8d87327c7d54eacfbfb6edab14a2f2be75/cloudpickle-3.1.1-py3-none-any.whl", hash = "sha256:c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e", size = 20992, upload-time = "2025-01-14T17:02:02.417Z" }, -] - -[[package]] -name = "cryptography" -version = "44.0.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cd/25/4ce80c78963834b8a9fd1cc1266be5ed8d1840785c0f2e1b73b8d128d505/cryptography-44.0.2.tar.gz", hash = "sha256:c63454aa261a0cf0c5b4718349629793e9e634993538db841165b3df74f37ec0", size = 710807, upload-time = "2025-03-02T00:01:37.692Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/ef/83e632cfa801b221570c5f58c0369db6fa6cef7d9ff859feab1aae1a8a0f/cryptography-44.0.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:efcfe97d1b3c79e486554efddeb8f6f53a4cdd4cf6086642784fa31fc384e1d7", size = 6676361, upload-time = "2025-03-02T00:00:06.528Z" }, - { url = "https://files.pythonhosted.org/packages/30/ec/7ea7c1e4c8fc8329506b46c6c4a52e2f20318425d48e0fe597977c71dbce/cryptography-44.0.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29ecec49f3ba3f3849362854b7253a9f59799e3763b0c9d0826259a88efa02f1", size = 3952350, upload-time = "2025-03-02T00:00:09.537Z" }, - { url = "https://files.pythonhosted.org/packages/27/61/72e3afdb3c5ac510330feba4fc1faa0fe62e070592d6ad00c40bb69165e5/cryptography-44.0.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc821e161ae88bfe8088d11bb39caf2916562e0a2dc7b6d56714a48b784ef0bb", size = 4166572, upload-time = "2025-03-02T00:00:12.03Z" }, - { url = "https://files.pythonhosted.org/packages/26/e4/ba680f0b35ed4a07d87f9e98f3ebccb05091f3bf6b5a478b943253b3bbd5/cryptography-44.0.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3c00b6b757b32ce0f62c574b78b939afab9eecaf597c4d624caca4f9e71e7843", size = 3958124, upload-time = "2025-03-02T00:00:14.518Z" }, - { url = "https://files.pythonhosted.org/packages/9c/e8/44ae3e68c8b6d1cbc59040288056df2ad7f7f03bbcaca6b503c737ab8e73/cryptography-44.0.2-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7bdcd82189759aba3816d1f729ce42ffded1ac304c151d0a8e89b9996ab863d5", size = 3678122, upload-time = "2025-03-02T00:00:17.212Z" }, - { url = "https://files.pythonhosted.org/packages/27/7b/664ea5e0d1eab511a10e480baf1c5d3e681c7d91718f60e149cec09edf01/cryptography-44.0.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4973da6ca3db4405c54cd0b26d328be54c7747e89e284fcff166132eb7bccc9c", size = 4191831, upload-time = "2025-03-02T00:00:19.696Z" }, - { url = "https://files.pythonhosted.org/packages/2a/07/79554a9c40eb11345e1861f46f845fa71c9e25bf66d132e123d9feb8e7f9/cryptography-44.0.2-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:4e389622b6927d8133f314949a9812972711a111d577a5d1f4bee5e58736b80a", size = 3960583, upload-time = "2025-03-02T00:00:22.488Z" }, - { url = "https://files.pythonhosted.org/packages/bb/6d/858e356a49a4f0b591bd6789d821427de18432212e137290b6d8a817e9bf/cryptography-44.0.2-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:f514ef4cd14bb6fb484b4a60203e912cfcb64f2ab139e88c2274511514bf7308", size = 4191753, upload-time = "2025-03-02T00:00:25.038Z" }, - { url = "https://files.pythonhosted.org/packages/b2/80/62df41ba4916067fa6b125aa8c14d7e9181773f0d5d0bd4dcef580d8b7c6/cryptography-44.0.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1bc312dfb7a6e5d66082c87c34c8a62176e684b6fe3d90fcfe1568de675e6688", size = 4079550, upload-time = "2025-03-02T00:00:26.929Z" }, - { url = "https://files.pythonhosted.org/packages/f3/cd/2558cc08f7b1bb40683f99ff4327f8dcfc7de3affc669e9065e14824511b/cryptography-44.0.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b721b8b4d948b218c88cb8c45a01793483821e709afe5f622861fc6182b20a7", size = 4298367, upload-time = "2025-03-02T00:00:28.735Z" }, - { url = "https://files.pythonhosted.org/packages/71/59/94ccc74788945bc3bd4cf355d19867e8057ff5fdbcac781b1ff95b700fb1/cryptography-44.0.2-cp37-abi3-win32.whl", hash = "sha256:51e4de3af4ec3899d6d178a8c005226491c27c4ba84101bfb59c901e10ca9f79", size = 2772843, upload-time = "2025-03-02T00:00:30.592Z" }, - { url = "https://files.pythonhosted.org/packages/ca/2c/0d0bbaf61ba05acb32f0841853cfa33ebb7a9ab3d9ed8bb004bd39f2da6a/cryptography-44.0.2-cp37-abi3-win_amd64.whl", hash = "sha256:c505d61b6176aaf982c5717ce04e87da5abc9a36a5b39ac03905c4aafe8de7aa", size = 3209057, upload-time = "2025-03-02T00:00:33.393Z" }, - { url = "https://files.pythonhosted.org/packages/9e/be/7a26142e6d0f7683d8a382dd963745e65db895a79a280a30525ec92be890/cryptography-44.0.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:8e0ddd63e6bf1161800592c71ac794d3fb8001f2caebe0966e77c5234fa9efc3", size = 6677789, upload-time = "2025-03-02T00:00:36.009Z" }, - { url = "https://files.pythonhosted.org/packages/06/88/638865be7198a84a7713950b1db7343391c6066a20e614f8fa286eb178ed/cryptography-44.0.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81276f0ea79a208d961c433a947029e1a15948966658cf6710bbabb60fcc2639", size = 3951919, upload-time = "2025-03-02T00:00:38.581Z" }, - { url = "https://files.pythonhosted.org/packages/d7/fc/99fe639bcdf58561dfad1faa8a7369d1dc13f20acd78371bb97a01613585/cryptography-44.0.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a1e657c0f4ea2a23304ee3f964db058c9e9e635cc7019c4aa21c330755ef6fd", size = 4167812, upload-time = "2025-03-02T00:00:42.934Z" }, - { url = "https://files.pythonhosted.org/packages/53/7b/aafe60210ec93d5d7f552592a28192e51d3c6b6be449e7fd0a91399b5d07/cryptography-44.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6210c05941994290f3f7f175a4a57dbbb2afd9273657614c506d5976db061181", size = 3958571, upload-time = "2025-03-02T00:00:46.026Z" }, - { url = "https://files.pythonhosted.org/packages/16/32/051f7ce79ad5a6ef5e26a92b37f172ee2d6e1cce09931646eef8de1e9827/cryptography-44.0.2-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d1c3572526997b36f245a96a2b1713bf79ce99b271bbcf084beb6b9b075f29ea", size = 3679832, upload-time = "2025-03-02T00:00:48.647Z" }, - { url = "https://files.pythonhosted.org/packages/78/2b/999b2a1e1ba2206f2d3bca267d68f350beb2b048a41ea827e08ce7260098/cryptography-44.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b042d2a275c8cee83a4b7ae30c45a15e6a4baa65a179a0ec2d78ebb90e4f6699", size = 4193719, upload-time = "2025-03-02T00:00:51.397Z" }, - { url = "https://files.pythonhosted.org/packages/72/97/430e56e39a1356e8e8f10f723211a0e256e11895ef1a135f30d7d40f2540/cryptography-44.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:d03806036b4f89e3b13b6218fefea8d5312e450935b1a2d55f0524e2ed7c59d9", size = 3960852, upload-time = "2025-03-02T00:00:53.317Z" }, - { url = "https://files.pythonhosted.org/packages/89/33/c1cf182c152e1d262cac56850939530c05ca6c8d149aa0dcee490b417e99/cryptography-44.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c7362add18b416b69d58c910caa217f980c5ef39b23a38a0880dfd87bdf8cd23", size = 4193906, upload-time = "2025-03-02T00:00:56.49Z" }, - { url = "https://files.pythonhosted.org/packages/e1/99/87cf26d4f125380dc674233971069bc28d19b07f7755b29861570e513650/cryptography-44.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8cadc6e3b5a1f144a039ea08a0bdb03a2a92e19c46be3285123d32029f40a922", size = 4081572, upload-time = "2025-03-02T00:00:59.995Z" }, - { url = "https://files.pythonhosted.org/packages/b3/9f/6a3e0391957cc0c5f84aef9fbdd763035f2b52e998a53f99345e3ac69312/cryptography-44.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6f101b1f780f7fc613d040ca4bdf835c6ef3b00e9bd7125a4255ec574c7916e4", size = 4298631, upload-time = "2025-03-02T00:01:01.623Z" }, - { url = "https://files.pythonhosted.org/packages/e2/a5/5bc097adb4b6d22a24dea53c51f37e480aaec3465285c253098642696423/cryptography-44.0.2-cp39-abi3-win32.whl", hash = "sha256:3dc62975e31617badc19a906481deacdeb80b4bb454394b4098e3f2525a488c5", size = 2773792, upload-time = "2025-03-02T00:01:04.133Z" }, - { url = "https://files.pythonhosted.org/packages/33/cf/1f7649b8b9a3543e042d3f348e398a061923ac05b507f3f4d95f11938aa9/cryptography-44.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:5f6f90b72d8ccadb9c6e311c775c8305381db88374c65fa1a68250aa8a9cb3a6", size = 3210957, upload-time = "2025-03-02T00:01:06.987Z" }, - { url = "https://files.pythonhosted.org/packages/99/10/173be140714d2ebaea8b641ff801cbcb3ef23101a2981cbf08057876f89e/cryptography-44.0.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:af4ff3e388f2fa7bff9f7f2b31b87d5651c45731d3e8cfa0944be43dff5cfbdb", size = 3396886, upload-time = "2025-03-02T00:01:09.51Z" }, - { url = "https://files.pythonhosted.org/packages/2f/b4/424ea2d0fce08c24ede307cead3409ecbfc2f566725d4701b9754c0a1174/cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0529b1d5a0105dd3731fa65680b45ce49da4d8115ea76e9da77a875396727b41", size = 3892387, upload-time = "2025-03-02T00:01:11.348Z" }, - { url = "https://files.pythonhosted.org/packages/28/20/8eaa1a4f7c68a1cb15019dbaad59c812d4df4fac6fd5f7b0b9c5177f1edd/cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7ca25849404be2f8e4b3c59483d9d3c51298a22c1c61a0e84415104dacaf5562", size = 4109922, upload-time = "2025-03-02T00:01:13.934Z" }, - { url = "https://files.pythonhosted.org/packages/11/25/5ed9a17d532c32b3bc81cc294d21a36c772d053981c22bd678396bc4ae30/cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:268e4e9b177c76d569e8a145a6939eca9a5fec658c932348598818acf31ae9a5", size = 3895715, upload-time = "2025-03-02T00:01:16.895Z" }, - { url = "https://files.pythonhosted.org/packages/63/31/2aac03b19c6329b62c45ba4e091f9de0b8f687e1b0cd84f101401bece343/cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:9eb9d22b0a5d8fd9925a7764a054dca914000607dff201a24c791ff5c799e1fa", size = 4109876, upload-time = "2025-03-02T00:01:18.751Z" }, - { url = "https://files.pythonhosted.org/packages/99/ec/6e560908349843718db1a782673f36852952d52a55ab14e46c42c8a7690a/cryptography-44.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2bf7bf75f7df9715f810d1b038870309342bff3069c5bd8c6b96128cb158668d", size = 3131719, upload-time = "2025-03-02T00:01:21.269Z" }, - { url = "https://files.pythonhosted.org/packages/d6/d7/f30e75a6aa7d0f65031886fa4a1485c2fbfe25a1896953920f6a9cfe2d3b/cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:909c97ab43a9c0c0b0ada7a1281430e4e5ec0458e6d9244c0e821bbf152f061d", size = 3887513, upload-time = "2025-03-02T00:01:22.911Z" }, - { url = "https://files.pythonhosted.org/packages/9c/b4/7a494ce1032323ca9db9a3661894c66e0d7142ad2079a4249303402d8c71/cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:96e7a5e9d6e71f9f4fca8eebfd603f8e86c5225bb18eb621b2c1e50b290a9471", size = 4107432, upload-time = "2025-03-02T00:01:24.701Z" }, - { url = "https://files.pythonhosted.org/packages/45/f8/6b3ec0bc56123b344a8d2b3264a325646d2dcdbdd9848b5e6f3d37db90b3/cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d1b3031093a366ac767b3feb8bcddb596671b3aaff82d4050f984da0c248b615", size = 3891421, upload-time = "2025-03-02T00:01:26.335Z" }, - { url = "https://files.pythonhosted.org/packages/57/ff/f3b4b2d007c2a646b0f69440ab06224f9cf37a977a72cdb7b50632174e8a/cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:04abd71114848aa25edb28e225ab5f268096f44cf0127f3d36975bdf1bdf3390", size = 4107081, upload-time = "2025-03-02T00:01:28.938Z" }, -] - -[[package]] -name = "deprecation" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788, upload-time = "2020-04-20T14:23:38.738Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178, upload-time = "2020-04-20T14:23:36.581Z" }, -] - -[[package]] -name = "grpcio" -version = "1.62.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/2e/1e2cd0edeaaeaae0ab9df2615492725d0f2f689d3f9aa3b088356af7a584/grpcio-1.62.3.tar.gz", hash = "sha256:4439bbd759636e37b66841117a66444b454937e27f0125205d2d117d7827c643", size = 26297933, upload-time = "2024-08-06T00:32:51.086Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/86/f1aa615ee551e8b4f59b0d1189a09e16eefb3d243487115ab7be56eecbec/grpcio-1.62.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:13571a5b868dcc308a55d36669a2d17d9dcd6ec8335213f6c49cc68da7305abe", size = 4766342, upload-time = "2024-08-06T00:20:32.775Z" }, - { url = "https://files.pythonhosted.org/packages/c5/63/ee244c4b64f0e71cef5314f9fa1d120c072e33c2e4c545dc75bd1af2a5c5/grpcio-1.62.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:f5def814c5a4c90c8fe389c526ab881f4a28b7e239b23ed8e02dd02934dfaa1a", size = 9991627, upload-time = "2024-08-06T00:20:35.662Z" }, - { url = "https://files.pythonhosted.org/packages/70/69/23bd58a27c472221fc340dd08eee2becf1a2c9d27d00e279c78a6b6f53cc/grpcio-1.62.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:7349cd7445ac65fbe1b744dcab9cc1ec02dae2256941a2e67895926cbf7422b4", size = 5290817, upload-time = "2024-08-06T00:20:38.718Z" }, - { url = "https://files.pythonhosted.org/packages/74/3a/875be32d9d1516398049ebc39cc6e7620d50d807093ce624f0469cee5e51/grpcio-1.62.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:646c14e9f3356d3f34a65b58b0f8d08daa741ba1d4fcd4966b79407543332154", size = 5829374, upload-time = "2024-08-06T00:20:41.631Z" }, - { url = "https://files.pythonhosted.org/packages/58/2f/f3fc773270cf17e7ca076c1f6435278f58641d475a25cdeea5b2d8d4845b/grpcio-1.62.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:807176971c504c598976f5a9ea62363cffbbbb6c7509d9808c2342b020880fa2", size = 5549649, upload-time = "2024-08-06T00:20:44.562Z" }, - { url = "https://files.pythonhosted.org/packages/b4/dc/deb8b1da1fa6111c3f44253433faf977678dea7dd381ce397ee33a1b4d8c/grpcio-1.62.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:43670a25b752b7ed960fcec3db50ae5886dc0df897269b3f5119cde9b731745f", size = 6113730, upload-time = "2024-08-06T00:20:47.107Z" }, - { url = "https://files.pythonhosted.org/packages/9d/e4/d3556f073563cea4aabfa340b08f462e8a748c7190f34a3467442d72ac48/grpcio-1.62.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:668211f3699bbee4deaf1d6e6b8df59328bf63f077bf2dc9b8bfa4a17df4a279", size = 5779201, upload-time = "2024-08-06T00:20:50.606Z" }, - { url = "https://files.pythonhosted.org/packages/15/c3/bf758db22525e1e3cd541f9bbfd33b248cf6866678a1285127cf5b6ec6a0/grpcio-1.62.3-cp310-cp310-win32.whl", hash = "sha256:216740723fc5971429550c374a0c039723b9d4dcaf7ba05227b7e0a500b06417", size = 3170437, upload-time = "2024-08-06T00:20:53.245Z" }, - { url = "https://files.pythonhosted.org/packages/d4/2f/1a4710440cc9e94a8d38af6dce0e670803a029ebc0f904929079a1c7ba58/grpcio-1.62.3-cp310-cp310-win_amd64.whl", hash = "sha256:b708401ede2c4cb8943e8a713988fcfe6cbea105b07cd7fa7c8a9f137c22bddb", size = 3730887, upload-time = "2024-08-06T00:20:56.018Z" }, - { url = "https://files.pythonhosted.org/packages/f5/3a/c3da1df7d55cfe481b02221d8e22e603f43fdf1646f2c02e7d69370d5e4b/grpcio-1.62.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:c8bb1a7aa82af6c7713cdf9dcb8f4ea1024ac7ce82bb0a0a82a49aea5237da34", size = 4774831, upload-time = "2024-08-06T00:20:58.985Z" }, - { url = "https://files.pythonhosted.org/packages/c8/12/c769a65437081cce5c7aece3fcc0a0e9e5293d85455cbdc9dd4edc9c56b9/grpcio-1.62.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:57823dc7299c4f258ae9c32fd327d29f729d359c34d7612b36e48ed45b3ab8d0", size = 10019810, upload-time = "2024-08-06T00:21:02.31Z" }, - { url = "https://files.pythonhosted.org/packages/18/08/b2a2c66f183240e55ca99a0dd85c2c2ad1cb0846e7ad628900843a49a155/grpcio-1.62.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:1de3d04d9a4ec31ebe848ae1fe61e4cbc367fb9495cbf6c54368e60609a998d9", size = 5294405, upload-time = "2024-08-06T00:21:05.586Z" }, - { url = "https://files.pythonhosted.org/packages/fe/ac/6e523ccaf068dc022de3cc798f539bd070fd45e4db953241cff23fd867a6/grpcio-1.62.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:325c56ce94d738c31059cf91376f625d3effdff8f85c96660a5fd6395d5a707f", size = 5830738, upload-time = "2024-08-06T00:21:08.512Z" }, - { url = "https://files.pythonhosted.org/packages/78/a0/3a1c81854f76d8c1462533e18cc754b9d3e434234678cb2273b1bff5885c/grpcio-1.62.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c175b252d063af388523a397dbe8edbc4319761f5ee892a8a0f5890acc067362", size = 5547176, upload-time = "2024-08-06T00:21:11.374Z" }, - { url = "https://files.pythonhosted.org/packages/7b/1a/9462e3c81429c4b299a7222df8cd3c1f84625eecc353967d5ddfec43f8f2/grpcio-1.62.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:25cd75dc73c5269932413e517be778640402f18cf9a81147e68645bd8af18ab0", size = 6116809, upload-time = "2024-08-06T00:21:13.99Z" }, - { url = "https://files.pythonhosted.org/packages/40/6c/99f922adeb6ca65b6f84f6bf1032f5b94c042eef65ab9b13d438819e9205/grpcio-1.62.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1b85d35a7d9638c03321dfe466645b87e23c30df1266f9e04bbb5f44e7579a9", size = 5779755, upload-time = "2024-08-06T00:21:17.029Z" }, - { url = "https://files.pythonhosted.org/packages/ae/b7/d48ff1a5f6ad59748df7717a3f101679e0e1b9004f5b6efa96831c2b847c/grpcio-1.62.3-cp311-cp311-win32.whl", hash = "sha256:6be243f3954b0ca709f56f9cae926c84ac96e1cce19844711e647a1f1db88b99", size = 3167821, upload-time = "2024-08-06T00:21:20.522Z" }, - { url = "https://files.pythonhosted.org/packages/9f/90/32ba95836adb03dd04a393f1b9a8bde7919e9f08ee5fd94dc77351f9305f/grpcio-1.62.3-cp311-cp311-win_amd64.whl", hash = "sha256:e9ffdb7bc9ccd56ec201aec3eab3432e1e820335b5a16ad2b37e094218dcd7a6", size = 3729044, upload-time = "2024-08-06T00:21:23.542Z" }, - { url = "https://files.pythonhosted.org/packages/33/3f/23748407c2fd739e983c366b805aeb86ed57c718f2619aa3a5856594ed67/grpcio-1.62.3-cp312-cp312-linux_armv7l.whl", hash = "sha256:4c9c1502c76cadbf2e145061b63af077b08d5677afcef91970d6db87b30e2f8b", size = 4733041, upload-time = "2024-08-06T00:21:26.788Z" }, - { url = "https://files.pythonhosted.org/packages/de/27/0f85db1ad84569c8c2f82c0d473b84dd09f8fe5e053298b1f35935b92d62/grpcio-1.62.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:abfe64811177e681edc81d9d9d1bd23edc5f599bd9846650864769264ace30cd", size = 9978073, upload-time = "2024-08-06T00:21:29.381Z" }, - { url = "https://files.pythonhosted.org/packages/4d/d1/df712e7f5cdd2676fde7a3459783f18dd8b6b8c6a201774551d431cfa50c/grpcio-1.62.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:3737e5ef0aa0fcdfeaf3b4ecc1a6be78b494549b28aec4b7f61b5dc357f7d8be", size = 5233910, upload-time = "2024-08-06T00:21:32.391Z" }, - { url = "https://files.pythonhosted.org/packages/12/b1/9e28e35382a4f29def23f7cbf5414a667d2249ce83eaf7024d31f88b0399/grpcio-1.62.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:940459d81685549afdfe13a6de102c52ea4cdda093477baa53056884aadf7c48", size = 5766972, upload-time = "2024-08-06T00:21:35.743Z" }, - { url = "https://files.pythonhosted.org/packages/82/2e/43218874d1852af1ea9801a2be62cc596ddd45984e7adba0fb9f66393c81/grpcio-1.62.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac9783d5679c8da612465168c820fd0b916e70ec5496c840bddba0be7f2d124c", size = 5492246, upload-time = "2024-08-06T00:21:38.978Z" }, - { url = "https://files.pythonhosted.org/packages/7a/59/8d83b5b52cf9a655633e36e7953899901fc93aefd15d3e1ff8129a7ef30e/grpcio-1.62.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:c95a0b76a44c548e6bd8c5f7dbecf89c77e2e16d3965be817b57769c4a30bea2", size = 6062547, upload-time = "2024-08-06T00:21:44.445Z" }, - { url = "https://files.pythonhosted.org/packages/99/8c/cf726cbee9a3e636adecc94a55136c72da8c36422c8c0173e0e3be535665/grpcio-1.62.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b097347441b86a8c3ad9579abaf5e5f7f82b1d74a898f47360433b2bca0e4536", size = 5729178, upload-time = "2024-08-06T00:21:48.757Z" }, - { url = "https://files.pythonhosted.org/packages/ea/a5/3a24d6d05da642e9d94902aa5c681ce9afd6f6af079d05a1d6d3aaa20cd6/grpcio-1.62.3-cp312-cp312-win32.whl", hash = "sha256:3fb7d966a976d762a31346353a19fce4afcffbeda3027dd563bc8cb521fcf799", size = 3156979, upload-time = "2024-08-06T00:21:51.846Z" }, - { url = "https://files.pythonhosted.org/packages/d7/9d/dc29922afbd0bb2616a14241508e6ee871b35f783a6b2e7104b44f82a2c6/grpcio-1.62.3-cp312-cp312-win_amd64.whl", hash = "sha256:454a6aed4ebd56198d37e1f3be6f1c70838e33dd62d1e2cea12f2bcb08efecc5", size = 3711573, upload-time = "2024-08-06T00:21:55.032Z" }, -] - -[[package]] -name = "grpcio-tools" -version = "1.62.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "grpcio" }, - { name = "protobuf" }, - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/54/fa/b69bd8040eafc09b88bb0ec0fea59e8aacd1a801e688af087cead213b0d0/grpcio-tools-1.62.3.tar.gz", hash = "sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833", size = 4538520, upload-time = "2024-08-06T00:37:11.035Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/eb/eb0a3aa9480c3689d31fd2ad536df6a828e97a60f667c8a93d05bdf07150/grpcio_tools-1.62.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2f968b049c2849540751ec2100ab05e8086c24bead769ca734fdab58698408c1", size = 5117556, upload-time = "2024-08-06T00:30:32.892Z" }, - { url = "https://files.pythonhosted.org/packages/f3/fb/8be3dda485f7fab906bfa02db321c3ecef953a87cdb5f6572ca08b187bcb/grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:0a8c0c4724ae9c2181b7dbc9b186df46e4f62cb18dc184e46d06c0ebeccf569e", size = 2719330, upload-time = "2024-08-06T00:30:36.113Z" }, - { url = "https://files.pythonhosted.org/packages/63/de/6978f8d10066e240141cd63d1fbfc92818d96bb53427074f47a8eda921e1/grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5782883a27d3fae8c425b29a9d3dcf5f47d992848a1b76970da3b5a28d424b26", size = 3070818, upload-time = "2024-08-06T00:30:38.797Z" }, - { url = "https://files.pythonhosted.org/packages/74/34/bb8f816893fc73fd6d830e895e8638d65d13642bb7a434f9175c5ca7da11/grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3d812daffd0c2d2794756bd45a353f89e55dc8f91eb2fc840c51b9f6be62667", size = 2804993, upload-time = "2024-08-06T00:30:41.72Z" }, - { url = "https://files.pythonhosted.org/packages/78/60/b2198d7db83293cdb9760fc083f077c73e4c182da06433b3b157a1567d06/grpcio_tools-1.62.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b47d0dda1bdb0a0ba7a9a6de88e5a1ed61f07fad613964879954961e36d49193", size = 3684915, upload-time = "2024-08-06T00:30:43.996Z" }, - { url = "https://files.pythonhosted.org/packages/61/20/56dbdc4ecb14d42a03cd164ff45e6e84572bbe61ee59c50c39f4d556a8d5/grpcio_tools-1.62.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ca246dffeca0498be9b4e1ee169b62e64694b0f92e6d0be2573e65522f39eea9", size = 3297482, upload-time = "2024-08-06T00:30:46.451Z" }, - { url = "https://files.pythonhosted.org/packages/4a/dc/e417a313c905744ce8cedf1e1edd81c41dc45ff400ae1c45080e18f26712/grpcio_tools-1.62.3-cp310-cp310-win32.whl", hash = "sha256:6a56d344b0bab30bf342a67e33d386b0b3c4e65868ffe93c341c51e1a8853ca5", size = 909793, upload-time = "2024-08-06T00:30:49.465Z" }, - { url = "https://files.pythonhosted.org/packages/d9/69/75e7ebfd8d755d3e7be5c6d1aa6d13220f5bba3a98965e4b50c329046777/grpcio_tools-1.62.3-cp310-cp310-win_amd64.whl", hash = "sha256:710fecf6a171dcbfa263a0a3e7070e0df65ba73158d4c539cec50978f11dad5d", size = 1052459, upload-time = "2024-08-06T00:30:51.98Z" }, - { url = "https://files.pythonhosted.org/packages/23/52/2dfe0a46b63f5ebcd976570aa5fc62f793d5a8b169e211c6a5aede72b7ae/grpcio_tools-1.62.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:703f46e0012af83a36082b5f30341113474ed0d91e36640da713355cd0ea5d23", size = 5147623, upload-time = "2024-08-06T00:30:54.894Z" }, - { url = "https://files.pythonhosted.org/packages/f0/2e/29fdc6c034e058482e054b4a3c2432f84ff2e2765c1342d4f0aa8a5c5b9a/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:7cc83023acd8bc72cf74c2edbe85b52098501d5b74d8377bfa06f3e929803492", size = 2719538, upload-time = "2024-08-06T00:30:57.928Z" }, - { url = "https://files.pythonhosted.org/packages/f9/60/abe5deba32d9ec2c76cdf1a2f34e404c50787074a2fee6169568986273f1/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ff7d58a45b75df67d25f8f144936a3e44aabd91afec833ee06826bd02b7fbe7", size = 3070964, upload-time = "2024-08-06T00:31:00.267Z" }, - { url = "https://files.pythonhosted.org/packages/bc/ad/e2b066684c75f8d9a48508cde080a3a36618064b9cadac16d019ca511444/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f2483ea232bd72d98a6dc6d7aefd97e5bc80b15cd909b9e356d6f3e326b6e43", size = 2805003, upload-time = "2024-08-06T00:31:02.565Z" }, - { url = "https://files.pythonhosted.org/packages/9c/3f/59bf7af786eae3f9d24ee05ce75318b87f541d0950190ecb5ffb776a1a58/grpcio_tools-1.62.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:962c84b4da0f3b14b3cdb10bc3837ebc5f136b67d919aea8d7bb3fd3df39528a", size = 3685154, upload-time = "2024-08-06T00:31:05.339Z" }, - { url = "https://files.pythonhosted.org/packages/f1/79/4dd62478b91e27084c67b35a2316ce8a967bd8b6cb8d6ed6c86c3a0df7cb/grpcio_tools-1.62.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8ad0473af5544f89fc5a1ece8676dd03bdf160fb3230f967e05d0f4bf89620e3", size = 3297942, upload-time = "2024-08-06T00:31:08.456Z" }, - { url = "https://files.pythonhosted.org/packages/b8/cb/86449ecc58bea056b52c0b891f26977afc8c4464d88c738f9648da941a75/grpcio_tools-1.62.3-cp311-cp311-win32.whl", hash = "sha256:db3bc9fa39afc5e4e2767da4459df82b095ef0cab2f257707be06c44a1c2c3e5", size = 910231, upload-time = "2024-08-06T00:31:11.464Z" }, - { url = "https://files.pythonhosted.org/packages/45/a4/9736215e3945c30ab6843280b0c6e1bff502910156ea2414cd77fbf1738c/grpcio_tools-1.62.3-cp311-cp311-win_amd64.whl", hash = "sha256:e0898d412a434e768a0c7e365acabe13ff1558b767e400936e26b5b6ed1ee51f", size = 1052496, upload-time = "2024-08-06T00:31:13.665Z" }, - { url = "https://files.pythonhosted.org/packages/2a/a5/d6887eba415ce318ae5005e8dfac3fa74892400b54b6d37b79e8b4f14f5e/grpcio_tools-1.62.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:d102b9b21c4e1e40af9a2ab3c6d41afba6bd29c0aa50ca013bf85c99cdc44ac5", size = 5147690, upload-time = "2024-08-06T00:31:16.436Z" }, - { url = "https://files.pythonhosted.org/packages/8a/7c/3cde447a045e83ceb4b570af8afe67ffc86896a2fe7f59594dc8e5d0a645/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:0a52cc9444df978438b8d2332c0ca99000521895229934a59f94f37ed896b133", size = 2720538, upload-time = "2024-08-06T00:31:18.905Z" }, - { url = "https://files.pythonhosted.org/packages/88/07/f83f2750d44ac4f06c07c37395b9c1383ef5c994745f73c6bfaf767f0944/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141d028bf5762d4a97f981c501da873589df3f7e02f4c1260e1921e565b376fa", size = 3071571, upload-time = "2024-08-06T00:31:21.684Z" }, - { url = "https://files.pythonhosted.org/packages/37/74/40175897deb61e54aca716bc2e8919155b48f33aafec8043dda9592d8768/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47a5c093ab256dec5714a7a345f8cc89315cb57c298b276fa244f37a0ba507f0", size = 2806207, upload-time = "2024-08-06T00:31:24.208Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ee/d8de915105a217cbcb9084d684abdc032030dcd887277f2ef167372287fe/grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d", size = 3685815, upload-time = "2024-08-06T00:31:26.917Z" }, - { url = "https://files.pythonhosted.org/packages/fd/d9/4360a6c12be3d7521b0b8c39e5d3801d622fbb81cc2721dbd3eee31e28c8/grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e02d7c1a02e3814c94ba0cfe43d93e872c758bd8fd5c2797f894d0c49b4a1dfc", size = 3298378, upload-time = "2024-08-06T00:31:30.401Z" }, - { url = "https://files.pythonhosted.org/packages/29/3b/7cdf4a9e5a3e0a35a528b48b111355cd14da601413a4f887aa99b6da468f/grpcio_tools-1.62.3-cp312-cp312-win32.whl", hash = "sha256:b881fd9505a84457e9f7e99362eeedd86497b659030cf57c6f0070df6d9c2b9b", size = 910416, upload-time = "2024-08-06T00:31:33.118Z" }, - { url = "https://files.pythonhosted.org/packages/6c/66/dd3ec249e44c1cc15e902e783747819ed41ead1336fcba72bf841f72c6e9/grpcio_tools-1.62.3-cp312-cp312-win_amd64.whl", hash = "sha256:11c625eebefd1fd40a228fc8bae385e448c7e32a6ae134e43cf13bbc23f902b7", size = 1052856, upload-time = "2024-08-06T00:31:36.519Z" }, -] - -[[package]] -name = "packaging" -version = "25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, -] - -[[package]] -name = "protobuf" -version = "4.25.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/48/d5/cccc7e82bbda9909ced3e7a441a24205ea07fea4ce23a772743c0c7611fa/protobuf-4.25.6.tar.gz", hash = "sha256:f8cfbae7c5afd0d0eaccbe73267339bff605a2315860bb1ba08eb66670a9a91f", size = 380631, upload-time = "2025-01-24T20:53:09.498Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/41/0ff3559d9a0fbdb37c9452f2b84e61f7784d8d7b9850182c7ef493f523ee/protobuf-4.25.6-cp310-abi3-win32.whl", hash = "sha256:61df6b5786e2b49fc0055f636c1e8f0aff263808bb724b95b164685ac1bcc13a", size = 392454, upload-time = "2025-01-24T20:52:51.08Z" }, - { url = "https://files.pythonhosted.org/packages/79/84/c700d6c3f3be770495b08a1c035e330497a31420e4a39a24c22c02cefc6c/protobuf-4.25.6-cp310-abi3-win_amd64.whl", hash = "sha256:b8f837bfb77513fe0e2f263250f423217a173b6d85135be4d81e96a4653bcd3c", size = 413443, upload-time = "2025-01-24T20:52:54.523Z" }, - { url = "https://files.pythonhosted.org/packages/b7/03/361e87cc824452376c2abcef0eabd18da78a7439479ec6541cf29076a4dc/protobuf-4.25.6-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:6d4381f2417606d7e01750e2729fe6fbcda3f9883aa0c32b51d23012bded6c91", size = 394246, upload-time = "2025-01-24T20:52:56.694Z" }, - { url = "https://files.pythonhosted.org/packages/64/d5/7dbeb69b74fa88f297c6d8f11b7c9cef0c2e2fb1fdf155c2ca5775cfa998/protobuf-4.25.6-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:5dd800da412ba7f6f26d2c08868a5023ce624e1fdb28bccca2dc957191e81fb5", size = 293714, upload-time = "2025-01-24T20:52:57.992Z" }, - { url = "https://files.pythonhosted.org/packages/d4/f0/6d5c100f6b18d973e86646aa5fc09bc12ee88a28684a56fd95511bceee68/protobuf-4.25.6-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:4434ff8bb5576f9e0c78f47c41cdf3a152c0b44de475784cd3fd170aef16205a", size = 294634, upload-time = "2025-01-24T20:52:59.671Z" }, - { url = "https://files.pythonhosted.org/packages/71/eb/be11a1244d0e58ee04c17a1f939b100199063e26ecca8262c04827fe0bf5/protobuf-4.25.6-py3-none-any.whl", hash = "sha256:07972021c8e30b870cfc0863409d033af940213e0e7f64e27fe017b929d2c9f7", size = 156466, upload-time = "2025-01-24T20:53:08.287Z" }, -] - -[[package]] -name = "pycparser" -version = "2.22" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload-time = "2024-03-30T13:22:22.564Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, -] - -[[package]] -name = "pymonik" -source = { editable = "." } -dependencies = [ - { name = "armonik" }, - { name = "cloudpickle" }, - { name = "grpcio" }, - { name = "pyyaml" }, -] - -[package.dev-dependencies] -dev = [ - { name = "ruff" }, -] - -[package.metadata] -requires-dist = [ - { name = "armonik", specifier = ">=3.25.0" }, - { name = "cloudpickle", specifier = ">=3.1.1" }, - { name = "grpcio" }, - { name = "pyyaml", specifier = ">=6.0.2" }, -] - -[package.metadata.requires-dev] -dev = [{ name = "ruff", specifier = ">=0.11.6" }] - -[[package]] -name = "pyyaml" -version = "6.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" }, - { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" }, - { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" }, - { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" }, - { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" }, - { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" }, - { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" }, - { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" }, - { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" }, - { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" }, - { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" }, - { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" }, - { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" }, - { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" }, - { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" }, - { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" }, - { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" }, - { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, - { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, - { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, - { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" }, - { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" }, - { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" }, - { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" }, - { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" }, - { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, - { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" }, - { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" }, - { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" }, - { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" }, - { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" }, - { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" }, - { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" }, - { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" }, - { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, -] - -[[package]] -name = "ruff" -version = "0.11.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d9/11/bcef6784c7e5d200b8a1f5c2ddf53e5da0efec37e6e5a44d163fb97e04ba/ruff-0.11.6.tar.gz", hash = "sha256:bec8bcc3ac228a45ccc811e45f7eb61b950dbf4cf31a67fa89352574b01c7d79", size = 4010053, upload-time = "2025-04-17T13:35:53.905Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/1f/8848b625100ebcc8740c8bac5b5dd8ba97dd4ee210970e98832092c1635b/ruff-0.11.6-py3-none-linux_armv6l.whl", hash = "sha256:d84dcbe74cf9356d1bdb4a78cf74fd47c740bf7bdeb7529068f69b08272239a1", size = 10248105, upload-time = "2025-04-17T13:35:14.758Z" }, - { url = "https://files.pythonhosted.org/packages/e0/47/c44036e70c6cc11e6ee24399c2a1e1f1e99be5152bd7dff0190e4b325b76/ruff-0.11.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9bc583628e1096148011a5d51ff3c836f51899e61112e03e5f2b1573a9b726de", size = 11001494, upload-time = "2025-04-17T13:35:18.444Z" }, - { url = "https://files.pythonhosted.org/packages/ed/5b/170444061650202d84d316e8f112de02d092bff71fafe060d3542f5bc5df/ruff-0.11.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f2959049faeb5ba5e3b378709e9d1bf0cab06528b306b9dd6ebd2a312127964a", size = 10352151, upload-time = "2025-04-17T13:35:20.563Z" }, - { url = "https://files.pythonhosted.org/packages/ff/91/f02839fb3787c678e112c8865f2c3e87cfe1744dcc96ff9fc56cfb97dda2/ruff-0.11.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63c5d4e30d9d0de7fedbfb3e9e20d134b73a30c1e74b596f40f0629d5c28a193", size = 10541951, upload-time = "2025-04-17T13:35:22.522Z" }, - { url = "https://files.pythonhosted.org/packages/9e/f3/c09933306096ff7a08abede3cc2534d6fcf5529ccd26504c16bf363989b5/ruff-0.11.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:26a4b9a4e1439f7d0a091c6763a100cef8fbdc10d68593df6f3cfa5abdd9246e", size = 10079195, upload-time = "2025-04-17T13:35:24.485Z" }, - { url = "https://files.pythonhosted.org/packages/e0/0d/a87f8933fccbc0d8c653cfbf44bedda69c9582ba09210a309c066794e2ee/ruff-0.11.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b5edf270223dd622218256569636dc3e708c2cb989242262fe378609eccf1308", size = 11698918, upload-time = "2025-04-17T13:35:26.504Z" }, - { url = "https://files.pythonhosted.org/packages/52/7d/8eac0bd083ea8a0b55b7e4628428203441ca68cd55e0b67c135a4bc6e309/ruff-0.11.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:f55844e818206a9dd31ff27f91385afb538067e2dc0beb05f82c293ab84f7d55", size = 12319426, upload-time = "2025-04-17T13:35:28.452Z" }, - { url = "https://files.pythonhosted.org/packages/c2/dc/d0c17d875662d0c86fadcf4ca014ab2001f867621b793d5d7eef01b9dcce/ruff-0.11.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d8f782286c5ff562e4e00344f954b9320026d8e3fae2ba9e6948443fafd9ffc", size = 11791012, upload-time = "2025-04-17T13:35:30.455Z" }, - { url = "https://files.pythonhosted.org/packages/f9/f3/81a1aea17f1065449a72509fc7ccc3659cf93148b136ff2a8291c4bc3ef1/ruff-0.11.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:01c63ba219514271cee955cd0adc26a4083df1956d57847978383b0e50ffd7d2", size = 13949947, upload-time = "2025-04-17T13:35:33.133Z" }, - { url = "https://files.pythonhosted.org/packages/61/9f/a3e34de425a668284e7024ee6fd41f452f6fa9d817f1f3495b46e5e3a407/ruff-0.11.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15adac20ef2ca296dd3d8e2bedc6202ea6de81c091a74661c3666e5c4c223ff6", size = 11471753, upload-time = "2025-04-17T13:35:35.416Z" }, - { url = "https://files.pythonhosted.org/packages/df/c5/4a57a86d12542c0f6e2744f262257b2aa5a3783098ec14e40f3e4b3a354a/ruff-0.11.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4dd6b09e98144ad7aec026f5588e493c65057d1b387dd937d7787baa531d9bc2", size = 10417121, upload-time = "2025-04-17T13:35:38.224Z" }, - { url = "https://files.pythonhosted.org/packages/58/3f/a3b4346dff07ef5b862e2ba06d98fcbf71f66f04cf01d375e871382b5e4b/ruff-0.11.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:45b2e1d6c0eed89c248d024ea95074d0e09988d8e7b1dad8d3ab9a67017a5b03", size = 10073829, upload-time = "2025-04-17T13:35:40.255Z" }, - { url = "https://files.pythonhosted.org/packages/93/cc/7ed02e0b86a649216b845b3ac66ed55d8aa86f5898c5f1691797f408fcb9/ruff-0.11.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bd40de4115b2ec4850302f1a1d8067f42e70b4990b68838ccb9ccd9f110c5e8b", size = 11076108, upload-time = "2025-04-17T13:35:42.559Z" }, - { url = "https://files.pythonhosted.org/packages/39/5e/5b09840fef0eff1a6fa1dea6296c07d09c17cb6fb94ed5593aa591b50460/ruff-0.11.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:77cda2dfbac1ab73aef5e514c4cbfc4ec1fbef4b84a44c736cc26f61b3814cd9", size = 11512366, upload-time = "2025-04-17T13:35:45.702Z" }, - { url = "https://files.pythonhosted.org/packages/6f/4c/1cd5a84a412d3626335ae69f5f9de2bb554eea0faf46deb1f0cb48534042/ruff-0.11.6-py3-none-win32.whl", hash = "sha256:5151a871554be3036cd6e51d0ec6eef56334d74dfe1702de717a995ee3d5b287", size = 10485900, upload-time = "2025-04-17T13:35:47.695Z" }, - { url = "https://files.pythonhosted.org/packages/42/46/8997872bc44d43df986491c18d4418f1caff03bc47b7f381261d62c23442/ruff-0.11.6-py3-none-win_amd64.whl", hash = "sha256:cce85721d09c51f3b782c331b0abd07e9d7d5f775840379c640606d3159cae0e", size = 11558592, upload-time = "2025-04-17T13:35:49.837Z" }, - { url = "https://files.pythonhosted.org/packages/d7/6a/65fecd51a9ca19e1477c3879a7fda24f8904174d1275b419422ac00f6eee/ruff-0.11.6-py3-none-win_arm64.whl", hash = "sha256:3567ba0d07fb170b1b48d944715e3294b77f5b7679e8ba258199a250383ccb79", size = 10682766, upload-time = "2025-04-17T13:35:52.014Z" }, -] - -[[package]] -name = "setuptools" -version = "79.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7d/19/fecb7e2825616270f34512b3394cdcf6f45a79b5b6d94fdbd86a509e67b5/setuptools-79.0.0.tar.gz", hash = "sha256:9828422e7541213b0aacb6e10bbf9dd8febeaa45a48570e09b6d100e063fc9f9", size = 1367685, upload-time = "2025-04-20T15:47:56.706Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/ea/d53f2f8897c46a36df085964d07761ea4c2d1f2cf92019693b6742b7aabb/setuptools-79.0.0-py3-none-any.whl", hash = "sha256:b9ab3a104bedb292323f53797b00864e10e434a3ab3906813a7169e4745b912a", size = 1256065, upload-time = "2025-04-20T15:47:54.242Z" }, -] diff --git a/pymonik_worker/Dockerfile b/pymonik_worker/Dockerfile deleted file mode 100644 index fbe8116..0000000 --- a/pymonik_worker/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim - -ARG USE_PYTHON_VERSION="3.10.12" -# Set up working directory -WORKDIR /app - -# Set up the ArmoniK user (required by ArmoniK) -RUN groupadd --gid 5000 armonikuser && \ - useradd --home-dir /home/armonikuser --create-home --uid 5000 --gid 5000 --shell /bin/sh --skel /dev/null armonikuser && \ - mkdir /cache && \ - chown armonikuser: /cache && \ - chown -R armonikuser: /app - - -COPY --chown=armonikuser:armonikuser pymonik /pymonik - -USER armonikuser - - -RUN echo "Writing Python version: $USE_PYTHON_VERSION" && echo "$USE_PYTHON_VERSION" >> .python-version -RUN cat .python-version - -COPY pymonik_worker/pyproject.toml . - -RUN sed -i 's/source = "uv-dynamic-versioning"/source = "env"/' /pymonik/pyproject.toml -ENV PYMONIK_BUILD_VERSION="0.0.0" - -RUN uv sync - -# Copy application code -COPY pymonik_worker/worker.py . - -# activate uv venv -ENV PATH="/app/.venv/bin:$PATH" -# Set environment for Python unbuffered output -ENV PYTHONUNBUFFERED=1 - - -# Default command to start as a worker -ENTRYPOINT ["python", "worker.py"] diff --git a/pymonik_worker/README.md b/pymonik_worker/README.md deleted file mode 100644 index 1150654..0000000 --- a/pymonik_worker/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# PymoniK Worker - -This is the default PymoniK worker that ships with ArmoniK and that is provided in the `aneoconsulting/harmonik_snake` Docker image. You're encouraged to edit this worker image and adapt it for your own use case. For instance, you might want to include additional Python packages, programs or files by default, or you might want to use a specific Python version that we do not build or support. You can refer to our guide in the docs for creating custom workers. - -## Building your own PymoniK worker - -From the root of the project, run: -``` -uv run automation.py build-docker -``` - -You can append `--help` to get additional options (such as refreshing the worker image used in your armonik deployment, setting the tag, python version, etc.) diff --git a/pymonik_worker/pyproject.toml b/pymonik_worker/pyproject.toml deleted file mode 100644 index c4a27dc..0000000 --- a/pymonik_worker/pyproject.toml +++ /dev/null @@ -1,12 +0,0 @@ -[project] -name = "pymonik-worker" -version = "0.1.0" -description = "Worker for PymoniK" -readme = "README.md" -requires-python = ">=3.10.12" -dependencies = [ - "pymonik", -] - -[tool.uv.sources] -pymonik = { path = "../pymonik" } diff --git a/pymonik_worker/worker.py b/pymonik_worker/worker.py deleted file mode 100644 index 04cafc3..0000000 --- a/pymonik_worker/worker.py +++ /dev/null @@ -1,3 +0,0 @@ -from pymonik import run_pymonik_worker - -run_pymonik_worker() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..33f4ec2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,74 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pymonik" +version = "2.0.0a0" +description = "Greenfield v2 of the PymoniK SDK for ArmoniK — anyio-first, msgspec-wire, easy to start." +readme = "README.md" +requires-python = "==3.11.*" # must match the worker image python; cloudpickle is not cross-minor +dependencies = [ + "armonik>=3.25.0", + "anyio>=4.3", + "cloudpickle>=3.1", + "msgspec>=0.19", + "grpcio>=1.60", + "structlog>=24.1", + "pyyaml>=6.0.3", + "click>=8.1", +] + +[project.optional-dependencies] +otel = [ + "opentelemetry-api>=1.27", + "opentelemetry-sdk>=1.27", + "opentelemetry-exporter-otlp-proto-grpc>=1.27", +] + +[project.scripts] +pymonik-worker = "pymonik.worker:run" +pymonik = "pymonik.cli.main:cli" + +[tool.hatch.build.targets.wheel] +packages = ["src/pymonik"] + +[dependency-groups] +dev = [ + "ruff>=0.11", + "basedpyright>=1.18", + "pytest>=8", + # anyio ships its own pytest plugin; no separate pytest-anyio package. + "trio>=0.27", + "types-pyyaml>=6.0.12.20260408", + # examples/ scripts that exercise the runtime-deps path import these at + # module level so cloudpickle ships the function with proper module refs. + "numpy>=2", + "polars>=1", + "opentelemetry-api>=1.27.0", + "opentelemetry-sdk>=1.27.0", + "opentelemetry-exporter-otlp-proto-grpc>=1.27.0", +] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "SIM"] + +[tool.basedpyright] +pythonVersion = "3.11" +typeCheckingMode = "standard" + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = [ + "slow: tests that take more than a few seconds (e.g. real `uv` env builds)", +] + +# Python 3.13 cap rationale: +# `armonik` transitively pulls `grpcio-tools==1.62.3`, whose sdist uses the +# removed `pkg_resources` stdlib module and fails to build on 3.13. The cap +# lifts once upstream bumps grpcio-tools (or drops it entirely from runtime +# deps). Tracked in ../IMPLEMENTATION_PLAN.md §7. diff --git a/src/pymonik/__init__.py b/src/pymonik/__init__.py new file mode 100644 index 0000000..0e20895 --- /dev/null +++ b/src/pymonik/__init__.py @@ -0,0 +1,102 @@ +"""PymoniK — an easy-to-start SDK for ArmoniK. + +Quick start: + + from pymonik import PymonikClient, task + + @task + def add(a: int, b: int) -> int: + return a + b + + @task + def sum_all(xs: list[int]) -> int: + return sum(xs) + + with PymonikClient() as client: # reads AKCONFIG + with client.session(partition="pymonik") as s: + # Pipelining: pass futures as args. No client-side blocking — ArmoniK + # chains the tasks via data_dependencies. Only the terminal .result() + # actually waits. + parts = add.map(range(16), range(1, 17)) + total = sum_all.spawn(parts) + print(total.result(timeout=60)) +""" + +from pymonik import blob, testing +from pymonik._internal._logging import enable_logging, silence_logging +from pymonik._internal.info import ( + PartitionInfo, + ResultInfo, + SessionInfo, + TaskInfo, +) +from pymonik._internal.query import ( + PartitionQuery, + ResultQuery, + SessionQuery, + TaskQuery, +) +from pymonik.blob import Blob, Materialize +from pymonik.client import PymonikClient +from pymonik.composition import ( + as_completed, + as_completed_sync, + gather, + gather_sync, +) +from pymonik.context import WorkerContext, current +from pymonik.errors import ( + ConnectionError as PymonikConnectionError, + NotInSessionError, + PymonikError, + TaskCancelled, + TaskFailed, + TaskTimeout, +) +from pymonik.future import Future, FutureList, MultiResultHandle, MultiResultView +from pymonik.multiresult import MultiResult, TailPromise +from pymonik.options import TaskOpts +from pymonik.task import Task, task + +__all__ = [ + "PymonikClient", + "task", + "Task", + "TaskOpts", + "Future", + "FutureList", + "MultiResult", + "MultiResultHandle", + "MultiResultView", + "TailPromise", + "gather", + "gather_sync", + "as_completed", + "as_completed_sync", + "current", + "WorkerContext", + "blob", + "Blob", + "Materialize", + "testing", + "enable_logging", + "silence_logging", + # introspection + "TaskQuery", + "ResultQuery", + "SessionQuery", + "PartitionQuery", + "TaskInfo", + "ResultInfo", + "SessionInfo", + "PartitionInfo", + # errors + "PymonikError", + "TaskFailed", + "TaskCancelled", + "TaskTimeout", + "NotInSessionError", + "PymonikConnectionError", +] + +__version__ = "2.0.0a3" diff --git a/pymonik/README.md b/src/pymonik/_internal/__init__.py similarity index 100% rename from pymonik/README.md rename to src/pymonik/_internal/__init__.py diff --git a/src/pymonik/_internal/_ast_introspect.py b/src/pymonik/_internal/_ast_introspect.py new file mode 100644 index 0000000..4f71ec7 --- /dev/null +++ b/src/pymonik/_internal/_ast_introspect.py @@ -0,0 +1,157 @@ +"""AST introspection of ``@task``-decorated functions. + +Used to extract the multi-output field set from ``MultiResult(...)`` +calls in a function body, so the submission pipeline can pre-allocate +``expected_output_ids`` for each task. + +Limits: + +- Only top-level ``MultiResult(...)`` literals in the function body + are examined. Constructions in helpers, lambdas, or nested + comprehensions are invisible. +- Aliased imports (``from pymonik import MultiResult as MR``) are + resolved by walking the function's module-level imports. +- ``MultiResult(**dynamic)`` (kwargs expansion) raises a hard error. +- Inconsistent field sets across branches raise a hard error. + +When the AST walk can't see the source (lambdas, REPL definitions, +generated code), :func:`extract_multi_fields` returns ``None`` to +signal "single-output task." Users who need a multi-output task with +an opaque body can declare the schema via ``@task(outputs=("a", "b"))`` +on the decorator (see :mod:`pymonik.task`). +""" + +from __future__ import annotations + +import ast +import inspect +import textwrap +from typing import Callable + +from pymonik.errors import PymonikError + + +def _multiresult_aliases(func: Callable[..., object]) -> set[str]: + """Names that resolve to :class:`pymonik.MultiResult` in func's module. + + Always includes the bare name ``"MultiResult"`` (the user might have + a ``from pymonik import MultiResult`` even without an alias). Adds + any ``from pymonik import MultiResult as `` aliases found + among the module's top-level imports. + """ + aliases: set[str] = {"MultiResult"} + try: + module = inspect.getmodule(func) + if module is None: + return aliases + src = inspect.getsource(module) + except (OSError, TypeError): + return aliases + + try: + tree = ast.parse(src) + except SyntaxError: + return aliases + + for node in tree.body: + if isinstance(node, ast.ImportFrom) and node.module in {"pymonik", "pymonik.multiresult"}: + for alias in node.names: + if alias.name == "MultiResult": + aliases.add(alias.asname or alias.name) + return aliases + + +def _function_def(tree: ast.AST, name: str) -> ast.FunctionDef | ast.AsyncFunctionDef | None: + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == name: + return node + return None + + +def extract_multi_fields( + func: Callable[..., object], +) -> tuple[str, ...] | None: + """Return the sorted field set if ``func`` returns ``MultiResult``s. + + ``None`` if no ``MultiResult(...)`` literal is found (single-output + task). Raises :class:`PymonikError` on inconsistent shapes or + dynamic ``**kwargs`` expansion. + """ + try: + src = inspect.getsource(func) + except (OSError, TypeError): + return None + + src = textwrap.dedent(src) + try: + tree = ast.parse(src) + except SyntaxError: + return None + + func_def = _function_def(tree, getattr(func, "__name__", "")) + if func_def is None: + return None + + aliases = _multiresult_aliases(func) + + seen: list[tuple[int, frozenset[str]]] = [] + for node in ast.walk(func_def): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id in aliases + ): + continue + # Reject any **kwargs spread (kw.arg is None for **expr). + if any(kw.arg is None for kw in node.keywords): + raise PymonikError( + f"@task {func.__name__!r}: MultiResult with **kwargs expansion " + f"is not supported (line {node.lineno}). Construct " + f"MultiResult with literal keyword arguments so the field " + f"set can be extracted at decoration time, or pass " + f"outputs=(...) to the @task decorator." + ) + # Reject any positional args (MultiResult takes only kwargs). + if node.args: + raise PymonikError( + f"@task {func.__name__!r}: MultiResult takes only keyword " + f"arguments (line {node.lineno})." + ) + fields = frozenset(kw.arg for kw in node.keywords if kw.arg) + seen.append((node.lineno, fields)) + + if not seen: + return None + + distinct = {f for _, f in seen} + if len(distinct) > 1: + lines = "\n".join( + f" line {lineno}: MultiResult({', '.join(sorted(fields))})" + for lineno, fields in seen + ) + raise PymonikError( + f"@task {func.__name__!r}: inconsistent MultiResult shapes:\n{lines}\n" + f"Every return path must use the same field names." + ) + + fields = seen[0][1] + # Reject field names that would shadow MultiResultHandle attributes. + # The runtime check in MultiResult.__init__ catches dynamic + # constructions; this catches the static cases at decoration. + from pymonik.multiresult import MultiResult + + bad = fields & MultiResult._RESERVED_FIELD_NAMES + if bad: + raise PymonikError( + f"@task {func.__name__!r}: MultiResult field names " + f"{sorted(bad)} collide with MultiResultHandle attributes. " + f"Reserved names: {sorted(MultiResult._RESERVED_FIELD_NAMES)}." + ) + bad_underscore = {n for n in fields if n.startswith("_")} + if bad_underscore: + raise PymonikError( + f"@task {func.__name__!r}: MultiResult field names " + f"{sorted(bad_underscore)} are invalid: underscore-prefixed " + f"names are reserved." + ) + return tuple(sorted(fields)) diff --git a/src/pymonik/_internal/_logging.py b/src/pymonik/_internal/_logging.py new file mode 100644 index 0000000..26b023f --- /dev/null +++ b/src/pymonik/_internal/_logging.py @@ -0,0 +1,120 @@ +"""Library-quiet logging. + +PymoniK uses ``structlog``'s structured-kwargs API for readability, but +defers level / handler control to the standard library's ``logging`` +module. The result: + +- All pymonik log calls go through ``logging.getLogger("pymonik.<…>")``. +- That logger has a ``NullHandler`` attached at module load, so the + library is **silent by default** — the conventional behaviour. +- :func:`enable_logging` attaches a console handler with a coloured + renderer for users (or examples) that want to see what's happening. +- :func:`silence_logging` drops the handler again (idempotent). +- We *never* call ``structlog.configure(...)`` so the user's own + structlog config — if they have one — stays untouched. + +Modules in this package use ``get_logger(__name__)`` from this module +rather than importing ``structlog`` directly. The structlog kwargs API +still works (``log.info("msg", x=1, y=2)``); under the hood the +processor chain renders to a string and the stdlib logger handles +levels and handlers. +""" + +from __future__ import annotations + +import logging +import sys +from typing import Union + +import structlog + +LIB_NAME = "pymonik" + + +def _processor_chain(use_color: bool): + return [ + structlog.contextvars.merge_contextvars, + structlog.stdlib.add_log_level, + structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S"), + structlog.dev.ConsoleRenderer(colors=use_color), + ] + + +def get_logger(name: str = LIB_NAME): + """Return a bound logger backed by ``logging.getLogger("pymonik.")``. + + Pass ``__name__`` from a module; the leading package name is + canonicalised to ``pymonik.`` so the stdlib logger tree stays + flat under one parent. + """ + if name == LIB_NAME or not name: + full = LIB_NAME + else: + # __name__ is typically "pymonik.session" or "pymonik._internal.submit" + # — keep the trailing component, anchor under "pymonik". + short = name.rsplit(".", 1)[-1] + full = f"{LIB_NAME}.{short}" + return structlog.wrap_logger( + logging.getLogger(full), + processors=_processor_chain(use_color=_OPTS["color"]), + ) + + +def enable_logging( + level: Union[int, str] = logging.INFO, + *, + color: bool = True, + stream=None, +) -> None: + """Turn on pretty console logging for pymonik. + + Args: + level: stdlib logging level (``"INFO"`` / ``"DEBUG"`` / int). + color: use ANSI colours in the renderer (auto-disabled when not + attached to a TTY for piped output). + stream: where to write log records. Defaults to ``sys.stderr``. + + Idempotent: subsequent calls replace the previous handler so you + can flip the level without leaking handlers. + """ + if isinstance(level, str): + level = getattr(logging, level.upper()) + if stream is None: + stream = sys.stderr + if color and not getattr(stream, "isatty", lambda: False)(): + # Don't emit ANSI codes into a pipe/file. + color = False + + _OPTS["color"] = color + + pmk = logging.getLogger(LIB_NAME) + pmk.handlers.clear() + handler = logging.StreamHandler(stream=stream) + # The structlog wrapper already produces a fully-formatted line; just + # echo it. Adding any stdlib formatter would double-stamp the time. + handler.setFormatter(logging.Formatter("%(message)s")) + pmk.addHandler(handler) + pmk.setLevel(level) + pmk.propagate = False + + +def silence_logging() -> None: + """Drop any handler added by :func:`enable_logging`. Idempotent.""" + pmk = logging.getLogger(LIB_NAME) + pmk.handlers.clear() + pmk.addHandler(logging.NullHandler()) + # Reset to default (libraries shouldn't propagate by default either). + pmk.propagate = False + + +# Module-level state for the renderer's colour choice. Hardly ever +# changes; held here so :func:`get_logger` can read it and so future +# modules pick up a flipped colour mode. +_OPTS: dict = {"color": True} + + +# Library-default: silent. Convention is to attach a NullHandler so +# stdlib's "no handler found" warning doesn't fire if the user logs +# without configuring. +logging.getLogger(LIB_NAME).addHandler(logging.NullHandler()) +logging.getLogger(LIB_NAME).propagate = False diff --git a/src/pymonik/_internal/_otel.py b/src/pymonik/_internal/_otel.py new file mode 100644 index 0000000..d1d5c71 --- /dev/null +++ b/src/pymonik/_internal/_otel.py @@ -0,0 +1,267 @@ +"""Optional OpenTelemetry instrumentation. + +Off by default, no overhead. Two ways to enable: + +- ``PymonikClient(otel=True)`` — explicit. +- Standard OTel env vars (``OTEL_EXPORTER_OTLP_ENDPOINT`` / + ``OTEL_TRACES_EXPORTER`` / ...) detected on construction → auto-enable. + +When ``opentelemetry-api`` is not installed, every primitive here is a +no-op. Pymonik runs unchanged. Install with ``pip install pymonik[otel]`` +to pull the OTel API + SDK + OTLP exporter. + +Two integration points, both zero-cost when disabled: + +- :func:`start_span(name, attrs)` — context manager wrapping a code block. + Yields the span (or None if otel is disabled / not installed) so call + sites can ``set_attribute`` lazily. +- :func:`inject_context(carrier)` / :func:`use_extracted_context(carrier)` + — W3C Trace Context propagation. The submit pipeline injects the + *current* span's context onto the task envelope on the client; the + worker extracts and attaches it before running the user function. + +Resulting trace shape: + + pymonik.session.open + └── pymonik.submit (count=N, func=...) + ├── pymonik.task.run [worker pod] (task_id=..., attempt=1) + ├── pymonik.task.run [worker pod] + └── ... +""" + +from __future__ import annotations + +import os +from contextlib import contextmanager +from typing import Any, Generator, Mapping + +_AVAILABLE = False +_trace: Any = None +_otel_context: Any = None +_propagate: Any = None +SpanKind: Any = None +Status: Any = None +StatusCode: Any = None +Span: Any = Any + +try: + from opentelemetry import context as _otel_context # noqa: F811 + from opentelemetry import propagate as _propagate # noqa: F811 + from opentelemetry import trace as _trace # noqa: F811 + from opentelemetry.trace import ( # noqa: F811 + Span, + SpanKind, + Status, + StatusCode, + ) + + _AVAILABLE = True +except ImportError: # pragma: no cover — covered by the no-op tests + pass + + +_initialised = False +_enabled = False + + +def _is_enabled_via_env() -> bool: + if os.getenv("OTEL_SDK_DISABLED", "").lower() == "true": + return False + return any( + k in os.environ + for k in ( + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_TRACES_EXPORTER", + ) + ) + + +def setup(*, force: bool | None = None, service_name: str = "pymonik") -> bool: + """Initialise OTel. Idempotent across a process. + + ``force=None`` (default) auto-enables when standard OTel env vars are + set. ``True`` enables unconditionally; ``False`` keeps everything + no-op even if env vars are present. + + Returns the resulting enabled state. + """ + global _initialised, _enabled + if _initialised: + return _enabled + _initialised = True + + if not _AVAILABLE: + return False + if force is False: + return False + + enable = force is True or (force is None and _is_enabled_via_env()) + if not enable: + return False + + # If the user already configured a TracerProvider (e.g. their app uses + # OTel for other things), don't fight them. Just enable our spans. + current = _trace.get_tracer_provider() + if not _is_noop_provider(current): + _enabled = True + return True + + try: + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + except ImportError: + # API installed but SDK isn't — user wants their own setup. + _enabled = True + return True + + resource = Resource.create({"service.name": service_name}) + provider = TracerProvider(resource=resource) + + exporter = _build_default_exporter() + if exporter is not None: + provider.add_span_processor(BatchSpanProcessor(exporter)) + _trace.set_tracer_provider(provider) + _enabled = True + return True + + +def _is_noop_provider(provider: Any) -> bool: + name = type(provider).__name__ + return name in ("NoOpTracerProvider", "ProxyTracerProvider", "DefaultTracerProvider") + + +def _build_default_exporter(): + # Honour OTEL_TRACES_EXPORTER first (the standard env var). "console" + # is convenient for smoke tests when you don't have a collector yet. + requested = os.getenv("OTEL_TRACES_EXPORTER", "").lower() + if requested == "console": + try: + from opentelemetry.sdk.trace.export import ConsoleSpanExporter + + return ConsoleSpanExporter() + except ImportError: + return None + if requested in ("otlp", "otlp-grpc", ""): + try: + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, + ) + + return OTLPSpanExporter() + except ImportError: + pass + if requested in ("otlp-http", ""): + try: + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( # type: ignore[import-not-found] + OTLPSpanExporter, + ) + + return OTLPSpanExporter() + except ImportError: + pass + # Last resort: print spans to stdout so the user sees *something*. + try: + from opentelemetry.sdk.trace.export import ConsoleSpanExporter + + return ConsoleSpanExporter() + except ImportError: + return None + + +def is_enabled() -> bool: + return _enabled + + +def _tracer(): + if not _enabled or not _AVAILABLE: + return None + return _trace.get_tracer("pymonik") + + +def _kind_for(kind: str) -> Any: + if not _AVAILABLE: + return None + return { + "client": SpanKind.CLIENT, + "server": SpanKind.SERVER, + "internal": SpanKind.INTERNAL, + "producer": SpanKind.PRODUCER, + "consumer": SpanKind.CONSUMER, + }.get(kind, SpanKind.INTERNAL) + + +@contextmanager +def start_span( + name: str, + *, + attrs: Mapping[str, Any] | None = None, + kind: str = "internal", +) -> Generator[Any, None, None]: + """Start a span; yield the span object (or ``None`` if otel is off). + + Records exceptions as span events and marks status=ERROR before + re-raising, so failures show up in the trace UI without extra code + at the call site. + """ + tracer = _tracer() + if tracer is None: + yield None + return + + with tracer.start_as_current_span( + name, kind=_kind_for(kind), attributes=dict(attrs) if attrs else None + ) as span: + try: + yield span + except BaseException as e: + span.set_status(Status(StatusCode.ERROR, f"{type(e).__name__}: {e}")) + span.record_exception(e) + raise + + +def inject_context(carrier: dict[str, str]) -> None: + """Inject the current trace context into ``carrier`` (W3C headers). + + No-op when otel isn't enabled. Safe to call unconditionally — + pymonik's submit pipeline does on every batch. + """ + if not _enabled or not _AVAILABLE: + return + _propagate.inject(carrier) + + +@contextmanager +def use_extracted_context(carrier: Mapping[str, str]) -> Generator[None, None, None]: + """Attach the trace context found in ``carrier`` for the duration. + + Used by the worker: read ``traceparent`` / ``tracestate`` (or any + propagator's keys) off ``task_handler.task_options.options``, attach, + run the task. Spans created inside the block become children of the + client's submit span. + """ + if not _enabled or not _AVAILABLE: + yield + return + ctx = _propagate.extract(dict(carrier)) + token = _otel_context.attach(ctx) + try: + yield + finally: + _otel_context.detach(token) + + +def current_trace_id_hex() -> str | None: + """Hex trace id of the current span, or ``None`` if not in a trace. + + Useful for log correlation — the user can grep their UI for the id + we logged at submission time. + """ + if not _enabled or not _AVAILABLE: + return None + span = _trace.get_current_span() + ctx = span.get_span_context() + if not ctx.is_valid: + return None + return f"{ctx.trace_id:032x}" diff --git a/src/pymonik/_internal/channel.py b/src/pymonik/_internal/channel.py new file mode 100644 index 0000000..3ef4f20 --- /dev/null +++ b/src/pymonik/_internal/channel.py @@ -0,0 +1,45 @@ +"""gRPC channel construction. Insecure by default; TLS optional. + +Wraps the upstream armonik helper so callers get one place to configure auth. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +import grpc +from armonik.common import create_channel as _armonik_create_channel + + +@dataclass(frozen=True, slots=True) +class Credentials: + """mTLS credentials — all three fields required when any is given.""" + + ca: Optional[str] = None + cert: Optional[str] = None + key: Optional[str] = None + + @property + def tls(self) -> bool: + return any((self.ca, self.cert, self.key)) + + +def _strip_scheme(endpoint: str) -> str: + for scheme in ("https://", "http://", "grpcs://", "grpc://"): + if endpoint.startswith(scheme): + endpoint = endpoint[len(scheme) :] + return endpoint.rstrip("/") + + +def open_channel(endpoint: str, credentials: Optional[Credentials] = None) -> grpc.Channel: + """Open a sync gRPC channel. Callers are responsible for closing it.""" + endpoint = _strip_scheme(endpoint) + if credentials and credentials.tls: + return _armonik_create_channel( + endpoint, + certificate_authority=credentials.ca, + client_certificate=credentials.cert, + client_key=credentials.key, + ) + return grpc.insecure_channel(endpoint) diff --git a/src/pymonik/_internal/env_builder.py b/src/pymonik/_internal/env_builder.py new file mode 100644 index 0000000..99dec4e --- /dev/null +++ b/src/pymonik/_internal/env_builder.py @@ -0,0 +1,266 @@ +"""Worker-side runtime environment builder. + +Given an :class:`EnvSpec` from a task envelope, produce a venv at +``/envs//`` containing the requested deps. Concurrent +first-uses for the same ``env_id`` serialise via an OS flock so the +install runs once. Subsequent tasks reuse the venv with ~0 overhead. + +Identity rule: ``env_id = sha256(canonical(deps) | py_minor | pmk_ver)``. +Two clients submitting the same deps land in the same venv. The +canonicalisation lower-cases and sorts the deps strings — see +:func:`compute_env_id`. + +Eviction: not our job. ```` is whatever the worker is configured +to use (typically ``/cache/internal``); the polling agent evicts the +whole tree when the cache is full. +""" + +from __future__ import annotations + +import errno +import fcntl +import hashlib +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path +from typing import Iterable + +from pymonik._internal._logging import get_logger +from pymonik.envelope import EnvSpec +from pymonik.errors import PymonikError + +log = get_logger(__name__) + + +def _pymonik_version() -> str: + try: + from pymonik import __version__ + + return __version__ + except Exception: + return "unknown" + + +def _py_minor() -> str: + v = sys.version_info + return f"{v.major}.{v.minor}" + + +def canonical_deps(deps: Iterable[str]) -> tuple[str, ...]: + """Stable representation for hashing: strip, drop empties, lowercase, sort. + + Lowercasing matches PEP 503 normalisation for the *name* portion of a + requirement; specifiers (``>=``, version numbers) are case-insensitive + in practice. We don't try to parse PEP 508 here — two textually + different specifiers that resolve to the same set are different envs, + by design. The user controls the strings; we don't second-guess. + """ + cleaned = sorted({d.strip().lower() for d in deps if d.strip()}) + return tuple(cleaned) + + +def compute_env_id(spec: EnvSpec) -> str: + """Hash an EnvSpec into a stable id used as the venv directory name.""" + h = hashlib.sha256() + h.update(b"v=3|") + h.update(f"py={_py_minor()}|".encode()) + h.update(f"pmk={_pymonik_version()}|".encode()) + h.update(f"index={spec.index_url}|".encode()) + h.update(b"deps=") + for d in canonical_deps(spec.deps): + h.update(d.encode()) + h.update(b"\n") + h.update(b"env=") + # Env tuple is already sorted client-side; defensively re-sort here. + for k, v in sorted(spec.env): + h.update(k.encode()) + h.update(b"=") + h.update(v.encode()) + h.update(b"\n") + return h.hexdigest()[:32] + + +def default_envs_root() -> Path: + """Worker-side root for venvs. + + Honours ``PYMONIK_ENVS_ROOT`` for tests / dev. In production the + worker image sets it to ``/cache/internal``; outside the cluster + we fall back to ``~/.cache/pymonik/envs`` so the same code path + works under ``LocalCluster``. + """ + env = os.getenv("PYMONIK_ENVS_ROOT") + if env: + return Path(env) + if Path("/cache/internal").is_dir() and os.access("/cache/internal", os.W_OK): + return Path("/cache/internal/envs") + return Path.home() / ".cache" / "pymonik" / "envs" + + +def _uv_cache_dir(root: Path) -> Path: + """``UV_CACHE_DIR`` for wheel reuse across env builds. + + Same parent as ``envs/`` so ``/cache/internal`` covers both. Falls + back when ``PYMONIK_ENVS_ROOT`` is set to something exotic. + """ + env = os.getenv("UV_CACHE_DIR") + if env: + return Path(env) + return root.parent / "uv-cache" + + +class EnvBuildError(PymonikError): + """``uv venv`` or ``uv pip install`` failed for an EnvSpec.""" + + +def _flock(fd: int, op: int) -> None: + while True: + try: + fcntl.flock(fd, op) + return + except OSError as e: + if e.errno == errno.EINTR: + continue + raise + + +def _venv_python(venv_dir: Path) -> Path: + return venv_dir / "bin" / "python" + + +def ensure_env(spec: EnvSpec, *, root: Path | None = None) -> Path: + """Resolve (or build) the venv for ``spec`` and return its directory. + + Concurrent calls for the same ``env_id`` serialise on a per-env + lockfile; only one process runs ``uv pip install``. Other callers + block until the install finishes, then reuse the same venv. + + Returns the venv root (``//.venv``) so the caller can + pick the python executable or extend ``sys.path`` from it. + """ + if not spec.deps: + raise EnvBuildError("ensure_env called with empty deps; nothing to build") + + root = root or default_envs_root() + env_id = compute_env_id(spec) + env_dir = root / env_id + venv_dir = env_dir / ".venv" + sentinel = env_dir / ".ready" + lockfile = env_dir / ".lock" + + if sentinel.is_file() and _venv_python(venv_dir).exists(): + return venv_dir + + env_dir.mkdir(parents=True, exist_ok=True) + lock_fd = os.open(str(lockfile), os.O_CREAT | os.O_RDWR, 0o644) + try: + _flock(lock_fd, fcntl.LOCK_EX) + # Re-check under the lock — another process may have built it. + if sentinel.is_file() and _venv_python(venv_dir).exists(): + return venv_dir + + log.info( + "env build start", + env_id=env_id, + deps=list(canonical_deps(spec.deps)), + index_url=spec.index_url or None, + ) + t0 = time.monotonic() + + if venv_dir.exists(): + shutil.rmtree(venv_dir, ignore_errors=True) + + env = os.environ.copy() + env.setdefault("UV_CACHE_DIR", str(_uv_cache_dir(root))) + env.setdefault("UV_PYTHON_DOWNLOADS", "never") + + # Build the venv against the worker's interpreter so cloudpickle + # bytecode works the same in both directions. ``uv venv -p`` with + # an absolute path pins it. + try: + subprocess.run( + ["uv", "venv", "-p", sys.executable, str(venv_dir)], + env=env, + check=True, + capture_output=True, + ) + except subprocess.CalledProcessError as e: + raise EnvBuildError( + f"`uv venv` failed for env_id={env_id}: {e.stderr.decode(errors='replace')}" + ) from e + except FileNotFoundError as e: + raise EnvBuildError( + "uv is not on PATH; the worker image must include `uv`" + ) from e + + install_cmd = ["uv", "pip", "install", "--python", str(_venv_python(venv_dir))] + if spec.index_url: + install_cmd.extend(["--index-url", spec.index_url]) + install_cmd.extend(spec.deps) + try: + subprocess.run( + install_cmd, + env=env, + check=True, + capture_output=True, + ) + except subprocess.CalledProcessError as e: + raise EnvBuildError( + f"`uv pip install` failed for env_id={env_id}: " + f"{e.stderr.decode(errors='replace')}" + ) from e + + sentinel.write_text(f"{env_id}\n") + log.info( + "env build done", + env_id=env_id, + elapsed_s=round(time.monotonic() - t0, 2), + venv=str(venv_dir), + ) + return venv_dir + finally: + try: + _flock(lock_fd, fcntl.LOCK_UN) + finally: + os.close(lock_fd) + + +def apply_env_overlay(env: tuple[tuple[str, str], ...]) -> dict[str, str | None]: + """Apply env vars to ``os.environ``, returning a snapshot of prior values. + + Use with :func:`restore_env_overlay`. Keys that didn't exist before + map to ``None`` so we can pop them on restore. + """ + prior: dict[str, str | None] = {} + for k, v in env: + prior[k] = os.environ.get(k) + os.environ[k] = v + return prior + + +def restore_env_overlay(prior: dict[str, str | None]) -> None: + for k, v in prior.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +def venv_site_packages(venv_dir: Path) -> Path: + """Return the ``site-packages`` directory inside ``venv_dir``. + + Used by the ``isolate=False`` path to splice the env into the + worker's ``sys.path`` without spawning a subprocess. + """ + lib = venv_dir / "lib" + if not lib.is_dir(): + raise EnvBuildError(f"venv has no lib/: {venv_dir}") + candidates = [p for p in lib.iterdir() if p.name.startswith("python")] + if not candidates: + raise EnvBuildError(f"venv has no lib/python*/: {venv_dir}") + sp = candidates[0] / "site-packages" + if not sp.is_dir(): + raise EnvBuildError(f"venv site-packages missing: {sp}") + return sp diff --git a/src/pymonik/_internal/exec_cache.py b/src/pymonik/_internal/exec_cache.py new file mode 100644 index 0000000..ade53ba --- /dev/null +++ b/src/pymonik/_internal/exec_cache.py @@ -0,0 +1,251 @@ +"""On-disk execution cache. + +Opt-in via ``PymonikClient(cache=...)`` (enables the cache *infrastructure*) +plus ``@task(cache=True)`` (opts a specific task in). When both are set, +``Task.spawn(...)`` / ``Task.map(...)`` compute a content hash of the +``(function, args, kwargs)`` triple and consult the cache *before* +submitting. A hit returns a ``Future`` that's already resolved with the +cached value — zero RPCs, zero workers scheduled. A miss submits as +normal and writes the result back when it lands. + +Layout +------ + +:: + + / + ab/ + ab12cd34…ef.pkl # cloudpickled task return value + cd/ + ... + +Two-char prefix avoids one giant directory; key is the SHA-256 hex of +the canonicalised hash inputs. + +What's safe to cache +-------------------- + +The user is responsible for declaring a task pure (``@task(cache=True)``). +Caching skips automatically when an arg can't be hashed deterministically: + +- ``Future`` and ``FutureList`` args → upstream value not yet known; we + can't compute a stable key without waiting. +- Anything cloudpickle can't dump. + +Blob and Materialize args contribute their content hash (stable across +sessions and machines), so they participate in cache keys without +forcing the whole call to be a miss. + +The key prefix includes ``pymonik.__version__`` and ``python_minor`` so +upgrading either invalidates entries cleanly. +""" + +from __future__ import annotations + +import hashlib +import os +import sys +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import cloudpickle +from pymonik._internal._logging import get_logger + +if TYPE_CHECKING: + pass + +log = get_logger(__name__) + + +def python_minor() -> str: + return f"{sys.version_info.major}.{sys.version_info.minor}" + + +def default_cache_dir() -> Path: + """``~/.cache/pymonik`` on most platforms; honours XDG_CACHE_HOME.""" + xdg = os.getenv("XDG_CACHE_HOME") + base = Path(xdg) if xdg else Path.home() / ".cache" + return base / "pymonik" + + +def _hash_arg(value: Any) -> bytes | None: + """Hash one arg-tree leaf. Returns ``None`` when the leaf makes the + call uncacheable (typically: contains a ``Future``). + """ + # Local imports keep this module light at top-level. + from pymonik.blob import Blob, Materialize + from pymonik.future import Future, FutureList + + if isinstance(value, (Future, FutureList)): + return None # upstream value not known yet — can't hash + if isinstance(value, Blob): + # blob.result_id is content-addressed locally (``local-blob-``) + # or stable per-session on the cluster — both are safe inputs. + return f"B:{value.encoding}:{value.result_id}".encode() + if isinstance(value, Materialize): + return f"M:{value.result_id}:{value.worker_path}".encode() + if isinstance(value, list): + parts: list[bytes] = [b"L"] + for v in value: + h = _hash_arg(v) + if h is None: + return None + parts.append(h) + return b":".join(parts) + if isinstance(value, tuple): + parts = [b"T"] + for v in value: + h = _hash_arg(v) + if h is None: + return None + parts.append(h) + return b":".join(parts) + if isinstance(value, dict): + parts = [b"D"] + for k in sorted(value.keys(), key=lambda k: repr(k)): + sub = _hash_arg(value[k]) + if sub is None: + return None + parts.append(repr(k).encode()) + parts.append(sub) + return b":".join(parts) + # Leaf — cloudpickle hash. Cloudpickle is deterministic for plain + # data; for closures the bytes capture the identity. + try: + return b"P" + hashlib.sha256(cloudpickle.dumps(value)).digest() + except Exception: + return None + + +def compute_cache_key( + *, + pymonik_version: str, + task_name: str, + function_pickle_hash: bytes, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> str | None: + """Stable hash for ``(version, python, task, function, args, kwargs)``. + + Returns ``None`` when any arg makes the call uncacheable. + """ + parts: list[bytes] = [ + b"v=" + pymonik_version.encode(), + b"py=" + python_minor().encode(), + b"task=" + task_name.encode(), + b"fn=" + function_pickle_hash, + ] + for a in args: + h = _hash_arg(a) + if h is None: + return None + parts.append(b"a=" + h) + for k in sorted(kwargs.keys()): + sub = _hash_arg(kwargs[k]) + if sub is None: + return None + parts.append(f"k:{k}=".encode() + sub) + return hashlib.sha256(b"||".join(parts)).hexdigest() + + +class ExecCache: + """Disk-backed result cache. + + Atomic writes via tempfile + rename — a crashed write leaves no + half-file in the cache. Reads that fail to unpickle (post-upgrade + incompatibility, partial old entry, etc.) are treated as misses + and the bad file is removed. + """ + + def __init__(self, root: Path) -> None: + self._root = root + self._root.mkdir(parents=True, exist_ok=True) + + @property + def root(self) -> Path: + return self._root + + def _path(self, key: str) -> Path: + return self._root / key[:2] / f"{key}.pkl" + + def get_bytes(self, key: str) -> bytes: + """Return the cloudpickled bytes for ``key`` or raise ``KeyError``.""" + p = self._path(key) + if not p.exists(): + raise KeyError(key) + try: + return p.read_bytes() + except OSError as e: + raise KeyError(key) from e + + def get(self, key: str) -> Any: + """Decoded equivalent of :meth:`get_bytes`.""" + raw = self.get_bytes(key) + try: + return cloudpickle.loads(raw) + except Exception as e: + # Post-upgrade-style incompatibility — drop and miss. + try: + self._path(key).unlink() + except OSError: + pass + log.warning( + "cache entry unreadable; dropped", + key=key[:16], + error=str(e), + ) + raise KeyError(key) from e + + def put_bytes(self, key: str, data: bytes) -> None: + """Atomic write of ``data`` (already cloudpickled) at ``key``.""" + p = self._path(key) + p.parent.mkdir(parents=True, exist_ok=True) + fd, tmpname = tempfile.mkstemp(dir=p.parent, prefix=".tmp-", suffix=".pkl") + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + os.replace(tmpname, p) + except Exception: + try: + os.unlink(tmpname) + except OSError: + pass + raise + + def clear(self) -> int: + """Delete every entry. Returns the number of files removed.""" + count = 0 + if not self._root.exists(): + return 0 + for sub in self._root.iterdir(): + if sub.is_dir(): + for f in sub.iterdir(): + if f.suffix == ".pkl": + try: + f.unlink() + count += 1 + except OSError: + pass + try: + sub.rmdir() + except OSError: + pass + return count + + def stats(self) -> dict[str, int]: + """Return ``{"entries": N, "bytes": M}`` for the current cache.""" + entries = 0 + total = 0 + if not self._root.exists(): + return {"entries": 0, "bytes": 0} + for sub in self._root.iterdir(): + if sub.is_dir(): + for f in sub.iterdir(): + if f.suffix == ".pkl": + try: + total += f.stat().st_size + entries += 1 + except OSError: + pass + return {"entries": entries, "bytes": total} diff --git a/src/pymonik/_internal/info.py b/src/pymonik/_internal/info.py new file mode 100644 index 0000000..6c33141 --- /dev/null +++ b/src/pymonik/_internal/info.py @@ -0,0 +1,164 @@ +"""Homogenised resource Info classes. + +The upstream ``armonik.common`` types use different id field names per +resource (``Task.id`` vs ``Result.result_id`` vs ``Session.session_id`` +vs ``Partition.id``). The fluent introspection layer wraps them in +matching Info classes that all expose ``.id`` so user code can iterate +over heterogeneous result lists without learning four name variants. + +Each ``*Info`` is a frozen dataclass — pure data, no methods that talk +to a session. Mutations (cancel / delete / download / etc.) live on the +``Query`` (batched, single round-trip) rather than on the row, so the +common case ("delete every result older than X") is one RPC. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from typing import TYPE_CHECKING, Any, Optional + +if TYPE_CHECKING: + from armonik.common import Partition as _ArmPartition + from armonik.common import Result as _ArmResult + from armonik.common import Session as _ArmSession + from armonik.common import Task as _ArmTask + + +@dataclass(frozen=True, slots=True, kw_only=True) +class TaskInfo: + """Snapshot of a task. ``id`` is always populated; everything else is + nullable since ArmoniK's list endpoints sometimes return summaries.""" + + id: str + session_id: str + status: Any # armonik.common.TaskStatus + partition_id: Optional[str] = None + priority: Optional[int] = None + created_at: Optional[datetime] = None + submitted_at: Optional[datetime] = None + started_at: Optional[datetime] = None + ended_at: Optional[datetime] = None + creation_to_end_duration: Optional[timedelta] = None + expected_output_ids: list[str] = field(default_factory=list) + data_dependencies: list[str] = field(default_factory=list) + payload_id: Optional[str] = None + pod_hostname: Optional[str] = None + error: Optional[str] = None + status_message: Optional[str] = None + + @classmethod + def from_armonik(cls, t: "_ArmTask") -> "TaskInfo": + # ``Task.error`` lives under ``output.error`` on the upstream model. + err: Optional[str] = None + out = getattr(t, "output", None) + if out is not None: + err = getattr(out, "error", None) or None + return cls( + id=t.id, + session_id=t.session_id, + status=t.status, + partition_id=getattr(getattr(t, "options", None), "partition_id", None), + priority=getattr(getattr(t, "options", None), "priority", None), + created_at=t.created_at, + submitted_at=t.submitted_at, + started_at=t.started_at, + ended_at=t.ended_at, + creation_to_end_duration=t.creation_to_end_duration, + expected_output_ids=list(t.expected_output_ids or []), + data_dependencies=list(t.data_dependencies or []), + payload_id=getattr(t, "payload_id", None), + pod_hostname=getattr(t, "pod_hostname", None), + error=err, + status_message=getattr(t, "status_message", None), + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class ResultInfo: + """Snapshot of a result. ``id`` (renamed from upstream ``result_id``) + plus the rest. ``size_bytes`` is the storage-backend payload size when + known.""" + + id: str + session_id: str + name: Optional[str] = None + status: Any # armonik.common.ResultStatus + size_bytes: Optional[int] = None + owner_task_id: Optional[str] = None + created_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + created_by: Optional[str] = None + + @classmethod + def from_armonik(cls, r: "_ArmResult") -> "ResultInfo": + return cls( + id=r.result_id, + session_id=r.session_id, + name=getattr(r, "name", None), + status=r.status, + size_bytes=getattr(r, "size", None), + owner_task_id=getattr(r, "owner_task_id", None), + created_at=getattr(r, "created_at", None), + completed_at=getattr(r, "completed_at", None), + created_by=getattr(r, "created_by", None), + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class SessionInfo: + """Snapshot of a session. ``id`` (renamed from upstream + ``session_id``).""" + + id: str + status: Any # armonik.common.SessionStatus + partition_ids: list[str] = field(default_factory=list) + client_submission: Optional[bool] = None + worker_submission: Optional[bool] = None + created_at: Optional[datetime] = None + cancelled_at: Optional[datetime] = None + closed_at: Optional[datetime] = None + purged_at: Optional[datetime] = None + deleted_at: Optional[datetime] = None + duration: Optional[timedelta] = None + + @classmethod + def from_armonik(cls, s: "_ArmSession") -> "SessionInfo": + return cls( + id=s.session_id, + status=s.status, + partition_ids=list(s.partition_ids or []), + client_submission=getattr(s, "client_submission", None), + worker_submission=getattr(s, "worker_submission", None), + created_at=getattr(s, "created_at", None), + cancelled_at=getattr(s, "cancelled_at", None), + closed_at=getattr(s, "closed_at", None), + purged_at=getattr(s, "purged_at", None), + deleted_at=getattr(s, "deleted_at", None), + duration=getattr(s, "duration", None), + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class PartitionInfo: + """Snapshot of a partition.""" + + id: str + priority: Optional[int] = None + pod_max: Optional[int] = None + pod_reserved: Optional[int] = None + preemption_percentage: Optional[int] = None + parent_partition_ids: list[str] = field(default_factory=list) + pod_configuration: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_armonik(cls, p: "_ArmPartition") -> "PartitionInfo": + return cls( + id=p.id, + priority=getattr(p, "priority", None), + pod_max=getattr(p, "pod_max", None), + pod_reserved=getattr(p, "pod_reserved", None), + preemption_percentage=getattr(p, "preemption_percentage", None), + parent_partition_ids=list(getattr(p, "parent_partition_ids", []) or []), + pod_configuration=dict(getattr(p, "pod_configuration", {}) or {}), + ) diff --git a/src/pymonik/_internal/notebook.py b/src/pymonik/_internal/notebook.py new file mode 100644 index 0000000..d74fb84 --- /dev/null +++ b/src/pymonik/_internal/notebook.py @@ -0,0 +1,204 @@ +"""Jupyter / IPython rich-display helpers for Future / FutureList. + +Loaded lazily from ``Future._repr_html_`` / ``_ipython_display_`` so the +core library has no IPython dependency. In notebook frontends we paint +an HTML snapshot and start a daemon thread that refreshes it (via +``display_id``) until every tracked future resolves — Modal-style live +progress without ipywidgets. Outside a notebook the thread is never +started; the static snapshot or plain ``__repr__`` is what the user sees. +""" + +from __future__ import annotations + +import html +import threading +from typing import TYPE_CHECKING, Any, Callable + +if TYPE_CHECKING: + from pymonik.future import Future, FutureList + + +def _state(fut: "Future[Any]") -> str: + if fut._error is not None: + return "error" + if fut._done.is_set(): + return "done" + return "pending" + + +# Scoped class names + a single keyframe; safe to inline more than once +# per cell since duplicate +""" + + +def future_html(fut: "Future[Any]") -> str: + state = _state(fut) + label = {"pending": "pending", "done": "done", "error": "failed"}[state] + tid = html.escape(fut._task_id) + extra = "" + if state == "error" and fut._error is not None: + etype = html.escape(type(fut._error).__name__) + extra = f' · {etype}' + return ( + _CSS + + f'
' + + f'Future{tid}' + + f'{label}{extra}
' + ) + + +def future_list_html(fl: "FutureList[Any]") -> str: + futs = list(fl._futures) + n = len(futs) + done = sum(1 for f in futs if f._done.is_set() and f._error is None) + failed = sum(1 for f in futs if f._error is not None) + pending = n - done - failed + pct = ((done + failed) * 100) // n if n else 100 + + # Cells shrink as N grows so the heatmap stays around the same width. + if n <= 256: + cell_px, cols = 12, min(32, n or 1) + elif n <= 1024: + cell_px, cols = 6, min(64, n) + else: + cell_px, cols = 3, min(128, n) + + cells: list[str] = [] + for f in futs: + st = _state(f) + title = html.escape(f._task_id) + cells.append(f'
') + grid_style = f"grid-template-columns: repeat({cols}, {cell_px}px);" + cell_size_css = ( + f"" + ) + + failed_str = ( + f' · {failed} failed' if failed else "" + ) + return ( + _CSS + cell_size_css + + '
' + + '
' + + 'FutureList' + + f'{done}/{n} done' + + f'· {pending} pending{failed_str}' + + f'
' + + '
' + + f'
' + "".join(cells) + "
" + + '
' + ) + + +class _Snapshot: + """Tiny carrier so ``display(...)`` picks HTML in notebooks, text elsewhere.""" + + __slots__ = ("_html", "_text") + + def __init__(self, html: str, text: str) -> None: + self._html = html + self._text = text + + def _repr_html_(self) -> str: + return self._html + + def __repr__(self) -> str: + return self._text + + +def _is_jupyter_frontend() -> bool: + """True only for kernel-backed frontends that re-render display_id updates. + + Terminal IPython would treat each ``handle.update`` as a fresh print — + spammy and pointless. We opt out there. + """ + try: + from IPython import get_ipython # type: ignore + except Exception: + return False + try: + ip = get_ipython() + except Exception: + return False + if ip is None: + return False + cls = ip.__class__.__name__ + # ZMQInteractiveShell = Jupyter; Shell = Google Colab. + return cls in ("ZMQInteractiveShell", "Shell") + + +def display_live( + obj: Any, + html_fn: Callable[[Any], str], + *, + futures: list["Future[Any]"], + interval: float = 0.5, + max_seconds: float = 3600.0, +) -> None: + """Display ``obj`` once, then refresh until ``futures`` are all done. + + A no-op outside Jupyter/Colab. Static snapshot only when every future + is already resolved at display time. + """ + try: + from IPython.display import display # type: ignore + except Exception: + return + + text_repr = repr(obj) + snap = _Snapshot(html_fn(obj), text_repr) + + if not _is_jupyter_frontend() or all(f._done.is_set() for f in futures): + display(snap) + return + + handle = display(snap, display_id=True) + if handle is None: + return + + def _update_loop() -> None: + import time as _t + + deadline = _t.monotonic() + max_seconds + while _t.monotonic() < deadline: + if all(f._done.is_set() for f in futures): + break + try: + handle.update(_Snapshot(html_fn(obj), repr(obj))) + except Exception: + return + _t.sleep(interval) + try: + handle.update(_Snapshot(html_fn(obj), repr(obj))) + except Exception: + pass + + threading.Thread(target=_update_loop, daemon=True, name="pymonik-display").start() diff --git a/src/pymonik/_internal/protocols.py b/src/pymonik/_internal/protocols.py new file mode 100644 index 0000000..0140c1c --- /dev/null +++ b/src/pymonik/_internal/protocols.py @@ -0,0 +1,48 @@ +"""Internal protocols. + +Capture the duck-typed contracts used across the codebase so static +checkers can see through them — the alternative is ``Any``-typed +``ContextVar`` slots, which both lie about what's reachable and hide +typos. + +``SubmittableSession`` is the union of what ``Task.spawn`` / +``Task.map`` / ``blob.upload`` need from whatever's stored in the +``_current_session`` ContextVar. Three concrete implementations: + +- :class:`pymonik.session.Session` — control-plane gRPC. +- :class:`pymonik.worker_session.WorkerSession` — agent-sidecar gRPC. +- :class:`pymonik.testing.local.LocalSession` — in-process executor. + +All three duck-type cleanly; the Protocol just makes that fact +explicit. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Protocol + +if TYPE_CHECKING: + from pymonik.future import Future, FutureList + from pymonik.task import Task + + +class SubmittableSession(Protocol): + """Minimum contract that ``Task.spawn`` / ``blob.upload`` rely on.""" + + @property + def session_id(self) -> str: ... + + def _submit_one( + self, + task: "Task[Any, Any]", + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> "Future[Any]": ... + + def _submit_many( + self, + task: "Task[Any, Any]", + calls: list[Any], + ) -> "FutureList[Any]": ... + + def _upload_blob(self, data: bytes) -> str: ... diff --git a/src/pymonik/_internal/query.py b/src/pymonik/_internal/query.py new file mode 100644 index 0000000..dd3ccab --- /dev/null +++ b/src/pymonik/_internal/query.py @@ -0,0 +1,778 @@ +"""Fluent introspection layer. + +Three resource queries — ``TaskQuery``, ``SessionQuery``, +``ResultQuery``, ``PartitionQuery`` — share a small chainable surface: + + .where(**kwargs) → AND-combined predicates + .where_expr(filter) → raw upstream Filter expression + .order_by(*fields) → ascending; '-field' for descending + .limit(n) / .offset(n) + .list() / .first() / .count() (sync terminals) + .list_async() / .first_async() / .count_async() + for x in q: / async for x in q: (paginated iteration) + +Plus per-resource mutation verbs: cancel/delete/download for +tasks/results, full session lifecycle (cancel/pause/resume/close/ +purge/delete/stop_submission) for sessions. + +Predicate suffixes (Django-style): + + field=v → == + field__ne=v → != + field__lt=v → < (ordered fields) + field__lte / __gt / __gte + field__in=[a, b, c] → OR-chain of == + field__startswith=s → string prefix + field__endswith=s → string suffix + field__contains=s → substring + field__notcontains=s + +Predicate names accept both the homogenised ``id`` and the upstream +``task_id`` / ``result_id`` / ``session_id`` so callers can use either +shape — that's also what makes ``session.tasks.where(id=tid)`` and +``client.tasks.where(task_id=tid)`` mean the same thing. + +Mutations always materialise the matching set first via paginated +list calls. Bulk-friendly verbs (cancel_tasks, delete_result_data) are +batched; per-session verbs iterate one at a time. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field, replace +from datetime import datetime, timedelta +from pathlib import Path +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterator, + Callable, + Generic, + Iterable, + Iterator, + Mapping, + Optional, + TypeVar, +) + +import anyio +from pymonik._internal._logging import get_logger +from armonik.client import ( + ArmoniKPartitions, + ArmoniKResults, + ArmoniKSessions, + ArmoniKTasks, + PartitionFieldFilter, + ResultFieldFilter, + SessionFieldFilter, + TaskFieldFilter, +) +from armonik.common import Direction +from armonik.common.filter import Filter + +from pymonik._internal.info import ( + PartitionInfo, + ResultInfo, + SessionInfo, + TaskInfo, +) + +log = get_logger(__name__) + + +T = TypeVar("T") + + +# Default page size when iterating; configurable via PymonikClient if needed. +_DEFAULT_PAGE_SIZE = 100 + +# Cap how many items .list() will materialise before forcing pagination +# semantics on the caller. Above this users should iterate. +_MAX_LIST_ITEMS = 10_000 + + +# Per-instance memoisation for ResultQuery's session-scoped candidate +# id enumeration. Keyed by id(query) to avoid touching the frozen state. +_SCOPED_CANDIDATE_CACHE: dict[int, list[str]] = {} + + +# ---------- predicate translation ---------- + +# Map "task_id" / "id" / "session_id" / etc. → upstream FieldFilter constant. +# ``id`` is the homogenised name; the resource-specific names also resolve. + +_TASK_FIELDS: dict[str, Filter] = { + "id": TaskFieldFilter.TASK_ID, + "task_id": TaskFieldFilter.TASK_ID, + "session_id": TaskFieldFilter.SESSION_ID, + "status": TaskFieldFilter.STATUS, + "priority": TaskFieldFilter.PRIORITY, + "partition_id": TaskFieldFilter.PARTITION_ID, + "max_retries": TaskFieldFilter.MAX_RETRIES, + "max_duration": TaskFieldFilter.MAX_DURATION, + "created_at": TaskFieldFilter.CREATED_AT, + "submitted_at": TaskFieldFilter.SUBMITTED_AT, + "received_at": TaskFieldFilter.RECEIVED_AT, + "acquired_at": TaskFieldFilter.ACQUIRED_AT, + "started_at": TaskFieldFilter.STARTED_AT, + "ended_at": TaskFieldFilter.ENDED_AT, + "pod_hostname": TaskFieldFilter.POD_HOSTNAME, + "owner_pod_id": TaskFieldFilter.OWNER_POD_ID, + "initial_task_id": TaskFieldFilter.INITIAL_TASK_ID, + "engine_type": TaskFieldFilter.ENGINE_TYPE, + "error": TaskFieldFilter.ERROR, + "application_name": TaskFieldFilter.APPLICATION_NAME, + "application_version": TaskFieldFilter.APPLICATION_VERSION, + "application_namespace": TaskFieldFilter.APPLICATION_NAMESPACE, + "application_service": TaskFieldFilter.APPLICATION_SERVICE, + "creation_to_end_duration": TaskFieldFilter.CREATION_TO_END_DURATION, + "processing_to_end_duration": TaskFieldFilter.PROCESSING_TO_END_DURATION, + "pod_ttl": TaskFieldFilter.POD_TTL, +} + +_RESULT_FIELDS: dict[str, Filter] = { + "id": ResultFieldFilter.RESULT_ID, + "result_id": ResultFieldFilter.RESULT_ID, + "status": ResultFieldFilter.STATUS, +} + +_SESSION_FIELDS: dict[str, Filter] = { + "status": SessionFieldFilter.STATUS, +} + +_PARTITION_FIELDS: dict[str, Filter] = { + "priority": PartitionFieldFilter.PRIORITY, +} + + +def _build_predicate( + fields: Mapping[str, Filter], + name: str, + op: Optional[str], + value: Any, +) -> Filter: + """Translate ``name__op=value`` into an upstream filter expression.""" + if name not in fields: + allowed = ", ".join(sorted(fields)) + raise ValueError( + f"unknown field {name!r} for this resource; " + f"upstream supports: {allowed}" + ) + f = fields[name] + + # datetime convenience: pass-through to the filter (upstream DateFilter + # handles datetime values directly). + if op is None: + return f == value + if op == "ne": + return f != value + if op == "lt": + return f < value + if op == "lte" or op == "le": + return f <= value + if op == "gt": + return f > value + if op == "gte" or op == "ge": + return f >= value + if op == "in": + try: + it = list(value) + except TypeError as e: + raise ValueError(f"{name}__in= expected an iterable, got {type(value).__name__}") from e + if not it: + raise ValueError(f"{name}__in= requires at least one value") + first, *rest = it + out = f == first + for v in rest: + out = out | (f == v) + return out + # String-only ops — these live as methods on the upstream filter. + if op == "startswith": + if not hasattr(f, "startswith"): + raise ValueError(f"{name} doesn't support startswith") + return f.startswith(value) + if op == "endswith": + if not hasattr(f, "endswith"): + raise ValueError(f"{name} doesn't support endswith") + return f.endswith(value) + if op == "contains": + if not hasattr(f, "contains_"): + raise ValueError(f"{name} doesn't support contains") + return f.contains_(value) + if op == "notcontains": + if not hasattr(f, "notcontains_"): + raise ValueError(f"{name} doesn't support notcontains") + return f.notcontains_(value) + raise ValueError(f"unknown predicate suffix __{op}=") + + +def _filters_from_kwargs( + fields: Mapping[str, Filter], + kwargs: Mapping[str, Any], +) -> list[Filter]: + """Translate a ``where(**kwargs)`` call into a list of filters.""" + out: list[Filter] = [] + for key, value in kwargs.items(): + if "__" in key: + name, op = key.rsplit("__", 1) + else: + name, op = key, None + out.append(_build_predicate(fields, name, op, value)) + return out + + +def _and_all(filters: Iterable[Filter]) -> Optional[Filter]: + """Combine a list of filters with AND. Returns None if empty.""" + items = list(filters) + if not items: + return None + out = items[0] + for f in items[1:]: + out = out & f + return out + + +# ---------- query state ---------- + +@dataclass(frozen=True, slots=True, kw_only=True) +class _QueryState: + filters: tuple[Filter, ...] = () + order: tuple[tuple[Filter, Direction], ...] = () + limit: Optional[int] = None + offset: int = 0 + page_size: int = _DEFAULT_PAGE_SIZE + + +# ---------- base query ---------- + +class _BaseQuery(Generic[T]): + """Shared chainable behaviour. Subclasses provide ``_FIELDS`` and the + list/mutation methods specific to the resource.""" + + _FIELDS: dict[str, Filter] = {} + + __slots__ = ("_ctx", "_state") + + def __init__(self, ctx: "_QueryContext", state: Optional[_QueryState] = None) -> None: + self._ctx = ctx + self._state = state or _QueryState() + + # ---- chainable builders ---- + + def _replace(self, **patch: Any): + return type(self)(self._ctx, replace(self._state, **patch)) + + def where(self, **kwargs: Any): + """AND new predicates with whatever's already there.""" + new_filters = self._state.filters + tuple(_filters_from_kwargs(self._FIELDS, kwargs)) + return self._replace(filters=new_filters) + + def where_expr(self, expr: Filter): + """AND a raw upstream filter expression with the existing predicates. + + Use for OR / complex predicates the kwargs can't express: + ``q.where_expr((TaskFieldFilter.STATUS == ERROR) | (TaskFieldFilter.STATUS == TIMEOUT))``. + """ + return self._replace(filters=self._state.filters + (expr,)) + + def order_by(self, *fields: str): + """Sort by field(s). Prefix with ``-`` for descending.""" + order: list[tuple[Filter, Direction]] = [] + for f in fields: + if f.startswith("-"): + name, direction = f[1:], Direction.DESC + elif f.startswith("+"): + name, direction = f[1:], Direction.ASC + else: + name, direction = f, Direction.ASC + if name not in self._FIELDS: + raise ValueError( + f"can't sort by {name!r}; valid fields: " + f"{', '.join(sorted(self._FIELDS))}" + ) + order.append((self._FIELDS[name], direction)) + return self._replace(order=tuple(order)) + + def limit(self, n: int): + if n < 0: + raise ValueError("limit must be ≥ 0") + return self._replace(limit=n) + + def offset(self, n: int): + if n < 0: + raise ValueError("offset must be ≥ 0") + return self._replace(offset=n) + + def page_size(self, n: int): + if n <= 0: + raise ValueError("page_size must be > 0") + return self._replace(page_size=n) + + # ---- filter / sort assembly ---- + + def _filter(self) -> Optional[Filter]: + return _and_all(self._state.filters) + + def _sort_args(self) -> tuple[Optional[Filter], Direction]: + if not self._state.order: + return (None, Direction.ASC) + first = self._state.order[0] + if len(self._state.order) > 1: + log.debug( + "multi-field sort requested; upstream supports only one — " + "applying the first key only", + fields=[(f.field_name if hasattr(f, "field_name") else "?") for f, _ in self._state.order], + ) + return first + + # ---- terminals (subclasses do the actual work) ---- + + def list(self) -> list[T]: + return list(self._iter_pages(stop_at_limit=True)) + + async def list_async(self) -> list[T]: + return await anyio.to_thread.run_sync(self.list) + + def first(self) -> Optional[T]: + for item in self._replace(limit=1)._iter_pages(stop_at_limit=True): + return item + return None + + async def first_async(self) -> Optional[T]: + return await anyio.to_thread.run_sync(self.first) + + def count(self) -> int: + # Default: ask upstream for one page just to read ``total``. Subclasses + # can override with a cheaper RPC where one exists (e.g. tasks). + total, _ = self._fetch_page(0, 1) + return total + + async def count_async(self) -> int: + return await anyio.to_thread.run_sync(self.count) + + # ---- iteration ---- + + def __iter__(self) -> Iterator[T]: + yield from self._iter_pages(stop_at_limit=True) + + async def __aiter__(self) -> AsyncIterator[T]: + # Keep semantics simple: fetch each page on a worker thread and + # yield items synchronously. For very large result sets a more + # streaming form would help; revisit when there's a use case. + for item in await anyio.to_thread.run_sync( + lambda: list(self._iter_pages(stop_at_limit=True)) + ): + yield item + + # ---- pagination plumbing ---- + + def _iter_pages(self, *, stop_at_limit: bool) -> Iterator[T]: + page_size = self._state.page_size + # Translate offset+limit into page-arithmetic. Upstream paginates + # by zero-indexed page; honour the offset by skipping items in + # the first page we pull. + emitted = 0 + offset = self._state.offset + # Compute starting page so we don't fetch the same skipped items + # for huge offsets. + start_page = offset // page_size + skip_in_first = offset - (start_page * page_size) + page = start_page + + while True: + total, items = self._fetch_page(page, page_size) + if not items: + return + if skip_in_first: + items = items[skip_in_first:] + skip_in_first = 0 + for it in items: + yield it + emitted += 1 + if stop_at_limit and self._state.limit is not None and emitted >= self._state.limit: + return + if (page + 1) * page_size >= total: + return + page += 1 + if emitted >= _MAX_LIST_ITEMS: + log.warning( + "iteration safety cap hit", + cap=_MAX_LIST_ITEMS, + hint="use .limit() / .page_size() to paginate explicitly", + ) + return + + def _fetch_page(self, page: int, page_size: int) -> tuple[int, list[T]]: + raise NotImplementedError + + +# ---------- context ---------- + +@dataclass(slots=True) +class _QueryContext: + """Holds the gRPC clients + an optional session scope.""" + + tasks: ArmoniKTasks + sessions: ArmoniKSessions + results: ArmoniKResults + partitions: ArmoniKPartitions + # When set, scoped queries (session.tasks / session.results) AND + # this filter into every list call so callers don't have to. + scoped_session_id: Optional[str] = None + + +def _make_context(channel: Any, *, scoped_session_id: Optional[str] = None) -> _QueryContext: + return _QueryContext( + tasks=ArmoniKTasks(channel), + sessions=ArmoniKSessions(channel), + results=ArmoniKResults(channel), + partitions=ArmoniKPartitions(channel), + scoped_session_id=scoped_session_id, + ) + + +# ---------- TaskQuery ---------- + +class TaskQuery(_BaseQuery[TaskInfo]): + """Query / mutate tasks. Bound either to a client (cluster-wide) or + to a session (auto-scoped to that ``session_id``).""" + + _FIELDS = _TASK_FIELDS + + def _filter(self) -> Optional[Filter]: + f = super()._filter() + if self._ctx.scoped_session_id is None: + return f + scope = TaskFieldFilter.SESSION_ID == self._ctx.scoped_session_id + return scope if f is None else (scope & f) + + def _fetch_page(self, page: int, page_size: int) -> tuple[int, list[TaskInfo]]: + sort_field, sort_dir = self._sort_args() + kwargs: dict[str, Any] = dict( + task_filter=self._filter(), + page=page, + page_size=page_size, + sort_direction=sort_dir, + with_errors=True, + ) + if sort_field is not None: + kwargs["sort_field"] = sort_field + total, items = self._ctx.tasks.list_tasks(**kwargs) + return total, [TaskInfo.from_armonik(t) for t in items] + + def count(self) -> int: + # Tasks have a dedicated count RPC that returns per-status totals; + # for a generic count we still want a single number. + # Use the page-of-one trick to read upstream's ``total``. + return super().count() + + # ---- mutations ---- + + def cancel(self, *, chunk_size: int = 500) -> int: + """Cancel every task matching the query. Returns the count cancelled.""" + ids = [t.id for t in self._iter_pages(stop_at_limit=True)] + if not ids: + return 0 + self._ctx.tasks.cancel_tasks(task_ids=ids, chunk_size=chunk_size) + log.info("tasks cancelled", count=len(ids)) + return len(ids) + + async def cancel_async(self, *, chunk_size: int = 500) -> int: + return await anyio.to_thread.run_sync(lambda: self.cancel(chunk_size=chunk_size)) + + +# ---------- ResultQuery ---------- + +class ResultQuery(_BaseQuery[ResultInfo]): + """Query / mutate results. + + Cluster-wide (``client.results``) lists everything visible to the + caller and supports the upstream filter fields (``id``, ``status``). + Session-scoped (``session.results``) is correct but slightly more + expensive: ``list_results`` doesn't expose ``session_id`` as a + filter field, so we enumerate the session's tasks first to collect + their ``expected_output_ids``, then query results by those ids in + chunks. + """ + + _FIELDS = _RESULT_FIELDS + + # Cap how big a single RESULT_ID OR-chain we'll send in one + # ``list_results`` call. Each clause adds bytes to the protobuf + # filter message; 100 keeps us well under the 4 MiB cap and matches + # the polling-loop chunking convention from session.py. + _RID_CHUNK = 100 + + def _fetch_page(self, page: int, page_size: int) -> tuple[int, list[ResultInfo]]: + sort_field, sort_dir = self._sort_args() + + if self._ctx.scoped_session_id is None: + # Cluster-wide: straight pass-through. + kwargs: dict[str, Any] = dict( + result_filter=self._filter(), + page=page, + page_size=page_size, + sort_direction=sort_dir, + ) + if sort_field is not None: + kwargs["sort_field"] = sort_field + total, items = self._ctx.results.list_results(**kwargs) + return total, [ResultInfo.from_armonik(r) for r in items] + + # Session-scoped: collect candidate result_ids from the session's + # tasks, paginate the candidates, query each page by id-chain. + candidates = self._scoped_candidate_ids() + start = page * page_size + end = start + page_size + chunk_ids = candidates[start:end] + if not chunk_ids: + return len(candidates), [] + + rid_field = ResultFieldFilter.RESULT_ID + # Sub-chunk RESULT_ID OR-chains to stay under the protobuf cap. + results: list[Any] = [] + for i in range(0, len(chunk_ids), self._RID_CHUNK): + sub = chunk_ids[i : i + self._RID_CHUNK] + id_filter = rid_field == sub[0] + for r in sub[1:]: + id_filter = id_filter | (rid_field == r) + user_filter = self._filter() + combined = id_filter if user_filter is None else (id_filter & user_filter) + list_kwargs: dict[str, Any] = dict( + result_filter=combined, + page=0, + page_size=len(sub), + sort_direction=sort_dir, + ) + if sort_field is not None: + list_kwargs["sort_field"] = sort_field + _t, items = self._ctx.results.list_results(**list_kwargs) + results.extend(items) + + return len(candidates), [ResultInfo.from_armonik(r) for r in results] + + def _scoped_candidate_ids(self) -> list[str]: + """Enumerate ``expected_output_ids`` for every task in the session. + + Memoised per-instance so a ``.list()`` followed by a ``.count()`` + doesn't re-walk the session twice. + """ + # __slots__ on the base prohibits arbitrary attrs; use the state + # dict-style by storing on the type's own __dict__ via a side cache. + cached = _SCOPED_CANDIDATE_CACHE.get(id(self)) + if cached is not None: + return cached + sid = self._ctx.scoped_session_id + assert sid is not None + task_filter = TaskFieldFilter.SESSION_ID == sid + ids: list[str] = [] + page = 0 + page_size = 500 + while True: + total, items = self._ctx.tasks.list_tasks( + task_filter=task_filter, + page=page, + page_size=page_size, + with_errors=False, + ) + for t in items: + ids.extend(t.expected_output_ids or []) + if (page + 1) * page_size >= total or not items: + break + page += 1 + _SCOPED_CANDIDATE_CACHE[id(self)] = ids + return ids + + # ---- mutations ---- + + def _require_scope(self, op: str) -> str: + if self._ctx.scoped_session_id is None: + raise ValueError( + f"{op}() requires a session-scoped query — call from " + f"``session.results`` rather than ``client.results``." + ) + return self._ctx.scoped_session_id + + def delete(self, *, batch_size: int = 100) -> int: + """Delete the bytes of every result matching the query. + + Operates only within a session scope (``session.results``). + Returns the number of results whose data was deleted. + """ + sid = self._require_scope("delete") + ids = [r.id for r in self._iter_pages(stop_at_limit=True)] + if not ids: + return 0 + self._ctx.results.delete_result_data( + result_ids=ids, session_id=sid, batch_size=batch_size + ) + log.info("result data deleted", count=len(ids), session=sid) + return len(ids) + + async def delete_async(self, *, batch_size: int = 100) -> int: + return await anyio.to_thread.run_sync(lambda: self.delete(batch_size=batch_size)) + + def download(self) -> dict[str, bytes]: + """Download the bytes of every matching result. + + Returns ``{result_id: bytes}``. Sequential — one + ``download_result_data`` per result. For huge result sets prefer + :meth:`download_to` (saves to disk as it goes). + """ + sid = self._require_scope("download") + out: dict[str, bytes] = {} + for r in self._iter_pages(stop_at_limit=True): + out[r.id] = self._ctx.results.download_result_data( + result_id=r.id, session_id=sid + ) + log.info("results downloaded", count=len(out), session=sid) + return out + + async def download_async(self) -> dict[str, bytes]: + return await anyio.to_thread.run_sync(self.download) + + def download_to( + self, + directory: str | os.PathLike[str], + *, + filename: Optional[Callable[[ResultInfo], str]] = None, + ) -> int: + """Download each matching result to a file in ``directory``. + + ``filename(info) -> str`` lets you choose the on-disk name; the + default is ``.bin``. Returns the number of files + written. + """ + sid = self._require_scope("download_to") + out_dir = Path(directory) + out_dir.mkdir(parents=True, exist_ok=True) + n = 0 + for r in self._iter_pages(stop_at_limit=True): + data = self._ctx.results.download_result_data( + result_id=r.id, session_id=sid + ) + name = filename(r) if filename else f"{r.id}.bin" + (out_dir / name).write_bytes(data) + n += 1 + log.info("results downloaded to disk", count=n, dir=str(out_dir)) + return n + + async def download_to_async( + self, + directory: str | os.PathLike[str], + *, + filename: Optional[Callable[[ResultInfo], str]] = None, + ) -> int: + return await anyio.to_thread.run_sync( + lambda: self.download_to(directory, filename=filename) + ) + + +# ---------- SessionQuery ---------- + +class SessionQuery(_BaseQuery[SessionInfo]): + """Query / mutate sessions. Cluster-wide; ignores any session scope + on the context (a session can't filter itself).""" + + _FIELDS = _SESSION_FIELDS + + def _fetch_page(self, page: int, page_size: int) -> tuple[int, list[SessionInfo]]: + sort_field, sort_dir = self._sort_args() + kwargs: dict[str, Any] = dict( + session_filter=self._filter(), + page=page, + page_size=page_size, + sort_direction=sort_dir, + ) + if sort_field is not None: + kwargs["sort_field"] = sort_field + total, items = self._ctx.sessions.list_sessions(**kwargs) + return total, [SessionInfo.from_armonik(s) for s in items] + + # ---- mutations: each is per-session, so we iterate ---- + + def _apply(self, op_name: str, fn: Callable[[str], Any]) -> int: + n = 0 + for s in self._iter_pages(stop_at_limit=True): + try: + fn(s.id) + n += 1 + except Exception as e: + log.warning(f"{op_name} failed for one session", id=s.id, error=str(e)) + log.info(f"sessions {op_name}", count=n) + return n + + def cancel(self) -> int: + """Cancel every matching session. Returns the count succeeded.""" + return self._apply("cancelled", self._ctx.sessions.cancel_session) + + def pause(self) -> int: + return self._apply("paused", self._ctx.sessions.pause_session) + + def resume(self) -> int: + return self._apply("resumed", self._ctx.sessions.resume_session) + + def close(self) -> int: + return self._apply("closed", self._ctx.sessions.close_session) + + def purge(self) -> int: + return self._apply("purged", self._ctx.sessions.purge_session) + + def delete(self) -> int: + return self._apply("deleted", self._ctx.sessions.delete_session) + + def stop_submission(self, *, client: bool = True, worker: bool = True) -> int: + """Block further submissions on every matching session. + + ``client=True`` blocks user clients; ``worker=True`` blocks + sub-task spawns from inside running tasks. Both default to True + (full freeze). + """ + return self._apply( + "stop_submission", + lambda sid: self._ctx.sessions.stop_submission_session( + session_id=sid, client=client, worker=worker + ), + ) + + # async siblings + async def cancel_async(self) -> int: + return await anyio.to_thread.run_sync(self.cancel) + + async def pause_async(self) -> int: + return await anyio.to_thread.run_sync(self.pause) + + async def resume_async(self) -> int: + return await anyio.to_thread.run_sync(self.resume) + + async def close_async(self) -> int: + return await anyio.to_thread.run_sync(self.close) + + async def purge_async(self) -> int: + return await anyio.to_thread.run_sync(self.purge) + + async def delete_async(self) -> int: + return await anyio.to_thread.run_sync(self.delete) + + +# ---------- PartitionQuery ---------- + +class PartitionQuery(_BaseQuery[PartitionInfo]): + """Query partitions. Read-only — partitions are managed via Terraform + / Helm at deploy time, not from the SDK.""" + + _FIELDS = _PARTITION_FIELDS + + def _fetch_page(self, page: int, page_size: int) -> tuple[int, list[PartitionInfo]]: + sort_field, sort_dir = self._sort_args() + kwargs: dict[str, Any] = dict( + partition_filter=self._filter(), + page=page, + page_size=page_size, + sort_direction=sort_dir, + ) + if sort_field is not None: + kwargs["sort_field"] = sort_field + total, items = self._ctx.partitions.list_partitions(**kwargs) + return total, [PartitionInfo.from_armonik(p) for p in items] diff --git a/src/pymonik/_internal/refs.py b/src/pymonik/_internal/refs.py new file mode 100644 index 0000000..35d30da --- /dev/null +++ b/src/pymonik/_internal/refs.py @@ -0,0 +1,203 @@ +"""Arg references — sentinels that replace structured inputs on the wire. + +Three kinds: + +- ``FutureRef`` — another task's eventual result. Wired as a + data_dependency; worker receives the upstream value after ArmoniK + downloads the result bytes. + +- ``BlobRef`` — a Blob uploaded via ``pymonik.blob.upload(...)`` or + produced by auto-spill. Same ArmoniK mechanism as FutureRef; the + encoding tells the worker whether to unpickle (for Python objects) or + hand the raw bytes to the function. + +- ``MaterializeRef`` — like a BlobRef but the worker writes the bytes to + a file on disk at ``worker_path`` and the function receives a + ``pathlib.Path`` to that file. + +Walker is recursive through ``list``, ``tuple``, ``dict``, ``FutureList``. +Other container types pass through unchanged (add if a user actually +needs them). +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable + +import cloudpickle + +from pymonik.blob import ENC_BYTES, ENC_PICKLE, Blob, Materialize + +if TYPE_CHECKING: + pass + + +class FutureRef: + """Sentinel standing in for a Future during pickle/unpickle.""" + + __slots__ = ("result_id",) + + def __init__(self, result_id: str) -> None: + self.result_id = result_id + + def __repr__(self) -> str: + return f"FutureRef({self.result_id!r})" + + +class BlobRef: + """Sentinel standing in for a Blob. ``encoding`` tells the worker what + type to surface to the function: ``"pickle"`` → the unpickled object, + ``"bytes"`` → raw bytes. + """ + + __slots__ = ("result_id", "encoding") + + def __init__(self, result_id: str, encoding: str) -> None: + self.result_id = result_id + self.encoding = encoding + + def __repr__(self) -> str: + return f"BlobRef({self.result_id!r}, encoding={self.encoding!r})" + + +class MaterializeRef: + """Sentinel standing in for a Materialize — bytes materialised at + ``worker_path`` before the task runs (file write, or zip-unpack when + ``is_dir=True``); ``pathlib.Path(worker_path)`` substituted in as the + argument value. + """ + + __slots__ = ("result_id", "worker_path", "is_dir") + + def __init__(self, result_id: str, worker_path: str, *, is_dir: bool = False) -> None: + self.result_id = result_id + self.worker_path = worker_path + self.is_dir = is_dir + + def __repr__(self) -> str: + kind = "dir" if self.is_dir else "file" + return f"MaterializeRef({self.result_id!r}, at={self.worker_path!r}, {kind})" + + +# ---------- client-side: turn user values into refs + deps ---------- + +def extract_deps(value: Any, deps: list[str]) -> Any: + """Recursive walk, replacing Future/Blob/Materialize with their ref + sentinels and appending the referenced result_ids to ``deps``. + + Does NOT handle auto-spill — the session runs that as a second pass + so it has the pickled bytes on hand to upload directly. + """ + # Local imports avoid circulars at module load. + from pymonik.future import Future, FutureList + + if isinstance(value, Future): + deps.append(value.result_id) + return FutureRef(value.result_id) + if isinstance(value, Blob): + deps.append(value.result_id) + return BlobRef(value.result_id, encoding=value.encoding) + if isinstance(value, Materialize): + deps.append(value.result_id) + return MaterializeRef( + value.result_id, worker_path=value.worker_path, is_dir=value.is_dir + ) + if isinstance(value, FutureList): + return [extract_deps(v, deps) for v in value] + if isinstance(value, list): + return [extract_deps(v, deps) for v in value] + if isinstance(value, tuple): + return tuple(extract_deps(v, deps) for v in value) + if isinstance(value, dict): + return {k: extract_deps(v, deps) for k, v in value.items()} + return value + + +def is_ref(value: Any) -> bool: + return isinstance(value, (FutureRef, BlobRef, MaterializeRef)) + + +# ---------- auto-spill: pickle top-level args, upload oversize ones ---------- + +def auto_spill( + value: Any, + deps: list[str], + *, + upload_blob: Callable[[bytes], str], + threshold: int, +) -> Any: + """Top-level spill for one positional arg or kwarg value. + + If ``value`` is already a ref sentinel, pass through. Otherwise + cloudpickle it; if the blob exceeds ``threshold``, upload it and + replace with a ``BlobRef`` sentinel. Sub-container elements are NOT + examined individually — a large list is uploaded as a whole rather + than split up. + """ + if is_ref(value): + return value + buf = cloudpickle.dumps(value) + if len(buf) <= threshold: + return value + result_id = upload_blob(buf) + deps.append(result_id) + return BlobRef(result_id, encoding=ENC_PICKLE) + + +# ---------- worker-side: resolve refs back to concrete values ---------- + +def resolve_refs(value: Any, data_dependencies: dict[str, bytes]) -> Any: + """Walk ``value`` recursively, replacing each ref with its concrete value.""" + if isinstance(value, FutureRef): + return cloudpickle.loads(data_dependencies[value.result_id]) + + if isinstance(value, BlobRef): + raw = data_dependencies[value.result_id] + if value.encoding == ENC_PICKLE: + return cloudpickle.loads(raw) + if value.encoding == ENC_BYTES: + return raw + raise ValueError(f"unknown blob encoding: {value.encoding!r}") + + if isinstance(value, MaterializeRef): + raw = data_dependencies[value.result_id] + if value.is_dir: + _unzip_materialized(value.worker_path, raw) + else: + _write_materialized(value.worker_path, raw) + return Path(value.worker_path) + + if isinstance(value, list): + return [resolve_refs(v, data_dependencies) for v in value] + if isinstance(value, tuple): + return tuple(resolve_refs(v, data_dependencies) for v in value) + if isinstance(value, dict): + return {k: resolve_refs(v, data_dependencies) for k, v in value.items()} + + return value + + +def _write_materialized(worker_path: str, data: bytes) -> None: + # Create parent dirs if the caller wrote e.g. at="/tmp/cfg/app.toml". + parent = os.path.dirname(worker_path) + if parent: + os.makedirs(parent, exist_ok=True) + with open(worker_path, "wb") as f: + f.write(data) + + +def _unzip_materialized(worker_path: str, data: bytes) -> None: + """Unpack zipped directory bytes into ``worker_path``. + + Creates ``worker_path`` if missing. Files inside the archive land at + ``/``. Existing files at the same path + are overwritten. + """ + import io + import zipfile + + os.makedirs(worker_path, exist_ok=True) + with zipfile.ZipFile(io.BytesIO(data)) as zf: + zf.extractall(worker_path) diff --git a/src/pymonik/_internal/submit.py b/src/pymonik/_internal/submit.py new file mode 100644 index 0000000..80a8d02 --- /dev/null +++ b/src/pymonik/_internal/submit.py @@ -0,0 +1,336 @@ +"""Shared submission pipeline. + +Three concrete sessions submit tasks for execution: the client-side +:class:`Session` (talks to ArmoniK's control plane), the worker-side +:class:`WorkerSession` (talks to the agent sidecar from inside a running +task), and the in-process :class:`LocalSession` (runs everything in a +thread pool). All three share the same logical pipeline: + + normalise calls + → extract refs (Future / Blob / Materialize) into the wire envelope + → auto-spill oversize args + → cloudpickle (function, args, kwargs) + → encode TaskEnvelope (msgspec) + → allocate output result_ids + → upload payloads + → submit task definitions + → wrap each (task_id, output_id) in a Future + apply retry policy + +The transport-specific bits — *how* you allocate, upload, and submit — +live behind :class:`SubmissionBackend`. The session-specific bits — what +flavour of Future to build, whether retries apply, whether to register +the future in a pending dict — are passed as small callables to +:func:`submit_many`. Every session does its work via the same +orchestrator; new pipeline features (e.g. ``import_data`` dedup, OTel +attachment) land in one place instead of three. +""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING, Any, Callable, Iterable, Optional, Protocol + +import cloudpickle +from pymonik._internal._logging import get_logger +from armonik.common import TaskDefinition, TaskOptions + +from pymonik import envelope as env_mod +from pymonik._internal import _otel +from pymonik._internal.refs import auto_spill, extract_deps +from pymonik.envelope import EnvSpec, TaskEnvelope +from pymonik.errors import PymonikError +from pymonik.future import Future, FutureList +from pymonik.options import TaskOpts, resolve_backoff + +log = get_logger(__name__) + +if TYPE_CHECKING: + from pymonik.task import Task + + +class SubmissionBackend(Protocol): + """Transport interface used by :func:`submit_many`. + + Three methods, plus a ``session_id`` so payload / output names get a + namespace prefix. The whole protocol is intentionally tiny so each + backend (control-plane gRPC, agent sidecar, in-process) can implement + it without inheriting infrastructure it doesn't need. + """ + + @property + def session_id(self) -> str: ... + + @property + def allowed_partitions(self) -> tuple[str, ...] | None: + """Partitions this backend's session can route into, or ``None`` + for no restriction (worker / local backends, where ArmoniK isn't + in the loop). + """ + ... + + def allocate_outputs(self, names: list[str]) -> list[str]: + """Reserve N output result_ids, one per task to submit.""" + ... + + def upload_payloads(self, named_data: dict[str, bytes]) -> dict[str, str]: + """Upload payload bytes. Returns ``{requested_name: result_id}``.""" + ... + + def submit( + self, + definitions: list[TaskDefinition], + default_options: TaskOptions, + ) -> list[str]: + """Submit N task definitions. Returns ``task_id``s in order.""" + ... + + +def normalise_calls( + calls: Iterable[Any], +) -> list[tuple[tuple[Any, ...], dict[str, Any]]]: + """Coerce ``spawn`` / ``map`` arg shapes into ``(args, kwargs)`` pairs. + + Accepts: + + - ``[(args_tuple, kwargs_dict), ...]`` — the canonical form used by + ``_submit_one``. + - ``[args_tuple, ...]`` — what ``Task.starmap([(1, 2), (3, 4)])`` produces. + - ``[scalar, ...]`` — single-arg shorthand for one-positional tasks. + """ + out: list[tuple[tuple[Any, ...], dict[str, Any]]] = [] + for call in calls: + if ( + isinstance(call, tuple) + and len(call) == 2 + and isinstance(call[0], tuple) + and isinstance(call[1], dict) + ): + out.append(call) + else: + args_t = call if isinstance(call, tuple) else (call,) + out.append((args_t, {})) + return out + + +def submit_many( + *, + task: "Task[Any, Any]", + calls: Iterable[Any], + backend: SubmissionBackend, + blob_uploader: Callable[[bytes], str], + spill_threshold: int, + default_opts: TaskOpts, + partition: str, + future_factory: Callable[ + [str, list[str], tuple[Any, ...], dict[str, Any]], Any + ], + on_submitted: Optional[Callable[[list[str], Any], None]] = None, + apply_retry_policy: bool = True, + existing_future: Optional[Future[Any]] = None, + attempt: int = 1, +) -> FutureList: + """Run the full submission pipeline. + + For single-output tasks, each task gets one ArmoniK ``expected_output_id``. + For multi-output tasks (``task.multi_fields`` is set), each task gets + N output ids, one per declared field, in sorted-field order. The + ``future_factory`` receives the full list per task and builds either + a :class:`Future` or a :class:`MultiResultHandle` accordingly. + + Args: + task: the decorated function (provides ``func`` + ``opts``). + calls: iterable of call shapes (see :func:`normalise_calls`). + backend: transport plug-in. + blob_uploader: callable used by auto-spill to upload oversize args. + spill_threshold: cloudpickle-size threshold above which inline args + are auto-spilled to a Blob and replaced with a ``BlobRef``. + default_opts: session default options (merged with @task opts). + partition: session partition (backstops ``opts.partition``). + future_factory: builds a future-shape from ``(task_id, output_ids, + args, kwargs)``. Returns ``Future`` for single-output, or + ``MultiResultHandle`` for multi-output. + on_submitted: optional ``(output_ids, future_or_handle)`` + callback. The session's pending-future dict registers each + output id keyed to the same future-or-handle so the + completion loop can resolve any field. + apply_retry_policy: when True (default), attaches retry state to + each future per ``task.opts.retry_on/retry_backoff``. Worker + sessions disable this — workers don't retry. + existing_future: retry path. Reuse this Future (rewriting its + ``_task_id``/``_result_id``) instead of constructing a new + one. Single-task, single-output only. + attempt: envelope ``attempt`` field. 1 for fresh submissions, ≥2 + for retries. + """ + normalised = normalise_calls(calls) + n = len(normalised) + if n == 0: + return FutureList([]) + + if existing_future is not None and n != 1: + raise PymonikError("existing_future is only valid for a single submission") + + # Validate partition selection BEFORE we hit the network — no point + # allocating ids for a request that's about to fail. + merged_opts = default_opts.merge(task.opts) + allowed = backend.allowed_partitions + if allowed is not None and merged_opts.partition is not None: + if merged_opts.partition not in allowed: + raise PymonikError( + f"task {task.name!r} requested partition " + f"{merged_opts.partition!r}, but the session is only bound " + f"to {list(allowed)}. Pass that partition to " + f"client.session(partition=[...]) to enable it." + ) + + multi_fields: tuple[str, ...] = task.multi_fields or () + n_outputs_per_task = len(multi_fields) if multi_fields else 1 + + if existing_future is not None and n_outputs_per_task != 1: + raise PymonikError( + "retry of a multi-output task is not yet supported" + ) + + _otel.setup() + + with _otel.start_span( + "pymonik.submit", + attrs={ + "pymonik.func": task.name, + "pymonik.count": n, + "pymonik.partition": merged_opts.partition or partition, + "pymonik.attempt": attempt, + "pymonik.outputs_per_task": n_outputs_per_task, + }, + kind="client", + ) as submit_span: + traceparent_carrier: dict[str, str] = {} + _otel.inject_context(traceparent_carrier) + otel_ctx_tuple: tuple[tuple[str, str], ...] = tuple( + sorted(traceparent_carrier.items()) + ) + + # 1. Allocate output result_ids. For multi-output tasks each + # task gets N ids, in stable (sorted-field) order. + output_names: list[str] = [] + for _ in range(n): + if multi_fields: + for field in multi_fields: + output_names.append( + f"{backend.session_id}__out__{task.name}__{field}__{uuid.uuid4()}" + ) + else: + output_names.append( + f"{backend.session_id}__out__{task.name}__{uuid.uuid4()}" + ) + all_output_ids = backend.allocate_outputs(output_names) + output_groups: list[list[str]] = [ + all_output_ids[i * n_outputs_per_task : (i + 1) * n_outputs_per_task] + for i in range(n) + ] + + # 2. Build envelopes per call, collect data deps. + fn_pickle = cloudpickle.dumps(task.func) + payload_blobs: dict[str, bytes] = {} + payload_names: list[str] = [] + task_deps: list[list[str]] = [] + + env_dict = merged_opts.env or {} + env_spec: EnvSpec | None = None + if merged_opts.deps or env_dict: + env_spec = EnvSpec( + deps=tuple(merged_opts.deps or ()), + isolate=merged_opts.isolate if merged_opts.isolate is not None else False, + index_url=merged_opts.index_url or "", + env=tuple(sorted(env_dict.items())), + ) + + for args, kwargs in normalised: + deps: list[str] = [] + args_rewritten = tuple(extract_deps(a, deps) for a in args) + kwargs_rewritten = {k: extract_deps(v, deps) for k, v in kwargs.items()} + args_rewritten = tuple( + auto_spill(a, deps, upload_blob=blob_uploader, threshold=spill_threshold) + for a in args_rewritten + ) + kwargs_rewritten = { + k: auto_spill(v, deps, upload_blob=blob_uploader, threshold=spill_threshold) + for k, v in kwargs_rewritten.items() + } + envelope = TaskEnvelope( + function_pickle=fn_pickle, + args_pickle=cloudpickle.dumps((args_rewritten, kwargs_rewritten)), + func_name=task.name, + attempt=attempt, + env_spec=env_spec, + otel_context=otel_ctx_tuple, + multi_fields=multi_fields, + ) + name = f"{backend.session_id}__pl__{task.name}__{uuid.uuid4()}" + payload_names.append(name) + payload_blobs[name] = env_mod.encode(envelope) + task_deps.append(sorted(set(deps))) + + # 3. Upload payloads. + payload_id_map = backend.upload_payloads(payload_blobs) + payload_ids = [payload_id_map[name] for name in payload_names] + + # 4. Per-task options uniform per batch — see comment in + # _ClientBackend on why options= isn't on each TaskDefinition. + per_task_options = merged_opts.to_armonik(default_partition=partition) + + # 5. Submit. + definitions = [ + TaskDefinition( + payload_id=pid, + expected_output_ids=oids, + data_dependencies=deps, + ) + for pid, oids, deps in zip(payload_ids, output_groups, task_deps) + ] + task_ids = backend.submit(definitions, per_task_options) + + # 6. Build / rewire futures, attach retry policy, register. + retry_policy: tuple[int, tuple[type[BaseException], ...], Any] | None = None + if apply_retry_policy and task.opts.retry_on: + backoff_fn = resolve_backoff(task.opts.retry_backoff) + max_retries = task.opts.retries if task.opts.retries is not None else 3 + retry_policy = (max_retries, tuple(task.opts.retry_on), backoff_fn) + + futures: list[Any] = [] + for (args, kwargs), task_id, output_ids in zip( + normalised, task_ids, output_groups + ): + if existing_future is not None: + fut = existing_future + fut._task_id = task_id + fut._result_id = output_ids[0] + else: + fut = future_factory(task_id, output_ids, args, kwargs) + if retry_policy is not None: + # Retries only fire for single-output tasks (the + # ``_retry_state`` slot lives on Future, not + # MultiResultHandle). + max_r, on_types, backoff_fn = retry_policy + if hasattr(fut, "_retry_state"): + fut._retry_state = (task, args, kwargs, max_r, on_types, backoff_fn) + if on_submitted is not None: + on_submitted(output_ids, fut) + futures.append(fut) + + if submit_span is not None and futures: + first_task_id = getattr(futures[0], "task_id", None) + if first_task_id is not None: + submit_span.set_attribute("pymonik.first_task_id", first_task_id) + + log.info( + "batch submitted", + func=task.name, + count=n, + first_task=getattr(futures[0], "task_id", None) if futures else None, + any_deps=any(task_deps), + attempt=attempt, + outputs_per_task=n_outputs_per_task, + trace_id=_otel.current_trace_id_hex(), + ) + return FutureList(futures) diff --git a/src/pymonik/_internal/subprocess_dispatch.py b/src/pymonik/_internal/subprocess_dispatch.py new file mode 100644 index 0000000..d4d98df --- /dev/null +++ b/src/pymonik/_internal/subprocess_dispatch.py @@ -0,0 +1,213 @@ +"""Parent-side dispatcher for the per-deps subprocess path. + +When the envelope carries ``env_spec.deps`` and ``env_spec.isolate=True`` +(default), the worker hands the task off to a child Python interpreter +booted from the env's venv. This module owns: + +- starting the child via :mod:`subprocess` with ``PYTHONPATH`` rigged so + the child can ``import pymonik`` without the venv needing pymonik + itself installed (we use the worker's pymonik, the user's deps); +- writing the framed envelope + data_deps to its stdin (see + :mod:`pymonik._internal.task_runner` for the protocol); +- reading the framed result back from stdout; +- timing out / killing the child if the worker is cancelled; +- surfacing stderr on failure so users see install / runtime tracebacks. +""" + +from __future__ import annotations + +import os +import struct +import subprocess +import sys +import threading +from pathlib import Path +from typing import Mapping + +from pymonik._internal._logging import get_logger +from pymonik._internal.env_builder import _venv_python, ensure_env, venv_site_packages +from pymonik.envelope import EnvSpec +from pymonik.errors import PymonikError, TaskFailed + +log = get_logger(__name__) + + +def _u32(n: int) -> bytes: + return struct.pack(">I", n) + + +def _frame_input(envelope_bytes: bytes, data_deps: Mapping[str, bytes]) -> bytes: + parts: list[bytes] = [_u32(len(envelope_bytes)), envelope_bytes, _u32(len(data_deps))] + for k, v in data_deps.items(): + kb = k.encode("utf-8") + parts.append(_u32(len(kb))) + parts.append(kb) + parts.append(_u32(len(v))) + parts.append(v) + return b"".join(parts) + + +def _read_result(stream) -> tuple[bytes, bytes]: + tag = stream.read(1) + if not tag: + raise PymonikError("subprocess produced no result on stdout") + length_bytes = stream.read(4) + if len(length_bytes) != 4: + raise PymonikError("subprocess truncated result frame (length)") + (length,) = struct.unpack(">I", length_bytes) + payload = b"" + remaining = length + while remaining > 0: + chunk = stream.read(remaining) + if not chunk: + raise PymonikError( + f"subprocess truncated result frame (payload, " + f"got {length - remaining} of {length})" + ) + payload += chunk + remaining -= len(chunk) + return tag, payload + + +def _parent_pythonpath() -> str: + """``PYTHONPATH`` for the child so it can ``import pymonik``. + + The venv only contains the user's deps. ``pymonik``, ``cloudpickle`` + and ``msgspec`` live in the worker process's site-packages; we + forward those so the runner module can run without us installing + pymonik into every venv. + """ + paths = [p for p in sys.path if p and not p.endswith("site-packages/pymonik")] + # Drop the cwd entry — child shouldn't pick up the parent's working dir. + paths = [p for p in paths if p not in (".", "")] + existing = os.environ.get("PYTHONPATH", "") + if existing: + return os.pathsep.join([existing] + paths) + return os.pathsep.join(paths) + + +def run_in_subprocess( + *, + env_spec: EnvSpec, + envelope_bytes: bytes, + data_deps: Mapping[str, bytes], + timeout_s: float | None = None, +) -> bytes: + """Build (or reuse) the venv, dispatch the task, return cloudpickled result. + + Raises :class:`TaskFailed` with the child's traceback on user-code + failure; raises :class:`PymonikError` on infra failure (env build, + subprocess crash, framing mismatch). + """ + venv_dir = ensure_env(env_spec) + py = _venv_python(venv_dir) + if not py.exists(): + raise PymonikError(f"venv python missing after build: {py}") + + env = os.environ.copy() + env["PYTHONPATH"] = _parent_pythonpath() + # Don't inherit a __PYVENV_LAUNCHER__ that would point to the worker's + # interpreter — the child must use its venv python. + env.pop("__PYVENV_LAUNCHER__", None) + # Suppress user-site so the child stays isolated to the venv. + env["PYTHONNOUSERSITE"] = "1" + # Apply EnvSpec.env on top — user vars win. + for k, v in env_spec.env: + env[k] = v + + proc = subprocess.Popen( + [str(py), "-m", "pymonik._internal.task_runner"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + cwd=os.getcwd(), + ) + + framed = _frame_input(envelope_bytes, data_deps) + + # Capture stderr in a background thread so a chatty child can't deadlock + # us by filling the pipe. + stderr_chunks: list[bytes] = [] + assert proc.stderr is not None + stderr_pipe = proc.stderr + + def _drain_stderr(): + for chunk in iter(lambda: stderr_pipe.read(65536), b""): + stderr_chunks.append(chunk) + + stderr_thread = threading.Thread(target=_drain_stderr, daemon=True) + stderr_thread.start() + + try: + assert proc.stdin is not None and proc.stdout is not None + try: + proc.stdin.write(framed) + proc.stdin.close() + except BrokenPipeError as e: + # Child died before consuming input. Wait for stderr and report. + proc.wait(timeout=5) + stderr_thread.join(timeout=2) + raise PymonikError( + f"subprocess died before reading input: " + f"{b''.join(stderr_chunks).decode(errors='replace')}" + ) from e + + try: + tag, payload = _read_result(proc.stdout) + except Exception: + proc.wait(timeout=5) + stderr_thread.join(timeout=2) + stderr_text = b"".join(stderr_chunks).decode(errors="replace") + raise PymonikError( + f"subprocess produced no usable result (rc={proc.returncode}); " + f"stderr:\n{stderr_text}" + ) + + rc = proc.wait(timeout=timeout_s) + stderr_thread.join(timeout=2) + + if tag == b"e": + raise TaskFailed( + "subprocess", payload.decode("utf-8", errors="replace") + ) + if tag != b"r": + raise PymonikError(f"subprocess sent unknown tag: {tag!r}") + if rc != 0: + log.warning("subprocess returned non-zero", rc=rc) + return payload + finally: + if proc.poll() is None: + proc.kill() + try: + proc.wait(timeout=5) + except Exception: + pass + + +def run_in_process_with_splice( + *, + env_spec: EnvSpec, + runner, +): + """``isolate=False`` escape hatch: build the env, splice site-packages, run. + + Imports of names already loaded into ``sys.modules`` win; the splice + only helps for new imports. Concurrent sessions on the same pod with + conflicting deps will see the first-imported version of a package. + """ + venv_dir = ensure_env(env_spec) + site = venv_site_packages(venv_dir) + site_str = str(site) + inserted = False + if site_str not in sys.path: + sys.path.insert(0, site_str) + inserted = True + try: + return runner() + finally: + if inserted: + try: + sys.path.remove(site_str) + except ValueError: + pass diff --git a/src/pymonik/_internal/task_runner.py b/src/pymonik/_internal/task_runner.py new file mode 100644 index 0000000..fffd983 --- /dev/null +++ b/src/pymonik/_internal/task_runner.py @@ -0,0 +1,133 @@ +"""Subprocess entry point for ``deps``-isolated task execution. + +Invoked as ``python -m pymonik._internal.task_runner`` by the worker +when ``env_spec.deps`` is non-empty and ``env_spec.isolate`` is True. + +Wire protocol on stdin (binary, length-prefixed): + + [4 bytes BE u32] envelope length + [N bytes] msgspec-encoded TaskEnvelope + [4 bytes BE u32] data_deps map length (number of entries) + repeated N times: + [4 bytes BE u32] key length, then UTF-8 key + [4 bytes BE u32] value length, then value bytes + +Wire protocol on stdout (binary): + + [1 byte] tag: b'r' for result, b'e' for error + [4 bytes BE u32] payload length + [N bytes] cloudpickled return value (tag=r) + OR utf-8 error message (tag=e) + +Stderr is for diagnostics (uv install logs, user prints, traceback on +unexpected crash). Mixing stdout for the result and stderr for diagnostics +keeps user ``print()`` calls from corrupting the wire — the parent only +reads stdout for the framed result. + +This module deliberately has minimal imports at module-load time — +it runs inside the per-deps venv, where pymonik *is* installed (parent +spawns with ``PYTHONPATH`` set to the worker's site-packages so we can +import ``pymonik`` without re-installing it into every venv). +""" + +from __future__ import annotations + +import struct +import sys +import traceback +from typing import Any + + +def _read_exact(stream, n: int) -> bytes: + chunks: list[bytes] = [] + remaining = n + while remaining > 0: + b = stream.read(remaining) + if not b: + raise EOFError(f"task_runner: stdin closed with {remaining} bytes pending") + chunks.append(b) + remaining -= len(b) + return b"".join(chunks) + + +def _read_u32(stream) -> int: + return struct.unpack(">I", _read_exact(stream, 4))[0] + + +def _read_input(stream) -> tuple[bytes, dict[str, bytes]]: + env_len = _read_u32(stream) + envelope_bytes = _read_exact(stream, env_len) + n_deps = _read_u32(stream) + deps: dict[str, bytes] = {} + for _ in range(n_deps): + klen = _read_u32(stream) + key = _read_exact(stream, klen).decode("utf-8") + vlen = _read_u32(stream) + deps[key] = _read_exact(stream, vlen) + return envelope_bytes, deps + + +def _write_result(stream, tag: bytes, payload: bytes) -> None: + stream.write(tag) + stream.write(struct.pack(">I", len(payload))) + stream.write(payload) + stream.flush() + + +def main() -> int: + stdin = sys.stdin.buffer + stdout = sys.stdout.buffer + try: + envelope_bytes, data_deps = _read_input(stdin) + except Exception as e: + msg = f"task_runner: failed to read input: {e}\n{traceback.format_exc()}" + try: + _write_result(stdout, b"e", msg.encode("utf-8", errors="replace")) + except Exception: + sys.stderr.write(msg) + return 1 + + try: + import os + + import cloudpickle + + from pymonik import envelope as env_mod + from pymonik._internal import _otel + from pymonik._internal.refs import resolve_refs + + # OTel: same env-driven auto-enable as the parent worker. Spans + # from the subprocess become children of pymonik.submit through + # the propagated trace context in the envelope. + _otel.setup(service_name=os.getenv("OTEL_SERVICE_NAME", "pymonik-worker")) + + env = env_mod.decode(envelope_bytes) + func = cloudpickle.loads(env.function_pickle) + args, kwargs = cloudpickle.loads(env.args_pickle) + args = tuple(resolve_refs(a, data_deps) for a in args) + kwargs = {k: resolve_refs(v, data_deps) for k, v in kwargs.items()} + with _otel.use_extracted_context(dict(env.otel_context)): + with _otel.start_span( + "pymonik.task.run", + attrs={ + "pymonik.func": env.func_name, + "pymonik.attempt": env.attempt, + "pymonik.subprocess": True, + }, + kind="server", + ): + result: Any = func(*args, **kwargs) + _write_result(stdout, b"r", cloudpickle.dumps(result)) + return 0 + except BaseException as e: + tb = traceback.format_exc() + msg = f"{type(e).__name__}: {e}\n{tb}" + try: + _write_result(stdout, b"e", msg.encode("utf-8", errors="replace")) + except Exception: + sys.stderr.write(msg) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/pymonik/blob.py b/src/pymonik/blob.py new file mode 100644 index 0000000..56ec1b9 --- /dev/null +++ b/src/pymonik/blob.py @@ -0,0 +1,197 @@ +"""Blobs — first-class inputs that don't live inline in the task envelope. + +Three flavours, one mental model: a ``Blob[T]`` is a typed handle whose +bytes live in ArmoniK's object store and flow into a task via +``data_dependencies``. The handle is what user code passes around on the +client; on the worker, the function receives the resolved value (not the +handle). + +- ``blob.upload(obj)`` — cloudpickles a Python object, uploads, returns + ``Blob[T]``. The task receives the object directly. + +- ``blob.upload(Path("file"))`` — uploads raw file bytes, returns + ``Blob[bytes]``. The task receives ``bytes``. + +- ``blob.materialize(Path("local.toml"), at="/etc/app.toml")`` — uploads + raw bytes and tells the worker to write them to ``/etc/app.toml`` + before the task runs. The task parameter receives a + ``pathlib.Path("/etc/app.toml")`` pointing at the written file. + +- ``blob.materialize(Path("./assets"), at="/opt/assets")`` — when + ``source`` is a directory, the contents are zipped client-side, + uploaded, and unpacked at ``at`` on the worker before the task runs. + Task parameter receives ``pathlib.Path("/opt/assets")``. + +Dedup: content is hashed (SHA-256). Two uploads of the same bytes in the +same session reuse the first result id — no re-upload. Cross-session +dedup is not implemented. + +Size threshold: user code never needs to think about it. If a plain arg +exceeds ``PymonikClient(spill_threshold=...)`` (default 256 KiB) it's +auto-spilled to a Blob during submission. ``blob.upload(...)`` is the +explicit form — useful when the same blob will be reused across many +tasks (the dedup cache saves the upload round-trip). +""" + +from __future__ import annotations + +import hashlib +import io +import zipfile +from pathlib import Path +from typing import Any, Generic, TypeVar + +import cloudpickle + +from pymonik.task import current_session + +T = TypeVar("T") + + +# Upload encoding — what the worker does with the downloaded bytes. +ENC_PICKLE = "pickle" # cloudpickle.loads(bytes) → value +ENC_BYTES = "bytes" # hand bytes straight to the task + + +class Blob(Generic[T]): + """A typed handle to bytes stored in ArmoniK's object store.""" + + __slots__ = ("_result_id", "_encoding", "_size") + + def __init__(self, result_id: str, *, encoding: str, size: int) -> None: + self._result_id = result_id + self._encoding = encoding + self._size = size + + @property + def result_id(self) -> str: + return self._result_id + + @property + def encoding(self) -> str: + return self._encoding + + @property + def size(self) -> int: + return self._size + + def __repr__(self) -> str: + return f"" + + +class Materialize: + """A blob with a target on-worker path. + + When passed to a task, the worker materialises the bytes at + ``worker_path``: for files (``is_dir=False``) it writes them + directly; for directories (``is_dir=True``) it unpacks the zip + contents into ``worker_path``. The task parameter receives a + ``pathlib.Path`` to the materialised location. + """ + + __slots__ = ("_result_id", "_worker_path", "_size", "_is_dir") + + def __init__( + self, + result_id: str, + *, + worker_path: str, + size: int, + is_dir: bool = False, + ) -> None: + self._result_id = result_id + self._worker_path = worker_path + self._size = size + self._is_dir = is_dir + + @property + def result_id(self) -> str: + return self._result_id + + @property + def worker_path(self) -> str: + return self._worker_path + + @property + def size(self) -> int: + return self._size + + @property + def is_dir(self) -> bool: + return self._is_dir + + def __repr__(self) -> str: + kind = "dir" if self._is_dir else "file" + return ( + f"" + ) + + +# ---------- public API ---------- + +def upload(value: Any) -> Blob[Any]: + """Upload a value to the current session's object store. + + If ``value`` is a ``bytes``/``bytearray`` or ``Path``, uploads raw bytes + and the worker will receive ``bytes``. Otherwise cloudpickles the value + and the worker will receive the deserialised object. + """ + data, encoding = _encode(value) + session = current_session() + result_id = session._upload_blob(data) + return Blob(result_id, encoding=encoding, size=len(data)) + + +def materialize(source: Path | str, *, at: str) -> Materialize: + """Upload ``source`` and request placement at ``at`` on the worker. + + Files: bytes are written to ``at`` before the task runs. + Directories: contents are zipped (deflated) client-side, uploaded, + and unpacked into ``at`` on the worker. + + ``at`` is absolute (or relative to the worker's working dir). The + task parameter receives a ``pathlib.Path(at)``. + """ + src = Path(source) + if src.is_dir(): + data = _zip_directory(src) + session = current_session() + result_id = session._upload_blob(data) + return Materialize( + result_id, worker_path=at, size=len(data), is_dir=True + ) + data = src.read_bytes() + session = current_session() + result_id = session._upload_blob(data) + return Materialize(result_id, worker_path=at, size=len(data), is_dir=False) + + +def _zip_directory(root: Path) -> bytes: + """Zip a directory into bytes, deterministically (sorted entries). + + Sorting keeps the SHA-256 stable, so re-uploads of the same dir + contents dedup via the session's blob cache. + """ + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + for path in sorted(root.rglob("*")): + if path.is_file(): + zf.write(path, path.relative_to(root)) + return buf.getvalue() + + +# ---------- helpers ---------- + +def _encode(value: Any) -> tuple[bytes, str]: + """Normalise a user-supplied value into (bytes, encoding).""" + if isinstance(value, (bytes, bytearray, memoryview)): + return bytes(value), ENC_BYTES + if isinstance(value, Path): + return value.read_bytes(), ENC_BYTES + return cloudpickle.dumps(value), ENC_PICKLE + + +def content_hash(data: bytes) -> str: + """Stable hash used for within-session dedup and result naming.""" + return hashlib.sha256(data).hexdigest() diff --git a/src/pymonik/cli/__init__.py b/src/pymonik/cli/__init__.py new file mode 100644 index 0000000..259f70f --- /dev/null +++ b/src/pymonik/cli/__init__.py @@ -0,0 +1,14 @@ +"""``pymonik`` CLI. + +Standalone executable (``pymonik ``); not an ArmoniK CLI +extension. The two are deliberately separate: the host ArmoniK CLI +covers operator verbs (sessions, partitions, results), and ``pymonik`` +covers the Python-specific things (worker images, doctor, run/shell +ergonomics, replay) that don't belong in a generic cluster CLI. + +Entry point is :func:`pymonik.cli.main.cli`. +""" + +from pymonik.cli.main import cli + +__all__ = ["cli"] diff --git a/src/pymonik/cli/doctor.py b/src/pymonik/cli/doctor.py new file mode 100644 index 0000000..91be780 --- /dev/null +++ b/src/pymonik/cli/doctor.py @@ -0,0 +1,153 @@ +"""``pymonik doctor`` — local-side health check. + +Verifies the things a new user would otherwise hit as cryptic failures: + +- AKCONFIG is set / readable / parseable. +- Endpoint is reachable (gRPC channel opens within a deadline). +- Versions: pymonik / armonik / Python / cluster (via ``ArmoniKVersions``). +- Listed partitions on the cluster (sanity-check that the user's + ``--partition`` actually exists before they submit). + +Exits 0 on full green, non-zero with a one-line summary otherwise. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import Any + +import click + + +@click.command("doctor") +@click.option( + "--akconfig", + type=click.Path(dir_okay=False), + default=None, + help="Path to the ArmoniK CLI YAML config. Falls back to $AKCONFIG.", +) +@click.option( + "--endpoint", + default=None, + help="Endpoint override (e.g. http://10.43.x.y:5001). Skips AKCONFIG.", +) +@click.option( + "--timeout", + default=5.0, + show_default=True, + type=float, + help="Connection deadline in seconds.", +) +def doctor(akconfig: str | None, endpoint: str | None, timeout: float) -> None: + """Check the local config + reach the cluster + report versions.""" + failures: list[str] = [] + + # ---- pymonik / armonik / python versions ---- + try: + import pymonik + + click.echo(f" pymonik {pymonik.__version__}") + except Exception as e: # pragma: no cover — defensive + failures.append(f"pymonik import: {e!r}") + + try: + import armonik + from importlib.metadata import version as _ver + + click.echo(f" armonik (py) {_ver('armonik')}") + except Exception as e: + failures.append(f"armonik import: {e!r}") + + click.echo( + f" python {sys.version.split()[0]} " + f"({sys.implementation.name})" + ) + + # ---- AKCONFIG / endpoint resolution ---- + cfg_endpoint: str | None = None + cfg_ca: str | None = None + + if endpoint is not None: + cfg_endpoint = endpoint + click.echo(f" endpoint {endpoint} (--endpoint override)") + else: + cfg_path = akconfig or os.getenv("AKCONFIG") + if cfg_path is None: + failures.append( + "no endpoint: pass --endpoint or set AKCONFIG to your " + "armonik-cli.yaml" + ) + else: + try: + from pymonik.client import _load_akconfig + + loaded = _load_akconfig(Path(cfg_path)) + cfg_endpoint = loaded["endpoint"] + cfg_ca = loaded.get("certificate_authority") + click.echo(f" AKCONFIG {cfg_path}") + click.echo(f" endpoint {cfg_endpoint}") + if cfg_ca: + click.echo(f" ca cert {cfg_ca}") + except Exception as e: + failures.append(f"AKCONFIG load: {e!r}") + + # ---- channel reachability ---- + if cfg_endpoint is not None and not failures: + try: + import grpc + from pymonik._internal.channel import Credentials, open_channel + + creds = ( + Credentials(ca=cfg_ca) if cfg_ca else None + ) + channel = open_channel(cfg_endpoint, creds) + try: + grpc.channel_ready_future(channel).result(timeout=timeout) + click.echo(f" channel ready (≤{timeout}s)") + except Exception as e: + failures.append(f"channel not ready within {timeout}s: {e!r}") + finally: + # Try cluster-side queries before tearing down. + if not failures: + _query_cluster(channel, failures) + channel.close() + except Exception as e: + failures.append(f"channel open failed: {e!r}") + + # ---- summary ---- + click.echo("") + if failures: + click.echo(click.style("FAIL", fg="red", bold=True)) + for f in failures: + click.echo(f" - {f}") + raise SystemExit(1) + click.echo(click.style("OK", fg="green", bold=True)) + + +def _query_cluster(channel: Any, failures: list[str]) -> None: + """Best-effort cluster-side queries: versions, partitions.""" + try: + from armonik.client import ArmoniKPartitions, ArmoniKVersions + + try: + v = ArmoniKVersions(channel) + versions = v.list_versions() + click.echo(f" cluster core={versions.get('core', '?')} " + f"api={versions.get('api', '?')}") + except Exception as e: + click.echo(f" cluster (versions: {e!r})") + + try: + p = ArmoniKPartitions(channel) + total, items = p.list_partitions(page=0, page_size=50) + names = [getattr(it, "id", str(it)) for it in items] + click.echo( + f" partitions {total} total: " + f"{', '.join(names) if names else '(none listed)'}" + ) + except Exception as e: + click.echo(f" partitions (failed: {e!r})") + except Exception as e: + failures.append(f"cluster query setup: {e!r}") diff --git a/src/pymonik/cli/main.py b/src/pymonik/cli/main.py new file mode 100644 index 0000000..04bf407 --- /dev/null +++ b/src/pymonik/cli/main.py @@ -0,0 +1,189 @@ +"""Top-level ``pymonik`` Click app. + +Subcommand layout (current state — many are stubs): + + pymonik doctor # AKCONFIG + reachability + cluster version checks + pymonik run script.py # ergonomic submitter (stub) + pymonik shell # IPython repl bound to a session (stub) + pymonik image build # bake a worker image from uv.lock (stub) + pymonik image push # push to a registry (stub) + pymonik image list # show baked images (stub) + pymonik logs # stream worker stdout/stderr (stub) + pymonik replay # local replay of a failed task (stub) + +The doctor command is wired up; the rest are scaffolded so users can +discover them via ``pymonik --help`` and so future PRs land in a +predictable place. +""" + +from __future__ import annotations + +import click + +from pymonik.cli.doctor import doctor as _doctor_cmd + + +@click.group( + name="pymonik", + context_settings={"help_option_names": ["-h", "--help"]}, +) +@click.version_option(package_name="pymonik", message="pymonik %(version)s") +def cli() -> None: + """Python-side companion CLI for ArmoniK. + + For session / partition / result operator verbs, use the host + ``armonik`` CLI. ``pymonik`` covers Python-specific tooling: worker + images, doctor checks, run/shell ergonomics, replay. + """ + + +# ---------- top-level commands ---------- + +cli.add_command(_doctor_cmd) + + +@cli.command("run") +@click.argument("script", type=click.Path(exists=True, dir_okay=False)) +@click.option("--partition", default=None, help="ArmoniK partition.") +def run_script(script: str, partition: str | None) -> None: + """[stub] Run a Python script with a default Pymonik session open. + + Will eventually wrap the script in ``with PymonikClient() as client: + with client.session(partition=...): ...`` so the user doesn't have + to. Today: not implemented. + """ + click.echo("pymonik run: not yet implemented", err=True) + raise SystemExit(2) + + +@cli.command("shell") +@click.option("--partition", default=None, help="ArmoniK partition.") +def shell(partition: str | None) -> None: + """[stub] Open an IPython REPL with a session pre-opened.""" + click.echo("pymonik shell: not yet implemented", err=True) + raise SystemExit(2) + + +@cli.group("image") +def image_group() -> None: + """[stub] Worker image management.""" + + +@image_group.command("build") +def image_build() -> None: + """[stub] Bake a worker image from ``uv.lock`` (Image.from_uv).""" + click.echo("pymonik image build: not yet implemented", err=True) + raise SystemExit(2) + + +@image_group.command("push") +def image_push() -> None: + """[stub] Push a built worker image to a registry.""" + click.echo("pymonik image push: not yet implemented", err=True) + raise SystemExit(2) + + +@image_group.command("list") +def image_list() -> None: + """[stub] List baked worker images.""" + click.echo("pymonik image list: not yet implemented", err=True) + raise SystemExit(2) + + +@cli.command("logs") +@click.argument("task_id") +@click.option("--follow", "-f", is_flag=True, help="Stream new log lines.") +def logs(task_id: str, follow: bool) -> None: + """[stub] Stream a task's worker stdout/stderr.""" + click.echo("pymonik logs: not yet implemented", err=True) + raise SystemExit(2) + + +@cli.command("replay") +@click.argument("task_id") +def replay(task_id: str) -> None: + """[stub] Re-run a failed task locally with its captured inputs.""" + click.echo("pymonik replay: not yet implemented", err=True) + raise SystemExit(2) + + +@cli.group("cache") +def cache_group() -> None: + """Local execution cache management. + + See ``PymonikClient(cache=...)`` for what enables it. By default the + cache lives at ``~/.cache/pymonik`` (or ``$XDG_CACHE_HOME/pymonik`` + when set). + """ + + +@cache_group.command("path") +@click.option( + "--root", + type=click.Path(), + default=None, + help="Override the cache root (otherwise ~/.cache/pymonik / XDG).", +) +def cache_path(root: str | None) -> None: + """Print the cache directory path.""" + from pathlib import Path + + from pymonik._internal.exec_cache import default_cache_dir + + p = Path(root) if root else default_cache_dir() + click.echo(str(p)) + + +@cache_group.command("stats") +@click.option( + "--root", + type=click.Path(), + default=None, + help="Override the cache root.", +) +def cache_stats(root: str | None) -> None: + """Show entry count and total bytes in the cache.""" + from pathlib import Path + + from pymonik._internal.exec_cache import ExecCache, default_cache_dir + + p = Path(root) if root else default_cache_dir() + if not p.exists(): + click.echo(f" cache not present at {p}") + return + cache = ExecCache(p) + s = cache.stats() + click.echo(f" root {p}") + click.echo(f" entries {s['entries']}") + mb = s["bytes"] / (1024 * 1024) + click.echo(f" size {s['bytes']} bytes ({mb:.2f} MiB)") + + +@cache_group.command("clear") +@click.option( + "--root", + type=click.Path(), + default=None, + help="Override the cache root.", +) +@click.confirmation_option( + prompt="Delete every cached task result?", + help="Skip the prompt.", +) +def cache_clear(root: str | None) -> None: + """Delete every cached entry under the cache root.""" + from pathlib import Path + + from pymonik._internal.exec_cache import ExecCache, default_cache_dir + + p = Path(root) if root else default_cache_dir() + if not p.exists(): + click.echo(f" cache not present at {p}") + return + cache = ExecCache(p) + n = cache.clear() + click.echo(f" cleared {n} entries from {p}") + + +if __name__ == "__main__": + cli() diff --git a/src/pymonik/client.py b/src/pymonik/client.py new file mode 100644 index 0000000..23ab34d --- /dev/null +++ b/src/pymonik/client.py @@ -0,0 +1,352 @@ +"""PymonikClient — the connection handle. + +Two front doors, one channel: + +- **Sync**: ``with PymonikClient() as c:`` — spins up a ``BlockingPortal`` + in ``__enter__`` so sync sessions can run async lifecycle hooks on an + asyncio loop hosted on a background thread. Drops the portal in + ``__exit__``. +- **Async**: ``async with PymonikClient() as c:`` — just opens the + channel; async sessions run on the caller's loop. +""" + +from __future__ import annotations + +import os +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any, Optional + +import anyio.from_thread +import grpc +from pymonik._internal._logging import get_logger +import yaml + +from pymonik._internal import _otel +from pymonik._internal.channel import Credentials, open_channel +from pymonik._internal.exec_cache import ExecCache, default_cache_dir +from pymonik._internal.query import ( + PartitionQuery, + ResultQuery, + SessionQuery, + TaskQuery, + _make_context, +) +from pymonik.options import EMPTY, TaskOpts +from pymonik.session import Session + +log = get_logger(__name__) + + +class PymonikClient: + """A connection to an ArmoniK cluster. + + Use as a sync context manager: + + with PymonikClient(endpoint="localhost:5001") as client: + with client.session(partition="pymonik") as s: + ... + + For mTLS: + + from pymonik._internal.channel import Credentials + + creds = Credentials(ca="ca.pem", cert="me.pem", key="me.key") + with PymonikClient(endpoint="grpcs://cluster:5001", credentials=creds) as client: + ... + """ + + def __init__( + self, + endpoint: Optional[str] = None, + *, + credentials: Optional[Credentials] = None, + akconfig: Optional[str | os.PathLike[str]] = None, + events: bool = True, + polling_interval: float = 0.5, + polling_chunk: int = 500, + spill_threshold: int = 256 * 1024, + cache: bool | str | os.PathLike[str] | None = None, + otel: bool | None = None, + otel_service_name: str = "pymonik", + ) -> None: + """Open a client. + + Three ways to configure the endpoint, in order of precedence: + + 1. Pass ``endpoint=`` (and optionally ``credentials=``) explicitly. + 2. Pass ``akconfig=/path/to/armonik-cli.yaml`` to load endpoint + CA + (and optional client cert/key) from a YAML config. + 3. Set the ``AKCONFIG`` env var; the client picks it up automatically. + + Matches the ArmoniK CLI's ``AKCONFIG`` convention so "install, export + AKCONFIG, go" works out of the box. + + ``events=True`` (default) resolves futures via the ``Events.GetEvents`` + server-stream — latency from result-ready to future-resolved is a few + ms. Set ``events=False`` to fall back to the polling loop (one + ``list_results`` RPC every ``polling_interval`` seconds, batched into + chunks of ``polling_chunk`` ids per RPC). Polling is handy if the + events stream misbehaves; events are the right default otherwise. + + ``cache`` enables the on-disk execution cache (see + ``pymonik._internal.exec_cache``). ``None``/``False`` disables it + entirely. ``True`` enables it under the default location + (``~/.cache/pymonik``). A path enables it under that directory. + Per-task opt-in still required: only ``@task(cache=True)`` tasks + actually consult the cache. Without the per-task flag, the + infrastructure is wired but unused. + + ``otel`` controls OpenTelemetry tracing. ``None`` + (default) auto-enables when standard OTel env vars are present + (``OTEL_EXPORTER_OTLP_ENDPOINT`` / ``OTEL_TRACES_EXPORTER``), + otherwise stays off. ``True`` enables unconditionally; ``False`` + forces off. Requires ``pip install pymonik[otel]``. + """ + if endpoint is None: + cfg_path = akconfig or os.getenv("AKCONFIG") + if cfg_path is None: + raise ValueError( + "no endpoint given and no AKCONFIG set. " + "Either pass endpoint=... or export AKCONFIG=/path/to/armonik-cli.yaml." + ) + loaded = _load_akconfig(Path(cfg_path)) + endpoint = loaded["endpoint"] + if credentials is None and loaded.get("certificate_authority"): + credentials = Credentials( + ca=loaded.get("certificate_authority"), + cert=loaded.get("client_certificate"), + key=loaded.get("client_key"), + ) + + self.endpoint = endpoint + self.credentials = credentials + self._events = events + self._polling_interval = polling_interval + self._polling_chunk = polling_chunk + self._spill_threshold = spill_threshold + self._otel_enabled = _otel.setup(force=otel, service_name=otel_service_name) + if self._otel_enabled: + log.info("otel tracing enabled", service=otel_service_name) + self._cache: ExecCache | None + if cache is None or cache is False: + self._cache = None + elif cache is True: + self._cache = ExecCache(default_cache_dir()) + else: + self._cache = ExecCache(Path(cache)) + if self._cache is not None: + log.info("exec cache enabled", root=str(self._cache.root)) + self._channel: grpc.Channel | None = None + self._portal: anyio.from_thread.BlockingPortal | None = None + self._portal_cm: Any = None + + # ---- sync lifecycle ---- + + def __enter__(self) -> "PymonikClient": + self._channel = open_channel(self.endpoint, self.credentials) + # A background asyncio loop lives here for the duration of the client, + # so sync Sessions can drive async completion-loop tasks via the portal. + self._portal_cm = anyio.from_thread.start_blocking_portal(backend="asyncio") + self._portal = self._portal_cm.__enter__() + log.info( + "client connected", + endpoint=self.endpoint, + tls=bool(self.credentials), + mode="sync", + ) + return self + + def __exit__(self, exc_type, exc, tb) -> None: + if self._portal_cm is not None: + try: + self._portal_cm.__exit__(exc_type, exc, tb) + finally: + self._portal_cm = None + self._portal = None + if self._channel is not None: + self._channel.close() + self._channel = None + + def session( + self, + *, + partition: str | list[str] | tuple[str, ...], + default_options: TaskOpts | None = None, + deps: list[str] | tuple[str, ...] | None = None, + isolate: bool | None = None, + index_url: str | None = None, + env: dict[str, str] | None = None, + attach_to: str | None = None, + ) -> Session: + """Open a session bound to one or more partitions (sync). + + Usage:: + + with client.session(partition="pymonik") as s: ... + with client.session(partition=["cpu", "gpu"]) as s: + heavy.with_options(partition="gpu").spawn(...) + + ``partition`` accepts a string (single partition; what most users + want) or a list/tuple of partition ids the session is allowed to + route into. The first element is the default for tasks that don't + explicitly select a partition; ``@task(partition="gpu")`` / + ``.with_options(partition="gpu")`` route to any of the others. + Selecting a partition not in the session's set raises at submit time. + + ``default_options`` sets session-wide task defaults (retries, timeout, + priority, partition override). Merge order: session default ← @task(...) + ← .with_options(...). + + ``deps`` / ``isolate`` / ``index_url`` are sugar for ``default_options`` + — they declare a runtime venv that the worker builds on demand. + ``env`` adds environment variables to the per-task environment along + with ``deps`` (different env values produce a distinct ``env_id`` + and a distinct on-disk venv). + + ``attach_to`` attaches to a pre-existing session id instead of + creating a new one. Tasks submitted in this block land on the + existing session; the events stream picks up completions for + any future this client owns. Exiting the ``with`` block does + **not** ``close_session()`` — the session belongs to whoever + created it. Useful for picking up where another process left + off, or for sharing a session across multiple driver scripts. + ``partition`` is still required (it backstops per-task + validation client-side); it should match what the original + ``create_session`` declared. ``default_options`` is informational + only when attached — the cluster-side defaults were fixed at + create time. + """ + merged = default_options or EMPTY + if ( + deps is not None + or isolate is not None + or index_url is not None + or env is not None + ): + merged = merged.merge( + TaskOpts( + deps=tuple(deps) if deps is not None else None, + isolate=isolate, + index_url=index_url, + env=dict(env) if env is not None else None, + ) + ) + return Session( + self, + partition=partition, + default_options=merged, + use_events=self._events, + polling_interval=self._polling_interval, + polling_chunk=self._polling_chunk, + spill_threshold=self._spill_threshold, + cache=self._cache, + attach_to=attach_to, + ) + + # ---- async lifecycle ---- + + async def __aenter__(self) -> "PymonikClient": + self._channel = open_channel(self.endpoint, self.credentials) + log.info( + "client connected", + endpoint=self.endpoint, + tls=bool(self.credentials), + mode="async", + ) + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + if self._channel is not None: + self._channel.close() + self._channel = None + + # ---- introspection ---- + + def _qctx(self): + if self._channel is None: + raise RuntimeError("client is not connected — open it first") + return _make_context(self._channel) + + @property + def tasks(self) -> TaskQuery: + """All tasks visible to this client. Cluster-wide.""" + return TaskQuery(self._qctx()) + + @property + def sessions(self) -> SessionQuery: + """All sessions visible to this client.""" + return SessionQuery(self._qctx()) + + @property + def results(self) -> ResultQuery: + """All results across the cluster. + + Mutation verbs (``delete()`` / ``download()`` / ``download_to()``) + require a session-scoped query; call from + ``session.results`` instead. + """ + return ResultQuery(self._qctx()) + + @property + def partitions(self) -> PartitionQuery: + """All partitions on the cluster. Read-only.""" + return PartitionQuery(self._qctx()) + + @asynccontextmanager + async def session_async( + self, + *, + partition: str | list[str] | tuple[str, ...], + default_options: TaskOpts | None = None, + deps: list[str] | tuple[str, ...] | None = None, + isolate: bool | None = None, + index_url: str | None = None, + env: dict[str, str] | None = None, + attach_to: str | None = None, + ): + """Open a session bound to one or more partitions (async). + + Mirrors :meth:`session`. See its docstring for ``partition`` / + ``deps`` / ``env`` / ``attach_to`` semantics. + """ + merged = default_options or EMPTY + if ( + deps is not None + or isolate is not None + or index_url is not None + or env is not None + ): + merged = merged.merge( + TaskOpts( + deps=tuple(deps) if deps is not None else None, + isolate=isolate, + index_url=index_url, + env=dict(env) if env is not None else None, + ) + ) + sess = Session( + self, + partition=partition, + default_options=merged, + use_events=self._events, + polling_interval=self._polling_interval, + polling_chunk=self._polling_chunk, + spill_threshold=self._spill_threshold, + cache=self._cache, + attach_to=attach_to, + ) + async with sess: + yield sess + + +def _load_akconfig(path: Path) -> dict[str, str]: + """Parse the ArmoniK CLI's YAML config.""" + with path.open("r") as f: + raw = yaml.safe_load(f) + if not isinstance(raw, dict): + raise ValueError(f"{path}: expected a YAML mapping at the top level") + out = {str(k): str(v) for k, v in raw.items() if v is not None} + if "endpoint" not in out: + raise ValueError(f"{path}: missing 'endpoint' key") + return out diff --git a/src/pymonik/composition.py b/src/pymonik/composition.py new file mode 100644 index 0000000..902a252 --- /dev/null +++ b/src/pymonik/composition.py @@ -0,0 +1,173 @@ +"""High-level fan-in primitives over Futures. + +Mirrors ``asyncio.gather`` / ``asyncio.as_completed`` semantics but operates +on PymoniK ``Future`` / ``FutureList``. Both async and sync forms are +provided; the async form is the canonical one and the sync form is a thin +wrapper for users not in an event loop. + +Inputs are flexible: pass varargs of ``Future``, a single ``FutureList``, +a list/iterable of futures, or any mix. Nested ``FutureList`` containers +are flattened one level (matching their iter protocol). +""" + +from __future__ import annotations + +import asyncio +import time +from typing import TYPE_CHECKING, Any, AsyncIterator, Iterable, Iterator + +from pymonik.future import Future, FutureList + +if TYPE_CHECKING: + pass + + +# Brief poll interval for sync as_completed when no future has resolved yet. +# Trade-off: smaller = more responsive, more CPU. 50 ms is invisible in +# practice and matches typical task latencies. +_AS_COMPLETED_POLL_S = 0.05 + + +def _flatten(items: Iterable[Any]) -> Iterator[Future[Any]]: + """Flatten a mix of Futures, FutureLists, and iterables of either.""" + for item in items: + if isinstance(item, Future): + yield item + elif isinstance(item, FutureList): + yield from item + elif hasattr(item, "__iter__") and not isinstance(item, (str, bytes)): + for sub in item: + if isinstance(sub, Future): + yield sub + elif isinstance(sub, FutureList): + yield from sub + else: + raise TypeError( + f"gather/as_completed: expected Future or FutureList, " + f"got {type(sub).__name__}" + ) + else: + raise TypeError( + f"gather/as_completed: expected Future / FutureList / iterable " + f"of Futures, got {type(item).__name__}" + ) + + +# ---------- async ---------- + +async def gather( + *futures: Any, + return_exceptions: bool = False, + timeout: float | None = None, +) -> list[Any]: + """Wait for every future and return their results in submission order. + + Args: + *futures: ``Future`` objects, a ``FutureList``, or a mix. + return_exceptions: if ``True``, exceptions are returned in-line + instead of raised — same semantics as ``asyncio.gather``. + timeout: per-future deadline; ``None`` waits forever. + + Returns: + A list of results (or exceptions when ``return_exceptions=True``). + """ + flat = list(_flatten(futures)) + coros = [f._await(timeout) for f in flat] + return await asyncio.gather(*coros, return_exceptions=return_exceptions) + + +async def as_completed( + *futures: Any, + timeout: float | None = None, +) -> AsyncIterator[Future[Any]]: + """Yield futures one at a time, in completion order, as they resolve. + + The yielded value is the resolved ``Future`` itself — call ``await + fut`` (or ``fut.result()``) to get its value or re-raise its error. + Mirrors ``asyncio.as_completed`` but yields Futures rather than + awaitables. + """ + flat = list(_flatten(futures)) + if not flat: + return + + # Wrap each Future in an asyncio.Task so we can wait on them as a set. + pending: dict[asyncio.Task[Any], Future[Any]] = { + asyncio.create_task(f._await(timeout)): f for f in flat + } + try: + while pending: + done, _ = await asyncio.wait( + pending.keys(), return_when=asyncio.FIRST_COMPLETED + ) + for d in done: + fut = pending.pop(d) + # Drain the task's exception state so asyncio doesn't warn. + if d.exception() is not None: + pass # the Future already carries the typed error + yield fut + finally: + # Cancel any tasks we didn't get to (caller broke out early). + for t in pending: + t.cancel() + + +# ---------- sync ---------- + +def gather_sync( + *futures: Any, + return_exceptions: bool = False, + timeout: float | None = None, +) -> list[Any]: + """Sync sibling of :func:`gather`. + + Blocks the calling thread until every future resolves; returns results + in submission order. + """ + flat = list(_flatten(futures)) + out: list[Any] = [] + for f in flat: + try: + out.append(f.result(timeout=timeout)) + except Exception as e: + if return_exceptions: + out.append(e) + else: + raise + return out + + +def as_completed_sync( + *futures: Any, + timeout: float | None = None, +) -> Iterator[Future[Any]]: + """Sync sibling of :func:`as_completed`. + + Polls every ~50 ms; suitable for batches up to a few hundred futures. + Above that, prefer the async form (``async for f in as_completed(...)``) + which uses ``asyncio.wait`` and scales without per-iteration polling. + + Args: + *futures: same shapes accepted as :func:`gather`. + timeout: total wall-clock deadline across the iteration; raises + :class:`TaskTimeout` on the first not-yet-done future when + the deadline elapses. + """ + flat = list(_flatten(futures)) + pending: list[Future[Any]] = list(flat) + deadline = None if timeout is None else time.monotonic() + timeout + + while pending: + # Look for any done future first. + for i, f in enumerate(pending): + if f.done: + yield pending.pop(i) + break + else: + # None done yet; either wait briefly or fail on deadline. + if deadline is not None and time.monotonic() >= deadline: + # Yield-then-raise from the next .result() call. + pending[0].result(timeout=0.0) # raises TaskTimeout + # Block on the first pending future for up to the poll + # interval; whoever completes first wakes us up. + pending[0]._done.wait(timeout=_AS_COMPLETED_POLL_S) diff --git a/src/pymonik/context.py b/src/pymonik/context.py new file mode 100644 index 0000000..a2f44a8 --- /dev/null +++ b/src/pymonik/context.py @@ -0,0 +1,137 @@ +"""Worker-side execution context. + +``pymonik.current()`` returns the currently-executing task's context: +structured logger bound with task/session ids, attempt counter, partition, +and a cancellation check. User code calls this *inside* a @task function +to get at worker-side state without polluting the function signature. + +Not available on the client; raises ``RuntimeError`` if called there. + +Cancellation +------------ +The gRPC server context passed into the worker's ``Process`` handler is +captured by ``worker.run`` (via a ``ContextVar``) and stashed here. When +ArmoniK's polling-agent cancels its outgoing gRPC call — the signal the +control plane sends on ``CancelTasks`` / ``CancelSession`` — the context +reports ``is_active() == False`` and +:meth:`WorkerContext.cancel_if_requested` raises :class:`TaskCancelled`. + +Cooperation is on the user: long-running tasks have to call +``pymonik.current().cancel_if_requested()`` at a safe point. A task that +never calls it runs to ``max_duration`` regardless of cluster state. +""" + +from __future__ import annotations + +import contextvars +from typing import TYPE_CHECKING, Any + +from pymonik._internal._logging import get_logger +from pymonik.errors import TaskCancelled + +if TYPE_CHECKING: + from armonik.worker import TaskHandler + + +class WorkerContext: + """What ``pymonik.current()`` returns inside a worker task.""" + + __slots__ = ("_th", "_log", "attempt", "_grpc_context", "_cancel_check") + + def __init__( + self, + task_handler: "TaskHandler", + *, + attempt: int = 1, + grpc_context: Any = None, + cancel_check: Any = None, + ) -> None: + self._th = task_handler + self.attempt = attempt + self._grpc_context = grpc_context + # Optional callable() -> bool. When set, takes precedence over + # grpc_context.is_active() in :meth:`cancelled` — used by + # ``LocalCluster`` to wire cancellation to a local threading.Event + # rather than a real gRPC server context. + self._cancel_check = cancel_check + self._log = get_logger("pymonik.task").bind( + task_id=task_handler.task_id, + session_id=task_handler.session_id, + attempt=attempt, + ) + + @property + def log(self) -> Any: + return self._log + + @property + def task_id(self) -> str: + return self._th.task_id + + @property + def session_id(self) -> str: + return self._th.session_id + + @property + def task_handler(self) -> "TaskHandler": + """Escape hatch for direct armonik TaskHandler access.""" + return self._th + + # ---- cancellation ---- + + @property + def cancelled(self) -> bool: + """``True`` if cancellation has been signalled for this task. + + Non-raising — use inside a boolean condition. See + :meth:`cancel_if_requested` for the raising form. + """ + if self._cancel_check is not None: + try: + return bool(self._cancel_check()) + except Exception: # pragma: no cover — defensive + return False + ctx = self._grpc_context + if ctx is None: + return False + try: + return not ctx.is_active() + except Exception: # pragma: no cover — defensive + return False + + def cancel_if_requested(self) -> None: + """Raise :class:`TaskCancelled` if cancellation has been signalled. + + Call from any point in your @task where it's safe to stop. + """ + if self.cancelled: + raise TaskCancelled(self._th.task_id) + + +_current: contextvars.ContextVar[WorkerContext | None] = contextvars.ContextVar( + "_pymonik_worker_ctx", default=None +) + + +def current() -> WorkerContext: + """Return the context for the currently-executing task. + + Raises: + RuntimeError: if called outside a @task function (e.g. from client code). + """ + ctx = _current.get() + if ctx is None: + raise RuntimeError( + "pymonik.current() called outside a worker task. " + "It is only meaningful inside a @task function running on a worker." + ) + return ctx + + +def _set(ctx: WorkerContext): + """Internal: set the current context and return the token.""" + return _current.set(ctx) + + +def _reset(token) -> None: + _current.reset(token) diff --git a/src/pymonik/envelope.py b/src/pymonik/envelope.py new file mode 100644 index 0000000..edd3840 --- /dev/null +++ b/src/pymonik/envelope.py @@ -0,0 +1,119 @@ +"""Wire envelope. + +One msgspec.Struct carrying two cloudpickle blobs — the function and a +``(args, kwargs)`` tuple — plus typed metadata (env spec, OTel context, +multi-output field names, retry attempt, client Python version). Args +that came in as Future / Blob / Materialize have already been replaced +with ``FutureRef`` / ``BlobRef`` / ``MaterializeRef`` sentinels (see +``_internal/refs.py``); the worker re-walks the unpickled tree and +swaps them for the corresponding ``data_dependencies`` bytes. + +``version`` lets older workers reject envelopes from a newer client +loudly instead of silently mis-decoding. +""" + +from __future__ import annotations + +import sys + +import msgspec + + +ENVELOPE_VERSION = 1 + + +def _current_python() -> str: + v = sys.version_info + return f"{v.major}.{v.minor}" + + +class EnvSpec(msgspec.Struct, frozen=True, kw_only=True): + """Runtime Python environment requested for a task. + + Empty ``deps`` means "no extras" — the worker runs the task in + its own process. Non-empty ``deps`` means the worker creates (or + reuses) a venv at ``/cache/internal/envs/`` and dispatches + the task with that venv on ``sys.path``. + + Default mode is **in-process splice** (``isolate=False``): we add + the venv's site-packages to the worker's ``sys.path`` and call the + function inline. ~1 ms per task once warm, but module imports + persist across tasks on the same pod (they share the worker's + interpreter), so concurrent sessions with *conflicting* deps lists + will collide. Opt in to ``isolate=True`` to spawn a fresh Python + per task — ~400-500 ms each, with full isolation. + + The wire footprint is the deps list itself — strings — never + a lockfile. + """ + + deps: tuple[str, ...] = () + isolate: bool = False + # Optional private index URL for the worker-side ``uv pip install``. + # ``""`` means PyPI default. + index_url: str = "" + # Environment variables applied to the task. Tuple-of-tuples (sorted) + # rather than dict so msgspec can hash a frozen Struct, and so the + # env_id hash is stable. + env: tuple[tuple[str, str], ...] = () + + +class TaskEnvelope(msgspec.Struct, frozen=True, kw_only=True): + """The payload bytes that travel from client to worker. + + Args: + version: Schema version. Workers reject envelopes whose version they + don't recognise. + python: ``major.minor`` version of the client's interpreter. + cloudpickle bytecode is not cross-minor-compatible, so the worker + raises a clear error instead of SIGSEGV'ing mid-unpickle when the + versions differ. + function_pickle: cloudpickle bytes of the user function. + args_pickle: cloudpickle bytes of a ``(args, kwargs)`` tuple. + func_name: Best-effort human-readable name of the function; surfaced in + logs on both sides. Non-authoritative — the function is identified + by its pickle bytes, not by name. + attempt: 1 for the original submission, 2+ for client-side retries. + env_spec: optional runtime environment. ``None`` (the default) means + the worker runs the task with its existing site-packages. + """ + + version: int = ENVELOPE_VERSION + python: str = msgspec.field(default_factory=_current_python) + function_pickle: bytes + args_pickle: bytes + func_name: str = "" + attempt: int = 1 + env_spec: "EnvSpec | None" = None + # W3C trace context propagated from the client (``traceparent``, + # ``tracestate``). Empty when OTel tracing is disabled. Workers + # extract before calling the user function so its spans nest under + # the submitter's. + otel_context: tuple[tuple[str, str], ...] = () + # Sorted field names for multi-output tasks. Empty for single-output + # tasks. The worker zips ``multi_fields`` against the task handler's + # ``expected_results`` to map each MultiResult field to its + # ArmoniK output id. + multi_fields: tuple[str, ...] = () + + +def encode(envelope: TaskEnvelope) -> bytes: + return msgspec.msgpack.encode(envelope) + + +def decode(data: bytes) -> TaskEnvelope: + env = msgspec.msgpack.decode(data, type=TaskEnvelope) + if env.version != ENVELOPE_VERSION: + raise ValueError( + f"incompatible envelope version: got {env.version}, " + f"this worker speaks v{ENVELOPE_VERSION}" + ) + worker_py = _current_python() + if env.python and env.python != worker_py: + raise ValueError( + f"python version mismatch: client sent cloudpickle bytecode from " + f"Python {env.python}, this worker runs Python {worker_py}. " + f"cloudpickle is not cross-minor-compatible; rebuild the worker " + f"image on Python {env.python} or switch the client to Python {worker_py}." + ) + return env diff --git a/src/pymonik/errors.py b/src/pymonik/errors.py new file mode 100644 index 0000000..775540e --- /dev/null +++ b/src/pymonik/errors.py @@ -0,0 +1,43 @@ +"""Typed exception hierarchy. Every error raised by PymoniK inherits from PymonikError.""" + +from __future__ import annotations + + +class PymonikError(Exception): + """Root of the PymoniK exception hierarchy.""" + + +class ConnectionError(PymonikError): + """Failed to reach the ArmoniK control plane.""" + + +class NotInSessionError(PymonikError): + """A task was spawned outside an open session context.""" + + +class TaskFailed(PymonikError): + """The worker raised an exception while executing the task. + + Holds the task_id and the worker-side error message (the traceback, if any). + """ + + def __init__(self, task_id: str, message: str) -> None: + super().__init__(f"task {task_id} failed: {message}") + self.task_id = task_id + self.worker_message = message + + +class TaskCancelled(PymonikError): + """Task was cancelled (session cancel, explicit cancel, or result aborted).""" + + def __init__(self, task_id: str) -> None: + super().__init__(f"task {task_id} cancelled") + self.task_id = task_id + + +class TaskTimeout(PymonikError): + """Task exceeded its max_duration.""" + + def __init__(self, task_id: str) -> None: + super().__init__(f"task {task_id} timed out") + self.task_id = task_id diff --git a/src/pymonik/future.py b/src/pymonik/future.py new file mode 100644 index 0000000..870256f --- /dev/null +++ b/src/pymonik/future.py @@ -0,0 +1,577 @@ +"""Future[T] — a handle to a task's not-yet-arrived result. + +Composes with other tasks *without* blocking the client: pass a Future as +an argument to another ``.spawn()`` and the new task runs with a +``data_dependencies`` edge in ArmoniK. ``.result()`` / ``await`` are only +needed on terminal results. + +Resolution bridge: the completion loop (events stream or polling) runs in +a thread and calls :meth:`_resolve_ok` / :meth:`_resolve_error`. Those set +a ``threading.Event`` (for sync ``.result()``) and, if any awaiter has +registered an ``asyncio.Event`` on this future, wake it via +``loop.call_soon_threadsafe``. +""" + +from __future__ import annotations + +import asyncio +import threading +from typing import TYPE_CHECKING, Any, Generic, TypeVar + +import cloudpickle + +from pymonik._internal import _otel +from pymonik.errors import PymonikError, TaskFailed, TaskTimeout + + +_WORKER_STUB_BLOCK_MSG = ( + "cannot .result() / await a Future from inside a @task — ArmoniK tasks " + "are ephemeral and must not block on other tasks. Pass the Future to " + "another .spawn() (creates a data_dependencies edge so ArmoniK runs the " + "next task once this one completes), or use task.tail(args) to delegate " + "your output to a sub-task." +) + +if TYPE_CHECKING: + from pymonik.session import Session + +T = TypeVar("T") + + +class Future(Generic[T]): + """A handle to the (eventual) result of a spawned task. + + Two wait primitives cooperate: + + - ``_done`` (threading.Event): sync ``.result()`` waits on this. + - ``_aio_done`` (asyncio.Event, lazy): async ``await`` waits on this. + + The completion thread sets both. Clients don't see the split. + """ + + __slots__ = ( + "_session", + "_task_id", + "_result_id", + "_done", + "_aio_done", + "_aio_loop", + "_outcome", + "_error", + # True when the future was created inside a worker via WorkerSession. + # Such futures have no poller, so .result() / await would hang. We + # raise a typed error instead — ArmoniK tasks are ephemeral and + # blocking inside one is illegal. + "_is_worker_stub", + # Client-side retry policy. None when retries aren't configured. + # Tuple shape: (task, args, kwargs, max_retries, on_types, backoff_fn) + # — see Session._submit_many. Read by _resolve_error to decide + # whether to suppress the error and trigger a re-submit. + "_retry_state", + "_retry_attempt", + # Set when this future was spawned with caching enabled. The + # session's resolver writes the cloudpickled result to the + # ExecCache under this key on success. + "_cache_key", + ) + + def __init__(self, session: "Session", task_id: str, result_id: str) -> None: + self._session = session + self._task_id = task_id + self._result_id = result_id + self._done = threading.Event() + # Built lazily on first await so we don't need an event loop just to + # construct the Future. Stored here so threaded resolvers can notify. + self._aio_done: asyncio.Event | None = None + self._aio_loop: asyncio.AbstractEventLoop | None = None + self._outcome: Any = None + self._error: PymonikError | None = None + self._is_worker_stub: bool = False + self._retry_state: Any = None + self._retry_attempt: int = 0 + self._cache_key: str | None = None + + @property + def task_id(self) -> str: + return self._task_id + + @property + def result_id(self) -> str: + return self._result_id + + @property + def done(self) -> bool: + return self._done.is_set() + + # ---- internal: resolved by the session completion loop (thread) ---- + def _resolve_ok(self, raw_bytes: bytes) -> None: + if self._done.is_set(): + return + try: + self._outcome = cloudpickle.loads(raw_bytes) + except Exception as e: + self._error = TaskFailed(self._task_id, f"could not unpickle result: {e!r}") + self._done.set() + self._wake_async() + + def _resolve_error(self, err: PymonikError) -> None: + if self._done.is_set(): + return + # Retry path: if a matching policy is configured and budget remains, + # suppress this error, trigger re-submission, and leave _done unset. + rs = self._retry_state + if rs is not None: + _task, _args, _kwargs, max_retries, on_types, _backoff = rs + if isinstance(err, on_types) and self._retry_attempt < max_retries: + self._retry_attempt += 1 + self._session._schedule_retry(self, attempt=self._retry_attempt) + return + self._error = err + self._done.set() + self._wake_async() + + def _wake_async(self) -> None: + """Called from the completion thread; wake any async awaiter.""" + if self._aio_done is None or self._aio_loop is None: + return + try: + # call_soon_threadsafe is the one asyncio primitive that is safe + # to call from another thread — it schedules .set() on the loop. + self._aio_loop.call_soon_threadsafe(self._aio_done.set) + except RuntimeError: + # Loop has closed before resolution landed; sync waiters unaffected. + pass + + # ---- public sync wait ---- + def result(self, timeout: float | None = None) -> T: + if self._is_worker_stub: + raise PymonikError(_WORKER_STUB_BLOCK_MSG) + with _otel.start_span( + "pymonik.future.wait", + attrs={"pymonik.task_id": self._task_id, "pymonik.mode": "sync"}, + kind="client", + ): + got = self._done.wait(timeout=timeout) + if not got: + raise TaskTimeout(self._task_id) + if self._error is not None: + raise self._error + return self._outcome # type: ignore[no-any-return] + + # ---- public sync wait (no value, no error raise) ---- + def wait(self, timeout: float | None = None) -> "Future[T]": + """Block until this future is resolved (success or failure). Return self. + + Unlike :meth:`result`, this does not raise on task failure / + cancellation and does not deliver the value. Use it when you + want to synchronise without surfacing the value or its errors — + either because you don't need the value yet, or because you + plan to inspect ``.done`` and the future's state explicitly. + + Chains naturally with :meth:`result` for a two-step access: + + value = fut.wait(timeout=60).result() + + Raises :class:`pymonik.TaskTimeout` if the timeout elapses + without resolution. + """ + if self._is_worker_stub: + raise PymonikError(_WORKER_STUB_BLOCK_MSG) + got = self._done.wait(timeout=timeout) + if not got: + raise TaskTimeout(self._task_id) + return self + + # ---- public async wait ---- + async def _await(self, timeout: float | None = None) -> T: + if self._is_worker_stub: + raise PymonikError(_WORKER_STUB_BLOCK_MSG) + # Lazy construction — only pay the cost when someone actually awaits. + if self._aio_done is None: + self._aio_loop = asyncio.get_running_loop() + self._aio_done = asyncio.Event() + # If the result landed before we got here, make sure the event + # is already set so the upcoming wait() returns immediately. + if self._done.is_set(): + self._aio_done.set() + + try: + if timeout is None: + await self._aio_done.wait() + else: + await asyncio.wait_for(self._aio_done.wait(), timeout=timeout) + except asyncio.TimeoutError: + raise TaskTimeout(self._task_id) from None + + if self._error is not None: + raise self._error + return self._outcome # type: ignore[no-any-return] + + async def wait_async(self, timeout: float | None = None) -> "Future[T]": + """Async sibling of :meth:`wait`. Block until resolved; return self. + + Does not surface the value or raise on task failure — same + rationale as :meth:`wait`. + """ + if self._is_worker_stub: + raise PymonikError(_WORKER_STUB_BLOCK_MSG) + if self._aio_done is None: + self._aio_loop = asyncio.get_running_loop() + self._aio_done = asyncio.Event() + if self._done.is_set(): + self._aio_done.set() + try: + if timeout is None: + await self._aio_done.wait() + else: + await asyncio.wait_for(self._aio_done.wait(), timeout=timeout) + except asyncio.TimeoutError: + raise TaskTimeout(self._task_id) from None + return self + + # ---- internal construction (cache hit) ---- + @classmethod + def _new_cached(cls, session: Any, cached_bytes: bytes) -> "Future[Any]": + """Build a Future that's already resolved with ``cached_bytes``. + + The caller (Session / LocalSession) uses this when the local + execution cache hits — no submission, no RPC, the user awaits a + future that immediately yields the cloudpickled value. + + Bypassing ``__init__`` keeps the slot-init tax low and lets us + commit to a state that's already done. + """ + fut: Future[Any] = cls.__new__(cls) + fut._session = session + fut._task_id = "cached" + fut._result_id = "cached" + fut._done = threading.Event() + fut._aio_done = None + fut._aio_loop = None + fut._outcome = None + fut._error = None + fut._is_worker_stub = False + fut._retry_state = None + fut._retry_attempt = 0 + fut._cache_key = None + fut._resolve_ok(cached_bytes) + return fut + + # ---- internal construction (worker-stub) ---- + @classmethod + def _new_worker_stub( + cls, + session: Any, + task_id: str, + result_id: str, + ) -> "Future[Any]": + """Build a Future for a task spawned *inside* a worker. + + No poller exists worker-side, so these futures cannot be awaited. + They're only useful as arguments to further ``.spawn()`` calls + (building data_dependencies edges). + """ + fut: Future[Any] = cls.__new__(cls) + fut._session = session + fut._task_id = task_id + fut._result_id = result_id + fut._done = threading.Event() + fut._aio_done = None + fut._aio_loop = None + fut._outcome = None + fut._error = None + fut._is_worker_stub = True + fut._retry_state = None + fut._retry_attempt = 0 + fut._cache_key = None + return fut + + # ---- cancellation (client side) ---- + def cancel(self) -> None: + """Ask ArmoniK to cancel the task this future points to. + + Fires CancelTasks on the cluster and resolves this future locally + with ``TaskCancelled``. The task may run a bit longer on the worker + before it observes the cancel; the result is discarded either way. + """ + if self._is_worker_stub: + raise PymonikError(_WORKER_STUB_BLOCK_MSG) + if self._done.is_set(): + return + # Session owns the gRPC plumbing. + self._session._cancel_future(self) + + def __await__(self): + return self._await().__await__() + + def __repr__(self) -> str: + state = "done" if self._done.is_set() else "pending" + return f"" + + def _repr_html_(self) -> str: + from pymonik._internal.notebook import future_html + + return future_html(self) + + def _ipython_display_(self) -> None: + from pymonik._internal.notebook import display_live, future_html + + # Worker-stub futures have no poller, so live updates would spin + # forever — fall back to a static paint. + if self._is_worker_stub: + try: + from IPython.display import HTML, display # type: ignore + except Exception: + print(repr(self)) + return + display(HTML(future_html(self))) + return + display_live(self, future_html, futures=[self]) + + +class FutureList(Generic[T]): + """A batch of futures from ``Task.map`` / ``starmap``.""" + + __slots__ = ("_futures",) + + def __init__(self, futures: list[Future[T]]) -> None: + self._futures = futures + + def __iter__(self): + return iter(self._futures) + + def __getitem__(self, idx): + return self._futures[idx] + + def __len__(self) -> int: + return len(self._futures) + + def results(self, timeout: float | None = None) -> list[T]: + """Sync: block until every future resolves; return them in submission order.""" + return [f.result(timeout=timeout) for f in self._futures] + + async def results_async(self, timeout: float | None = None) -> list[T]: + """Async: await every future concurrently; return them in submission order.""" + return await asyncio.gather(*(f._await(timeout) for f in self._futures)) + + def wait(self, timeout: float | None = None) -> "FutureList[T]": + """Block until every future is resolved; return self. + + Doesn't surface values or raise on task failures — see + :meth:`Future.wait`. Apply the timeout per future, same as + :meth:`results`. + """ + for f in self._futures: + f.wait(timeout=timeout) + return self + + async def wait_async(self, timeout: float | None = None) -> "FutureList[T]": + """Async sibling of :meth:`wait`.""" + await asyncio.gather(*(f.wait_async(timeout) for f in self._futures)) + return self + + def __repr__(self) -> str: + done = sum(1 for f in self._futures if f.done) + return f"" + + def _repr_html_(self) -> str: + from pymonik._internal.notebook import future_list_html + + return future_list_html(self) + + def _ipython_display_(self) -> None: + from pymonik._internal.notebook import display_live, future_list_html + + if any(f._is_worker_stub for f in self._futures): + try: + from IPython.display import HTML, display # type: ignore + except Exception: + print(repr(self)) + return + display(HTML(future_list_html(self))) + return + display_live(self, future_list_html, futures=list(self._futures)) + + +class MultiResultView: + """The resolved view of a ``MultiResult``. Supports both attribute + and dict-style access on the named fields:: + + out = split.spawn(7) + view = out.result() + view.double # 14 + view["double"] # 14 + dict(view) # {"double": 14, "triple": 21} + view == {"double": 14, "triple": 21} # True + + Iteration yields field names (matching ``dict``'s default). + """ + + __slots__ = ("_data",) + + def __init__(self, data: dict[str, Any]) -> None: + object.__setattr__(self, "_data", dict(data)) + + def __getattr__(self, name: str) -> Any: + # Reach through to _data; raise AttributeError on miss so + # introspection (hasattr, etc.) behaves correctly. + if name.startswith("_"): + raise AttributeError(name) + try: + return object.__getattribute__(self, "_data")[name] + except KeyError: + raise AttributeError( + f"{name!r} is not a field of this MultiResult " + f"(available: {list(object.__getattribute__(self, '_data'))})" + ) + + def __getitem__(self, key: str) -> Any: + return self._data[key] + + def __iter__(self): + return iter(self._data) + + def __len__(self) -> int: + return len(self._data) + + def __contains__(self, key: object) -> bool: + return key in self._data + + def keys(self): + return self._data.keys() + + def values(self): + return self._data.values() + + def items(self): + return self._data.items() + + def __eq__(self, other: object) -> bool: + if isinstance(other, MultiResultView): + return self._data == other._data + if isinstance(other, dict): + return self._data == other + return NotImplemented + + def __repr__(self) -> str: + parts = ", ".join(f"{k}={v!r}" for k, v in self._data.items()) + return f"MultiResultView({parts})" + + +class MultiResultHandle: + """Handle returned by ``.spawn()`` for a multi-output ``@task``. + + Field access (``handle.field_name``) returns a :class:`Future` for + that one ArmoniK output. Awaiting / ``.result()`` on the handle as + a whole blocks on every field and returns a :class:`MultiResultView`. + + See :class:`pymonik.MultiResult`. + """ + + __slots__ = ("_session", "_task_id", "_field_to_future") + + def __init__( + self, + session: Any, + task_id: str, + field_to_future: dict[str, Future[Any]], + ) -> None: + self._session = session + self._task_id = task_id + self._field_to_future = field_to_future + + @property + def task_id(self) -> str: + return self._task_id + + @property + def fields(self) -> tuple[str, ...]: + return tuple(self._field_to_future.keys()) + + def __getattr__(self, name: str) -> Future[Any]: + # ``__slots__``-bound names are handled by normal attribute access; + # this hook only fires for non-slot attribute reads. + if name.startswith("_"): + raise AttributeError(name) + try: + return object.__getattribute__(self, "_field_to_future")[name] + except KeyError: + fields = ", ".join(object.__getattribute__(self, "_field_to_future")) + raise AttributeError( + f"{name!r} is not a field of this MultiResultHandle " + f"(available: {fields})" + ) + + def __getitem__(self, name: str) -> Future[Any]: + return self._field_to_future[name] + + def __iter__(self): + return iter(self._field_to_future.values()) + + def result(self, timeout: float | None = None) -> MultiResultView: + """Block until every field resolves; return a :class:`MultiResultView`. + + The view supports attribute access (``view.double``) and + dict-style access (``view["double"]``); compares equal to a + plain ``dict`` of the same values. + """ + return MultiResultView( + {f: fut.result(timeout=timeout) for f, fut in self._field_to_future.items()} + ) + + def wait(self, timeout: float | None = None) -> "MultiResultHandle": + """Block until every field is resolved; return self. + + Doesn't surface values or raise on task failures — see + :meth:`Future.wait`. + """ + for fut in self._field_to_future.values(): + fut.wait(timeout=timeout) + return self + + async def _await(self, timeout: float | None = None) -> MultiResultView: + results = await asyncio.gather( + *(fut._await(timeout) for fut in self._field_to_future.values()) + ) + return MultiResultView(dict(zip(self._field_to_future.keys(), results))) + + async def wait_async(self, timeout: float | None = None) -> "MultiResultHandle": + """Async sibling of :meth:`wait`.""" + await asyncio.gather( + *(fut.wait_async(timeout) for fut in self._field_to_future.values()) + ) + return self + + def __await__(self): + return self._await().__await__() + + def cancel(self) -> None: + """Cancel the task that produces all of this handle's outputs. + + ArmoniK's CancelTasks operates per-task; cancelling one field + wouldn't make sense (one task writes all fields). All field + Futures resolve to ``TaskCancelled``. + """ + # All field futures share the same task_id, so we only need one + # cancel_tasks RPC. Issue it via the first non-done future, then + # resolve the rest locally without re-issuing. + from pymonik.errors import TaskCancelled + + any_fut = next(iter(self._field_to_future.values())) + if not any_fut.done: + any_fut.cancel() # one CancelTasks RPC + local resolution + for fut in self._field_to_future.values(): + if not fut.done: + fut._resolve_error(TaskCancelled(fut.task_id)) + + @property + def done(self) -> bool: + return all(fut.done for fut in self._field_to_future.values()) + + def __repr__(self) -> str: + done = sum(1 for f in self._field_to_future.values() if f.done) + n = len(self._field_to_future) + return ( + f"" + ) diff --git a/src/pymonik/multiresult.py b/src/pymonik/multiresult.py new file mode 100644 index 0000000..56e7780 --- /dev/null +++ b/src/pymonik/multiresult.py @@ -0,0 +1,128 @@ +"""Multiple-named-output tasks and lazy tail-call promises. + +Two shapes a ``@task`` body can return: + +- :class:`MultiResult` — a runtime container of named outputs. The + ``@task`` decorator extracts the field set by walking the function's + AST at decoration time, so the framework knows ahead of time how many + ArmoniK ``expected_output_ids`` to allocate for each task. Downstream + consumers depend on individual fields, not on the whole task — fast + fields don't gate slow ones. + +- :class:`TailPromise` — a lazy submission marker returned by + :meth:`pymonik.Task.tail`. The framework decides which output id to + bind it to (the parent's output, or a specific MultiResult field's + output) and submits only when the parent ``@task`` returns. + +Valid uses of a ``TailPromise``: + +- Returned directly from a ``@task`` (whole-task tail-call). +- As a field value inside a returned ``MultiResult`` (per-field + tail-call). + +Anything else — passing a TailPromise to another ``.spawn()``, awaiting +one, storing one and dropping it on the floor — raises a clear error. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Generic, TypeVar + +from pymonik.errors import PymonikError + +if TYPE_CHECKING: + from pymonik.task import Task + +R = TypeVar("R") + + +class TailPromise(Generic[R]): + """A lazy task submission. Bound to an output id by the framework.""" + + __slots__ = ("_task", "_args", "_kwargs") + + def __init__( + self, + task: "Task[Any, R]", + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> None: + self._task = task + self._args = args + self._kwargs = kwargs + + @property + def task(self) -> "Task[Any, R]": + return self._task + + def __await__(self): + raise PymonikError( + "TailPromise cannot be awaited. `task.tail(...)` is for use inside a " + "@task body — return it (whole-task tail-call) or place it as a " + "MultiResult field value (per-field tail-call). To submit and " + "await, use `task.spawn(...)` instead." + ) + + def __repr__(self) -> str: + return f"TailPromise({self._task.name})" + + +class MultiResult: + """A bag of named outputs returned by a multi-output ``@task``. + + Construct in the function body:: + + @task + def split(x: int): + return MultiResult(double=x * 2, triple=x * 3) + + Field values can be plain Python values (cloudpickled and written + by the worker) or :class:`TailPromise` instances (submitted as + delegated child tasks whose outputs land on the field's + ``result_id``). A :class:`pymonik.Future` from ``.spawn()`` is + rejected — use ``.tail()`` for delegation. + + The framework reads the field set from this constructor's keyword + arguments via AST analysis at ``@task`` decoration time. Every + ``MultiResult(...)`` literal in the function body must use the + same field names; otherwise the decoration raises. + """ + + __slots__ = ("_fields",) + + # Names that ``MultiResultHandle`` exposes as properties or methods. + # A field with one of these names would be shadowed by attribute + # lookup on the handle (e.g. ``out.result()`` would call the handle + # method, not return the field Future). Reject at construction. + _RESERVED_FIELD_NAMES = frozenset( + {"task_id", "fields", "done", "result", "cancel"} + ) + + def __init__(self, **fields: Any) -> None: + if not fields: + raise PymonikError( + "MultiResult requires at least one field" + ) + for name in fields: + if name.startswith("_"): + raise PymonikError( + f"MultiResult field name {name!r} is invalid: " + f"underscore-prefixed names are reserved." + ) + if name in MultiResult._RESERVED_FIELD_NAMES: + raise PymonikError( + f"MultiResult field name {name!r} collides with a " + f"MultiResultHandle attribute. Reserved names: " + f"{sorted(MultiResult._RESERVED_FIELD_NAMES)}." + ) + # Avoid __setattr__ collision if we ever add @dataclass-like behaviour. + object.__setattr__(self, "_fields", dict(fields)) + + @property + def fields(self) -> dict[str, Any]: + """The field mapping. Values may be plain or ``TailPromise``s.""" + return self._fields + + def __repr__(self) -> str: + parts = ", ".join(f"{k}={v!r}" for k, v in self._fields.items()) + return f"MultiResult({parts})" diff --git a/src/pymonik/options.py b/src/pymonik/options.py new file mode 100644 index 0000000..a85d190 --- /dev/null +++ b/src/pymonik/options.py @@ -0,0 +1,174 @@ +"""Task-level options with sane merge semantics. + +User-facing fields use Pythonic names (``partition``, ``retries``, ``timeout``, +``priority``) and translate to ``armonik.common.TaskOptions`` at submission +time. Merge order at the call site is: + + session default ← @task(...) ← .with_options(...) + +``None`` means "inherit"; a concrete value means "override". + +Retry semantics +--------------- +``retries=N`` alone → ArmoniK ``max_retries=N`` (cluster-side, blanket +retries for infra failures and user-code errors alike) — what most +users want. + +When ``retry_on=(SomeError, ...)`` is also set, ``retries`` becomes the +*client-side* retry budget — the SDK observes the failure type, optionally +sleeps a backoff, and re-spawns. Cluster ``max_retries`` is held at the +default 2 in that case (still covers infra crashes), and `retries` no +longer leaks into the per-task `TaskOptions` sent to ArmoniK. + +This split lets users have either "blind retry on the cluster" (cheap, +no filtering) or "selective retry on the client" (filterable, with +backoff) without two competing knobs. +""" + +from __future__ import annotations + +from dataclasses import dataclass, fields, replace +from datetime import timedelta +from typing import Callable, Optional, Tuple, Type, Union + +from armonik.common import TaskOptions + +_TimeoutLike = Union[timedelta, float, int] +BackoffSpec = Union[str, float, int, Callable[[int], float], None] + + +def _as_timedelta(value: Optional[_TimeoutLike]) -> Optional[timedelta]: + if value is None: + return None + if isinstance(value, timedelta): + return value + return timedelta(seconds=float(value)) + + +def _exponential(attempt: int) -> float: + # 0.5, 1.0, 2.0, 4.0, ...; capped at 30 s to avoid runaway delays. + return min(30.0, 0.5 * (2**attempt)) + + +def _linear(attempt: int) -> float: + return 0.5 * (attempt + 1) + + +def _constant(_attempt: int) -> float: + return 1.0 + + +def resolve_backoff(spec: BackoffSpec) -> Callable[[int], float]: + """Turn a user-provided backoff spec into a callable ``attempt -> seconds``. + + ``attempt`` is 0 for the *first* retry, 1 for the second, etc. + """ + if spec is None or spec == "exponential": + return _exponential + if callable(spec): + return spec + if isinstance(spec, (int, float)): + seconds = float(spec) + return lambda _attempt: seconds + if isinstance(spec, str): + if spec == "linear": + return _linear + if spec == "constant": + return _constant + raise ValueError( + f"unknown backoff spec {spec!r}; expected 'exponential' / 'linear' / " + f"'constant', a number of seconds, or a callable(attempt) -> seconds" + ) + + +@dataclass(frozen=True, slots=True, kw_only=True) +class TaskOpts: + """Python-ergonomic view of ``armonik.common.TaskOptions`` plus client-side + retry knobs. + + All fields are optional. ``merge(self, other)`` returns a new TaskOpts + where ``other``'s non-None fields override ``self``'s. + """ + + partition: Optional[str] = None + retries: Optional[int] = None + timeout: Optional[_TimeoutLike] = None + priority: Optional[int] = None + # Client-side retry: tuple of exception types that should be retried. + # When set, ``retries`` becomes the client retry budget (default 3 if + # not specified) and cluster-side max_retries is pinned to 2. + retry_on: Optional[Tuple[Type[BaseException], ...]] = None + # Backoff strategy between client-side retries. See ``resolve_backoff``. + retry_backoff: BackoffSpec = None + # Local execution cache opt-in. None inherits (effectively off — the + # client-level cache is opt-in *infrastructure*; per-task ``cache=True`` + # opts that task in). True = cache this task; False = don't (overrides + # any ambient default). + cache: Optional[bool] = None + # Runtime pip dependencies. List of PEP-508 specifiers (e.g. + # ``("numpy>=2", "polars")``). Hashed into an env_id; the worker + # builds (or reuses) a venv per env_id and runs the task against it. + # Empty / None = no extra deps (worker uses its own site-packages). + deps: Optional[Tuple[str, ...]] = None + # When ``deps`` is non-empty: False (default) splices the venv's + # site-packages into the worker process via ``sys.path`` — ~1 ms + # per task once warm, but module state leaks across tasks on the + # same pod. True spawns a fresh Python per task against the env's + # venv (full isolation, ~400-500 ms startup with a numpy-class dep). + isolate: Optional[bool] = None + # Optional private PyPI-style index for the worker's ``uv pip install``. + index_url: Optional[str] = None + # Per-task environment variables applied alongside ``deps``. Merged + # key-wise. Different env values produce a distinct ``env_id`` so two + # sessions with the same deps but different env vars do NOT share the + # on-disk venv (deliberate — env vars often select install behaviour, + # CUDA build, private index credentials, etc.). + env: Optional[dict[str, str]] = None + # Free-form string map handed through to TaskOptions.options; used for + # things like OTel trace context and application tags. Merged key-wise. + options: Optional[dict[str, str]] = None + + def merge(self, other: "TaskOpts") -> "TaskOpts": + patch: dict = {} + for f in fields(self): + v = getattr(other, f.name) + if v is None: + continue + if f.name == "options": + merged = dict(self.options or {}) + merged.update(v) + patch["options"] = merged + elif f.name == "env": + # key-wise merge: per-task env adds to/overrides session env + merged = dict(self.env or {}) + merged.update(v) + patch["env"] = merged + else: + patch[f.name] = v + return replace(self, **patch) + + def to_armonik(self, *, default_partition: str) -> TaskOptions: + """Build an armonik.common.TaskOptions for submission. + + ``default_partition`` backstops ``partition`` — ArmoniK rejects a + TaskOptions without a partition_id, and a task without an explicit + partition should run on the session's default. + + When ``retry_on`` is set, ``max_retries`` is fixed at 2 (covers + cluster infra failures only) — the client owns the application + retry loop. Otherwise ``retries`` flows straight through. + """ + if self.retry_on is not None: + max_retries = 2 + else: + max_retries = self.retries if self.retries is not None else 2 + return TaskOptions( + max_duration=_as_timedelta(self.timeout) or timedelta(minutes=10), + priority=self.priority if self.priority is not None else 1, + max_retries=max_retries, + partition_id=self.partition or default_partition, + options=dict(self.options) if self.options else {}, + ) + + +EMPTY = TaskOpts() diff --git a/src/pymonik/session.py b/src/pymonik/session.py new file mode 100644 index 0000000..948f2dc --- /dev/null +++ b/src/pymonik/session.py @@ -0,0 +1,834 @@ +"""Session — unit of work against an ArmoniK cluster (client-side). + +Opens an ArmoniK session, owns the default task options, registers +in-flight futures, and runs a background poller thread that resolves +futures as results complete. + +Submission, retry, and re-submission all go through the shared pipeline +in :mod:`pymonik._internal.submit`. This module's job is the +control-plane lifecycle (create / close / cancel session, run the +events stream, turn aborted results into typed errors) and the +plumbing that wires the pipeline to ArmoniK's gRPC clients. +""" + +from __future__ import annotations + +import threading +import time +from typing import TYPE_CHECKING, Any + +import anyio +import grpc +from pymonik._internal._logging import get_logger +from armonik.client import ArmoniKEvents, ArmoniKResults, ArmoniKSessions, ArmoniKTasks +from armonik.common import EventTypes, Result, ResultStatus, TaskDefinition, TaskOptions +from armonik.common.events import ResultStatus as EvResultStatus +from armonik.common.events import ResultStatusUpdateEvent + +import hashlib + +import cloudpickle + +from pymonik import blob as blob_mod +from pymonik._internal import _otel +from pymonik._internal.exec_cache import ExecCache, compute_cache_key +from pymonik._internal.query import ( + ResultQuery, + TaskQuery, + _make_context, +) +from pymonik._internal.submit import SubmissionBackend, normalise_calls, submit_many +from pymonik.errors import TaskCancelled, TaskFailed +from pymonik.future import Future, FutureList +from pymonik.options import EMPTY, TaskOpts +from pymonik.task import Task, _current_session + +if TYPE_CHECKING: + from pymonik.client import PymonikClient + +log = get_logger(__name__) + +# How often the session poller scans for completed results (seconds). +_POLL_INTERVAL = 0.5 + +# Maximum pending result_ids to pack into a single ``list_results`` RPC. +# The filter is an OR chain over ``Result.result_id == `` predicates; +# once ``len(pending_ids)`` gets into the thousands the serialised request +# approaches gRPC's default 4 MiB cap. Chunk to stay well under. +_POLL_CHUNK = 500 + +# Default auto-spill threshold: args cloudpickled larger than this are +# uploaded as blobs and passed via data_dependencies. Chosen so tiny +# collections stay inline and typical numpy arrays / large dicts spill +# before they hit the gRPC default 4 MiB message cap. +_DEFAULT_SPILL_THRESHOLD = 256 * 1024 + + +class Session: + """An open ArmoniK session bound to a specific partition.""" + + def __init__( + self, + client: "PymonikClient", + partition: str | list[str] | tuple[str, ...], + default_options: TaskOpts = EMPTY, + *, + use_events: bool = True, + polling_interval: float = _POLL_INTERVAL, + polling_chunk: int = _POLL_CHUNK, + spill_threshold: int = _DEFAULT_SPILL_THRESHOLD, + cache: ExecCache | None = None, + attach_to: str | None = None, + ) -> None: + self._client = client + # Normalise partitions: first element is the default; the full + # list is what the session advertises to ArmoniK on create. When + # attaching to an existing session, the partition list is + # informational (used only for client-side per-task partition + # validation); the cluster-side declaration was done on create. + if isinstance(partition, str): + self._partitions: tuple[str, ...] = (partition,) + else: + parts = tuple(partition) + if not parts: + raise ValueError("partition list cannot be empty") + self._partitions = parts + self._partition = self._partitions[0] + self._default_opts = default_options + self._use_events = use_events + self._polling_interval = polling_interval + self._polling_chunk = polling_chunk + self._spill_threshold = spill_threshold + self._cache = cache + # Existing session id we're attaching to. None = create a fresh + # session on open. When attached, ``__exit__`` doesn't issue + # ``close_session()`` — other consumers may still be using the + # session and aren't ours to terminate. + self._attach_to = attach_to + + self._session_id: str | None = None + self._sessions: ArmoniKSessions | None = None + self._tasks: ArmoniKTasks | None = None + self._results: ArmoniKResults | None = None + self._events: ArmoniKEvents | None = None + + self._pending: dict[str, Future[Any]] = {} # result_id -> future + # Within-session content-addressable blob cache. Keyed by SHA-256 + # hex of the bytes; value is the result_id. + self._blob_cache: dict[str, str] = {} + self._lock = threading.Lock() + self._stop = threading.Event() + self._runner: threading.Thread | None = None + self._ctx_token: Any = None + # Set after ``cancel()``: the cluster has already terminated the + # session, so we skip ``close_session()`` in ``__exit__`` to avoid + # a noisy warning about "state that cannot be closed". + self._cancelled: bool = False + + @property + def session_id(self) -> str: + if self._session_id is None: + raise RuntimeError("session is not open") + return self._session_id + + @property + def partition(self) -> str: + """The session's *default* partition (first of the partition list).""" + return self._partition + + @property + def partitions(self) -> tuple[str, ...]: + """All partitions this session can route into.""" + return self._partitions + + # ---- introspection (session-scoped) ---- + + def _qctx(self): + if self._client._channel is None: + raise RuntimeError("session is not open") + return _make_context( + self._client._channel, + scoped_session_id=self.session_id, + ) + + @property + def tasks(self) -> TaskQuery: + """Tasks in this session (auto-scoped via ``session_id``).""" + return TaskQuery(self._qctx()) + + @property + def results(self) -> ResultQuery: + """Results in this session. + + Mutation verbs (``delete()`` / ``download()`` / ``download_to()``) + operate on this session. + """ + return ResultQuery(self._qctx()) + + # ---- lifecycle (shared between sync and async) ---- + + def _open_resources(self) -> None: + """Blocking setup: armonik clients, session creation, completion thread. + + Called from sync ``__enter__`` directly, and from async + ``__aenter__`` via ``anyio.to_thread.run_sync`` so the event loop + isn't blocked on gRPC. + """ + channel = self._client._channel + assert channel is not None, "client channel is not open" + self._sessions = ArmoniKSessions(channel) + self._tasks = ArmoniKTasks(channel) + self._results = ArmoniKResults(channel) + self._events = ArmoniKEvents(channel) + + if self._attach_to is not None: + # Attaching: skip create_session, trust the user-supplied id. + # We don't validate the id exists up front — the first + # submission RPC will fail clearly enough if it doesn't. + self._session_id = self._attach_to + log.info( + "session attached", + session_id=self._session_id, + partitions=list(self._partitions), + completion="events" if self._use_events else "poll", + ) + else: + default_armonik = self._default_opts.to_armonik(default_partition=self._partition) + with _otel.start_span( + "pymonik.session.open", + attrs={ + "pymonik.partitions": ",".join(self._partitions), + "pymonik.completion": "events" if self._use_events else "poll", + }, + kind="client", + ) as span: + self._session_id = self._sessions.create_session( + default_task_options=default_armonik, + partition_ids=list(self._partitions), + ) + if span is not None: + span.set_attribute("pymonik.session_id", self._session_id) + log.info( + "session opened", + session_id=self._session_id, + partitions=list(self._partitions), + completion="events" if self._use_events else "poll", + ) + + target = self._events_loop if self._use_events else self._poll_loop + self._runner = threading.Thread( + target=target, + name=f"pymonik-{self._session_id}", + daemon=True, + ) + self._runner.start() + + def _close_resources(self) -> None: + """Blocking teardown: stop thread, fail pending, close session.""" + self._stop.set() + if self._runner is not None and self._runner.is_alive(): + # The events stream is blocking on a server-push; closing the + # session below (or the channel on client exit) breaks it out. + self._runner.join(timeout=2.0) + + with self._lock: + pending = list(self._pending.values()) + self._pending.clear() + for fut in pending: + fut._resolve_error(TaskCancelled(fut.task_id)) + + # Don't close the session on exit when we attached — it isn't + # ours to terminate. Other consumers may still be using it. + if ( + self._session_id + and self._sessions + and not self._cancelled + and self._attach_to is None + ): + try: + self._sessions.close_session(self._session_id) + except Exception as e: + log.warning("close_session failed", error=str(e)) + + # ---- context manager (sync) ---- + + def __enter__(self) -> "Session": + self._open_resources() + self._ctx_token = _current_session.set(self) + return self + + def __exit__(self, exc_type, exc, tb) -> None: + try: + self._close_resources() + finally: + if self._ctx_token is not None: + _current_session.reset(self._ctx_token) + self._ctx_token = None + + # ---- context manager (async) ---- + + async def __aenter__(self) -> "Session": + # gRPC calls are blocking — run on a worker thread so we don't stall + # the event loop while the control plane creates our session. + await anyio.to_thread.run_sync(self._open_resources) + self._ctx_token = _current_session.set(self) + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + try: + await anyio.to_thread.run_sync(self._close_resources) + finally: + if self._ctx_token is not None: + _current_session.reset(self._ctx_token) + self._ctx_token = None + + # ---- client-side retry ---- + + def _schedule_retry(self, fut: Future[Any], *, attempt: int) -> None: + """Sleep the backoff, then re-spawn the task underlying ``fut``. + + Runs on a daemon thread so the events / poll loop isn't blocked + while we wait. Updates ``fut`` in place — the user's reference is + rewired to the new task / result_id atomically. + """ + rs = fut._retry_state + assert rs is not None + task, args, kwargs, _max_retries, _on_types, backoff_fn = rs + delay = max(0.0, float(backoff_fn(attempt - 1))) + log.info( + "task retrying", + task=task.name, + attempt=attempt, + delay_s=round(delay, 3), + old_task_id=fut.task_id, + ) + + def _run(): + try: + if delay > 0.0: + if self._stop.wait(timeout=delay): + # Session was torn down during backoff. + return + self._resubmit_for_retry(fut, task, args, kwargs) + except Exception as e: + # If the resubmit itself blows up, fail the public future + # so the user sees a real error instead of hanging. + log.error("retry resubmit failed", task=task.name, error=str(e)) + fut._error = TaskFailed(fut.task_id, f"retry resubmit failed: {e!r}") + fut._done.set() + fut._wake_async() + + threading.Thread( + target=_run, + name=f"pymonik-retry-{fut.task_id[:8]}", + daemon=True, + ).start() + + def _resubmit_for_retry( + self, + fut: Future[Any], + task: Task[Any, Any], + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> None: + """Re-submit one task and rewire ``fut`` to the new task / output. + + Goes through the same shared pipeline as :meth:`_submit_many`, + with ``existing_future=fut`` so the user-visible Future is mutated + in place (no new object). The retry state is preserved so further + failures can retry again until the budget is exhausted. + """ + backend = _ClientBackend(self) + + def make_future(*_a, **_k) -> Future[Any]: # unused for the retry path + raise AssertionError("future_factory should not be called when existing_future is set") + + def register(output_ids: list[str], registered_fut: Any) -> None: + with self._lock: + # Old result_id was already popped from _pending by + # _resolve_result before our error handler ran; just + # register the new one. Retry path is single-output. + self._pending[output_ids[0]] = registered_fut + + submit_many( + task=task, + calls=[(args, kwargs)], + backend=backend, + blob_uploader=self._upload_blob, + spill_threshold=self._spill_threshold, + default_opts=self._default_opts, + partition=self._partition, + future_factory=make_future, + on_submitted=register, + apply_retry_policy=False, # already on the future from the original submit + existing_future=fut, + attempt=fut._retry_attempt + 1, + ) + log.info( + "task retried", + task=task.name, + attempt=fut._retry_attempt, + new_task_id=fut.task_id, + ) + + # ---- cancellation ---- + + def _cancel_future(self, fut: Future[Any]) -> None: + """Cancel one task on the cluster and resolve its future locally. + + Called by :meth:`Future.cancel`. Uses ArmoniK's ``CancelTasks``. + + Ordering matters: resolve locally **before** issuing the RPC. + Otherwise the events stream can deliver a ``RESULT_STATUS_UPDATE`` + with ``ABORTED`` while we're still round-tripping to the control + plane, beating us to ``_resolve_error`` and leaving the future + with ``TaskFailed("result aborted")`` instead of ``TaskCancelled``. + """ + assert self._tasks is not None + with self._lock: + self._pending.pop(fut.result_id, None) + fut._resolve_error(TaskCancelled(fut.task_id)) + try: + self._tasks.cancel_tasks(task_ids=[fut.task_id]) + except Exception as e: + log.warning("cancel_tasks failed", task_id=fut.task_id, error=str(e)) + + def cancel(self) -> None: + """Cancel this session and every in-flight task it holds. + + Marks every pending future as :class:`TaskCancelled` locally so + callers blocking on them wake up; the cluster finishes the wind-down + asynchronously. Same race-with-events-stream ordering as + :meth:`_cancel_future` — resolve first, then RPC. + """ + assert self._sessions is not None + # Flip the flag *under the lock*, before we resolve the futures. + # Otherwise the main thread wakes on its .result() before we get to + # `self._cancelled = True` and hits ``__exit__`` → ``close_session`` + # while _cancelled is still False (race with cancel_session). + with self._lock: + pending = list(self._pending.values()) + self._pending.clear() + self._cancelled = True + for fut in pending: + fut._resolve_error(TaskCancelled(fut.task_id)) + try: + self._sessions.cancel_session(self.session_id) + except Exception as e: + log.warning("cancel_session failed", error=str(e)) + log.info("session cancelled", session_id=self.session_id, cancelled_count=len(pending)) + + def pause(self) -> None: + """Pause submissions on this session (Pause RPC). + + New tasks can't be submitted while paused. In-flight tasks + continue. Call :meth:`resume` to lift the pause. + """ + assert self._sessions is not None + self._sessions.pause_session(self.session_id) + log.info("session paused", session_id=self.session_id) + + def resume(self) -> None: + """Resume submissions after a previous :meth:`pause`.""" + assert self._sessions is not None + self._sessions.resume_session(self.session_id) + log.info("session resumed", session_id=self.session_id) + + def stop_submission(self, *, client: bool = True, worker: bool = True) -> None: + """Block further submissions on this session. + + ``client=True`` blocks user clients; ``worker=True`` blocks + sub-task spawns from inside running tasks. Both default to True + (full freeze). Unlike :meth:`pause` this is one-way: you can't + un-stop submissions, only cancel and re-create the session. + """ + assert self._sessions is not None + self._sessions.stop_submission_session( + session_id=self.session_id, client=client, worker=worker + ) + log.info( + "session submissions stopped", + session_id=self.session_id, + client=client, + worker=worker, + ) + + # ---- blob upload (content-addressable, within-session dedup) ---- + + def _upload_blob(self, data: bytes) -> str: + """Upload ``data`` (cloudpickled object bytes or raw file bytes). + + Deduplicates within the session by SHA-256 content hash — passing + the same bytes twice returns the same result_id and skips the + network round-trip. + """ + assert self._results is not None + h = blob_mod.content_hash(data) + with self._lock: + cached = self._blob_cache.get(h) + if cached is not None: + return cached + + with _otel.start_span( + "pymonik.blob.upload", + attrs={"pymonik.bytes": len(data), "pymonik.hash": h[:16]}, + kind="client", + ): + name = f"{self.session_id}__blob__{h[:16]}" + result_map = self._results.create_results( + results_data={name: data}, + session_id=self.session_id, + ) + rid = result_map[name].result_id + with self._lock: + cached2 = self._blob_cache.get(h) + if cached2 is not None: + return cached2 + self._blob_cache[h] = rid + log.info("blob uploaded", hash=h[:16], size=len(data), result_id=rid) + return rid + + # ---- submission ---- + + def _submit_one( + self, + task: Task[Any, Any], + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Future[Any]: + futures = self._submit_many(task, [(args, kwargs)]) + return futures[0] + + def _submit_many( + self, + task: Task[Any, Any], + calls: list[Any], + ) -> FutureList[Any]: + """Submit N invocations of one task via the shared pipeline. + + Cache-active calls (``self._cache is not None and + task.opts.cache is True``) short-circuit on hit: a pre-resolved + Future is returned without any RPC. Misses go through the normal + pipeline and are tagged with their cache key so + :meth:`_resolve_result` can write them back when they land. + """ + normalised = normalise_calls(calls) + + # Cache filter pass. + cached_hits, miss_idxs, keys = self._cache_classify(task, normalised) + + # Submit only the misses. + miss_calls = [normalised[i] for i in miss_idxs] + if miss_calls: + miss_futures = self._submit_through_pipeline(task, miss_calls) + else: + miss_futures = FutureList([]) + + # Stitch back to original order; tag misses with their cache key. + out: list[Future[Any]] = [None] * len(normalised) # type: ignore[list-item] + for i, raw in cached_hits.items(): + out[i] = Future._new_cached(self, raw) + for j, idx in enumerate(miss_idxs): + fut = miss_futures[j] + if idx in keys: + fut._cache_key = keys[idx] + out[idx] = fut + + # If any calls were uncacheable but task was cache-eligible, log once. + if self._cache is not None and task.opts.cache is True: + n_skip = sum( + 1 + for i in range(len(normalised)) + if i not in cached_hits and i not in keys + ) + if n_skip: + log.debug( + "cache skipped (uncacheable args)", + task=task.name, + skipped=n_skip, + ) + + return FutureList(out) + + def _cache_classify( + self, + task: Task[Any, Any], + normalised: list[tuple[tuple[Any, ...], dict[str, Any]]], + ) -> tuple[dict[int, bytes], list[int], dict[int, str]]: + """Decide hit / miss / uncacheable for each call. + + Returns ``(cached_hits, miss_idxs, keys)`` where: + - ``cached_hits``: ``{idx: cloudpickled_bytes}`` for entries + that already exist on disk. + - ``miss_idxs``: indices that need to go through submission + (a superset of ``keys`` keys). + - ``keys``: ``{idx: cache_key}`` for misses we want to write + back to the cache when their result arrives. Indices with no + entry are uncacheable (Future args, unpicklable values). + """ + if self._cache is None or task.opts.cache is not True: + return {}, list(range(len(normalised))), {} + + import pymonik + + fn_pickle_hash = hashlib.sha256(cloudpickle.dumps(task.func)).digest() + cached_hits: dict[int, bytes] = {} + miss_idxs: list[int] = [] + keys: dict[int, str] = {} + + for i, (args, kwargs) in enumerate(normalised): + key = compute_cache_key( + pymonik_version=pymonik.__version__, + task_name=task.name, + function_pickle_hash=fn_pickle_hash, + args=args, + kwargs=kwargs, + ) + if key is None: + miss_idxs.append(i) + continue + try: + raw = self._cache.get_bytes(key) + cached_hits[i] = raw + log.info("cache hit", task=task.name, key=key[:16]) + except KeyError: + miss_idxs.append(i) + keys[i] = key + if cached_hits: + log.info( + "cache batch summary", + task=task.name, + hits=len(cached_hits), + misses=len(miss_idxs), + ) + return cached_hits, miss_idxs, keys + + def _submit_through_pipeline( + self, + task: Task[Any, Any], + calls: list[tuple[tuple[Any, ...], dict[str, Any]]], + ) -> FutureList[Any]: + """Hand a (post-cache-filter) list of calls to the shared pipeline.""" + from pymonik.future import MultiResultHandle + + backend = _ClientBackend(self) + multi_fields = task.multi_fields + + def make_future( + task_id: str, + output_ids: list[str], + _args: tuple[Any, ...], + _kwargs: dict[str, Any], + ) -> Any: + if multi_fields: + field_to_future = { + field: Future(self, task_id=task_id, result_id=oid) + for field, oid in zip(multi_fields, output_ids) + } + return MultiResultHandle(self, task_id, field_to_future) + return Future(self, task_id=task_id, result_id=output_ids[0]) + + def register(output_ids: list[str], handle: Any) -> None: + with self._lock: + if multi_fields: + # Each output id keys into the same handle; the + # completion loop resolves whichever field's + # output id arrives. The MultiResultHandle holds + # the per-field Futures. + for field, oid in zip(multi_fields, output_ids): + self._pending[oid] = handle._field_to_future[field] + else: + self._pending[output_ids[0]] = handle + + return submit_many( + task=task, + calls=calls, + backend=backend, + blob_uploader=self._upload_blob, + spill_threshold=self._spill_threshold, + default_opts=self._default_opts, + partition=self._partition, + future_factory=make_future, + on_submitted=register, + apply_retry_policy=True, + attempt=1, + ) + + # ---- events stream (default) ---- + + def _events_loop(self) -> None: + """Run the ``Events.GetEvents`` server-stream and resolve futures. + + Stops when ``self._stop`` is set (on the next event) or when the + stream errors out (channel close during shutdown). + """ + assert self._events is not None + + def handler(_session_id, event_type, event) -> bool: + if self._stop.is_set(): + return True # break the stream + if ( + event_type == EventTypes.RESULT_STATUS_UPDATE + and isinstance(event, ResultStatusUpdateEvent) + ): + with self._lock: + known = event.result_id in self._pending + if not known: + return False + if event.status == EvResultStatus.COMPLETED: + self._resolve_result(event.result_id, ok=True) + elif event.status == EvResultStatus.ABORTED: + self._resolve_result(event.result_id, ok=False) + return False + + try: + self._events.get_events( + session_id=self.session_id, + event_types=[EventTypes.RESULT_STATUS_UPDATE], + event_handlers=[handler], + ) + except grpc.RpcError as e: + if not self._stop.is_set(): + log.warning("events stream terminated", error=str(e)) + except Exception as e: # noqa: BLE001 — bg thread; surface for the user + log.error("events loop failure", error=str(e)) + + # ---- polling fallback ---- + + def _poll_loop(self) -> None: + while not self._stop.is_set(): + if self._stop.wait(timeout=self._polling_interval): + return + try: + self._poll_once() + except Exception as e: + log.warning("poll iteration failed", error=str(e)) + + def _poll_once(self) -> None: + with self._lock: + pending_ids = list(self._pending.keys()) + if not pending_ids: + return + for i in range(0, len(pending_ids), self._polling_chunk): + self._poll_chunk(pending_ids[i : i + self._polling_chunk]) + + def _poll_chunk(self, chunk_ids: list[str]) -> None: + assert self._results is not None + filt = None + for rid in chunk_ids: + cond = Result.result_id == rid + filt = cond if filt is None else filt | cond + + _total, items = self._results.list_results( + result_filter=filt, + page=0, + page_size=len(chunk_ids), + ) + + for res in items: + if res.status == ResultStatus.COMPLETED: + self._resolve_result(res.result_id, ok=True) + elif res.status == ResultStatus.ABORTED: + self._resolve_result(res.result_id, ok=False) + + def _resolve_result(self, result_id: str, *, ok: bool) -> None: + assert self._results is not None + assert self._tasks is not None + session_id = self.session_id + + with self._lock: + fut = self._pending.pop(result_id, None) + if fut is None: + return + + if not ok: + # Extra RPC: fetch the task's worker-side error output so the + # user sees something more useful than "result aborted". Failures + # are the exception path; the extra call is worth the UX. + msg = "result aborted" + try: + t = self._tasks.get_task(fut.task_id) + if t.output is not None and getattr(t.output, "error", None): + msg = t.output.error + elif getattr(t, "status_message", None): + msg = t.status_message + except Exception as e: + log.debug("get_task for error details failed", error=str(e)) + fut._resolve_error(TaskFailed(fut.task_id, msg)) + return + + try: + data = self._results.download_result_data( + result_id=result_id, + session_id=session_id, + ) + except Exception as e: + fut._resolve_error(TaskFailed(fut.task_id, f"download failed: {e!r}")) + return + + # Cache write FIRST so that by the time fut._resolve_ok wakes + # the user's .result() / await, the entry is already on disk. + # Otherwise an immediate re-spawn could race the events-thread + # writer and miss the cache it should have hit. Failure path is + # never cached. + if self._cache is not None and fut._cache_key is not None: + try: + self._cache.put_bytes(fut._cache_key, data) + log.info( + "cache stored", + task_id=fut.task_id, + key=fut._cache_key[:16], + bytes=len(data), + ) + except Exception as e: # noqa: BLE001 — never block a happy path + log.warning("cache write failed", error=str(e)) + + fut._resolve_ok(data) + + +class _ClientBackend: + """Control-plane SubmissionBackend. + + Adapts ``ArmoniKResults`` / ``ArmoniKTasks`` to the three primitives + :func:`pymonik._internal.submit.submit_many` calls. + """ + + __slots__ = ("_s",) + + def __init__(self, session: "Session") -> None: + self._s = session + + @property + def session_id(self) -> str: + return self._s.session_id + + @property + def allowed_partitions(self) -> tuple[str, ...] | None: + return self._s._partitions + + def allocate_outputs(self, names: list[str]) -> list[str]: + assert self._s._results is not None + m = self._s._results.create_results_metadata( + result_names=names, session_id=self._s.session_id + ) + return [m[n].result_id for n in names] + + def upload_payloads(self, named_data: dict[str, bytes]) -> dict[str, str]: + assert self._s._results is not None + m = self._s._results.create_results( + results_data=named_data, session_id=self._s.session_id + ) + return {n: r.result_id for n, r in m.items()} + + def submit( + self, + definitions: list[TaskDefinition], + default_options: TaskOptions, + ) -> list[str]: + assert self._s._tasks is not None + submitted = self._s._tasks.submit_tasks( + session_id=self._s.session_id, + tasks=definitions, + default_task_options=default_options, + ) + return [s.id for s in submitted] diff --git a/src/pymonik/task.py b/src/pymonik/task.py new file mode 100644 index 0000000..94fc18c --- /dev/null +++ b/src/pymonik/task.py @@ -0,0 +1,343 @@ +"""@task decorator, Task wrapper, .spawn / .map / .tail / .with_options. + +The decorator preserves the wrapped function's signature via ParamSpec so +``add(2, 3)`` (local call) and ``add.spawn(2, 3)`` (remote submission) both +type-check. Options merge left-to-right: + + session default ← @task(...) ← .with_options(...) + +``.spawn()`` returns a ``Future[T]``. ``.map()`` returns a +``FutureList[T]``. ``.tail()`` returns a lazily-submitted +``TailPromise[T]`` for sub-tasking. Passing futures as args is +transparently rewritten into ArmoniK data dependencies (see +``_internal/refs.py``). + +Multi-output tasks return a :class:`pymonik.MultiResult`. The decorator +walks the function body's AST to find every ``MultiResult(...)`` literal +and extracts the field set, so the submission pipeline can pre-allocate +the right number of ``expected_output_ids`` per task. Inconsistent +shapes between branches raise at decoration time. +""" + +from __future__ import annotations + +import contextvars +import functools +from typing import Any, Callable, Generic, Iterable, ParamSpec, TypeVar, overload + +import anyio + +from pymonik._internal._ast_introspect import extract_multi_fields +from pymonik._internal.protocols import SubmittableSession +from pymonik.errors import NotInSessionError, PymonikError +from pymonik.future import Future, FutureList +from pymonik.multiresult import TailPromise +from pymonik.options import EMPTY, TaskOpts + +P = ParamSpec("P") +R = TypeVar("R") + + +# The "currently open session" — set by Session.__enter__ / +# WorkerSession.__init__ (via worker.py) / LocalSession.__enter__. +# Held here rather than in session.py to avoid an import cycle. +_current_session: contextvars.ContextVar["SubmittableSession | None"] = ( + contextvars.ContextVar("_current_session", default=None) +) + + +def current_session() -> SubmittableSession: + s = _current_session.get() + if s is None: + raise NotInSessionError( + "no session open. Wrap your spawn() calls in `with client.session(): ...`." + ) + return s + + +class Task(Generic[P, R]): + """A function wrapped for ArmoniK submission.""" + + __slots__ = ("func", "name", "opts", "multi_fields") + + def __init__( + self, + func: Callable[P, R], + *, + name: str | None = None, + opts: TaskOpts = EMPTY, + multi_fields: tuple[str, ...] | None = None, + ) -> None: + self.func: Callable[P, R] = func + self.name = name or getattr(func, "__name__", "") + self.opts = opts + # Sorted field names for multi-output tasks. ``None`` for plain + # single-output tasks. Set by the @task decorator via AST + # introspection (or via ``@task(outputs=(...))``). + self.multi_fields = multi_fields + + # Local call — just runs the function. + def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R: + return self.func(*args, **kwargs) + + def with_options( + self, + *, + partition: str | None = None, + retries: int | None = None, + timeout: Any = None, + priority: int | None = None, + retry_on: tuple[type[BaseException], ...] | None = None, + retry_backoff: Any = None, + cache: bool | None = None, + deps: list[str] | tuple[str, ...] | None = None, + isolate: bool | None = None, + index_url: str | None = None, + env: dict[str, str] | None = None, + options: dict[str, str] | None = None, + ) -> "Task[P, R]": + """Return a new Task with overridden options. Never mutates self.""" + patch = TaskOpts( + partition=partition, + retries=retries, + timeout=timeout, + priority=priority, + retry_on=retry_on, + retry_backoff=retry_backoff, + cache=cache, + deps=tuple(deps) if deps is not None else None, + isolate=isolate, + index_url=index_url, + env=dict(env) if env is not None else None, + options=options, + ) + return Task( + self.func, + name=self.name, + opts=self.opts.merge(patch), + multi_fields=self.multi_fields, + ) + + def spawn(self, *args: P.args, **kwargs: P.kwargs) -> Future[R]: + """Submit this task for remote execution. Returns a Future. + + For multi-output tasks (those that return :class:`MultiResult`), + ``.spawn()`` returns a :class:`MultiResultHandle` whose + ``.field_name`` attributes are individual Futures. Awaiting the + handle blocks on every field; awaiting one field blocks only on + that one. + + This is sync. From async code it blocks the event loop briefly + (~few ms of gRPC) while the submission happens. For tight inner + loops where that matters, use :meth:`spawn_async`. + """ + if "_delegate" in kwargs: + raise PymonikError( + "_delegate=True is no longer supported. Use task.tail(*args) " + "for tail-call sub-tasking." + ) + session = current_session() + return session._submit_one(self, args, kwargs) + + def tail(self, *args: P.args, **kwargs: P.kwargs) -> "TailPromise[R]": + """Build a lazily-submitted tail-call promise. + + ``return other.tail(args)`` from a ``@task`` body delegates the + parent's expected output to ``other``. Inside a ``MultiResult`` + field, ``other.tail(args)`` delegates that one field's output. + + The promise is not submitted until the parent ``@task``'s + worker dispatcher binds it to an output id. Awaiting a + ``TailPromise`` directly raises — use :meth:`spawn` if you + want to submit and await. + """ + return TailPromise(self, args, kwargs) + + def map(self, *iterables: Iterable[Any]) -> FutureList[R]: + """Apply this task elementwise across one or more iterables. + + Mirrors Python's built-in :func:`map`: + + square.map([1, 2, 3]) -> square(1), square(2), square(3) + add.map([1, 3], [2, 4]) -> add(1, 2), add(3, 4) + + The N iterables are zipped (stopping at the shortest) and one + task is submitted per zipped tuple. Submission is batched into a + single RPC. See :meth:`starmap` for the + already-have-tuples-of-args shape, and :meth:`map_async` for + the offloaded variant. + """ + if not iterables: + raise TypeError("Task.map requires at least one iterable") + calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = [ + (args, {}) for args in zip(*iterables) + ] + session = current_session() + return session._submit_many(self, calls) + + def starmap(self, args_iter: Iterable[tuple[Any, ...]]) -> FutureList[R]: + """Apply this task to each tuple after unpacking it as positional args. + + Mirrors :func:`itertools.starmap`: + + add.starmap([(1, 2), (3, 4)]) -> add(1, 2), add(3, 4) + + Use this when you already have tuples-of-args; use :meth:`map` + when you have one (or more) parallel iterables. + """ + session = current_session() + return session._submit_many(self, list(args_iter)) + + async def spawn_async(self, *args: P.args, **kwargs: P.kwargs) -> Future[R]: + """Async sibling of :meth:`spawn`. + + Offloads the (otherwise blocking) gRPC submission to a worker + thread via :func:`anyio.to_thread.run_sync` so the calling event + loop keeps running. ContextVars (the current session, OTel + context) are propagated automatically by anyio. + + The returned ``Future`` is the same shape as ``spawn()``'s — use + ``await fut`` to get the value. + """ + if "_delegate" in kwargs: + raise PymonikError( + "_delegate=True is no longer supported. Use task.tail(*args) " + "for tail-call sub-tasking." + ) + session = current_session() + fn = functools.partial(session._submit_one, self, args, kwargs) + return await anyio.to_thread.run_sync(fn) + + async def map_async(self, *iterables: Iterable[Any]) -> FutureList[R]: + """Async sibling of :meth:`map`. + + Offloads the batched submission RPC off the event loop. Useful + when ``map`` is called with many tasks (the per-batch round-trip + gets bigger with N). + """ + if not iterables: + raise TypeError("Task.map_async requires at least one iterable") + items: list[tuple[tuple[Any, ...], dict[str, Any]]] = [ + (args, {}) for args in zip(*iterables) + ] + session = current_session() + fn = functools.partial(session._submit_many, self, items) + return await anyio.to_thread.run_sync(fn) + + async def starmap_async( + self, args_iter: Iterable[tuple[Any, ...]] + ) -> FutureList[R]: + """Async sibling of :meth:`starmap`.""" + session = current_session() + items = list(args_iter) + fn = functools.partial(session._submit_many, self, items) + return await anyio.to_thread.run_sync(fn) + + def __repr__(self) -> str: + return f"" + + +# --- decorator --- + +@overload +def task(func: Callable[P, R], /) -> Task[P, R]: ... +@overload +def task( + *, + partition: str | None = None, + retries: int | None = None, + timeout: Any = None, + priority: int | None = None, + retry_on: tuple[type[BaseException], ...] | None = None, + retry_backoff: Any = None, + cache: bool | None = None, + deps: list[str] | tuple[str, ...] | None = None, + isolate: bool | None = None, + index_url: str | None = None, + env: dict[str, str] | None = None, + options: dict[str, str] | None = None, + outputs: tuple[str, ...] | list[str] | None = None, +) -> Callable[[Callable[P, R]], Task[P, R]]: ... +def task(func: Callable[P, R] | None = None, /, **kwargs: Any) -> Any: + """Decorate a function for ArmoniK submission. + + Usage: + + @task + def add(a: int, b: int) -> int: + return a + b + + @task(partition="gpu", retries=3, timeout=timedelta(minutes=5)) + def render(scene: Scene) -> bytes: + ... + + @task(retries=5, retry_on=(ConnectionError,), retry_backoff="exponential") + def flaky_call(): ... + + Plain ``retries=N`` is cluster-side: ArmoniK retries up to N times. + Adding ``retry_on=(...)`` switches to *client-side* retry — same + budget, but the SDK observes the failure type, sleeps the configured + backoff, and re-spawns. Cluster ``max_retries`` then sits at 2 to + cover infra failures. + + For multi-output tasks (those returning ``MultiResult``), the + decorator extracts the field set from the function body's AST. If + your construction is dynamic (a helper function, a comprehension) + or you'd rather declare it explicitly, pass + ``outputs=("field_a", "field_b", ...)``. + + Decorator-level options are merged with session defaults and with + ``.with_options(...)`` overrides at submission time. + """ + deps = kwargs.pop("deps", None) + env = kwargs.pop("env", None) + explicit_outputs = kwargs.pop("outputs", None) + opts = TaskOpts( + partition=kwargs.pop("partition", None), + retries=kwargs.pop("retries", None), + timeout=kwargs.pop("timeout", None), + priority=kwargs.pop("priority", None), + retry_on=kwargs.pop("retry_on", None), + retry_backoff=kwargs.pop("retry_backoff", None), + cache=kwargs.pop("cache", None), + deps=tuple(deps) if deps is not None else None, + isolate=kwargs.pop("isolate", None), + index_url=kwargs.pop("index_url", None), + env=dict(env) if env is not None else None, + options=kwargs.pop("options", None), + ) + if kwargs: + raise TypeError(f"@task got unexpected kwargs: {sorted(kwargs)}") + + def _wrap(f: Callable[P, R]) -> Task[P, R]: + if explicit_outputs is not None: + from pymonik.multiresult import MultiResult as _MR + + multi_fields: tuple[str, ...] | None = tuple(sorted(explicit_outputs)) + bad = set(multi_fields) & _MR._RESERVED_FIELD_NAMES + if bad: + raise PymonikError( + f"@task {f.__name__!r}: outputs={sorted(bad)} collide " + f"with MultiResultHandle attributes. Reserved names: " + f"{sorted(_MR._RESERVED_FIELD_NAMES)}." + ) + bad_uscore = {n for n in multi_fields if n.startswith("_")} + if bad_uscore: + raise PymonikError( + f"@task {f.__name__!r}: outputs={sorted(bad_uscore)} " + f"are invalid: underscore-prefixed names are reserved." + ) + else: + multi_fields = extract_multi_fields(f) + if multi_fields is not None and opts.cache is True: + raise PymonikError( + f"@task {f.__name__!r}: cache=True is not compatible with " + f"multi-output tasks. The execution cache stores one bytes " + f"blob per task; per-field caching for MultiResult isn't " + f"implemented." + ) + return Task(f, opts=opts, multi_fields=multi_fields) + + if func is not None: + return _wrap(func) + return _wrap diff --git a/src/pymonik/testing/__init__.py b/src/pymonik/testing/__init__.py new file mode 100644 index 0000000..30e0ac0 --- /dev/null +++ b/src/pymonik/testing/__init__.py @@ -0,0 +1,18 @@ +"""In-process backend for unit tests, examples, and demos. + +Usage: + + from pymonik.testing import LocalCluster + + with LocalCluster() as client: + with client.session(partition="local") as s: + assert add.spawn(2, 3).result() == 5 + +See :class:`LocalCluster` for what's supported and the few cluster-only +features that are no-ops (``pause`` / ``resume`` / ``stop_submission`` +have no in-process meaning and just log). +""" + +from pymonik.testing.local import LocalCluster, LocalSession + +__all__ = ["LocalCluster", "LocalSession"] diff --git a/src/pymonik/testing/local.py b/src/pymonik/testing/local.py new file mode 100644 index 0000000..751a88d --- /dev/null +++ b/src/pymonik/testing/local.py @@ -0,0 +1,1080 @@ +"""In-process backend for tests / examples / iteration loops. + +``LocalCluster`` mimics :class:`pymonik.PymonikClient` minus the network: +tasks run in a thread pool, but the public surface (``@task``, +``.spawn``, ``.map``, ``Future``, ``await fut``, blobs, ``current()``, +``.cancel()``, retries) behaves the same so user code is portable. + +Fidelity +-------- +Submission goes through the same shared pipeline +(:func:`pymonik._internal.submit.submit_many`) as the real client: +``extract_deps`` rewrites Future / Blob / Materialize args into wire +refs, ``auto_spill`` handles oversize values, the +:class:`~pymonik.envelope.TaskEnvelope` is encoded with msgspec, and a +worker function on the thread pool decodes that envelope, looks up data +dependencies in a session-local dict, runs the function, and pickles +the result. The wire format is exercised end-to-end in-process — bugs +in envelope encoding, ref resolution, or auto-spill surface here the +same way they would on the cluster. + +What's still local-only: + +- No pod scheduling latency, no partition routing, no autoscaling. +- No worker isolation — everything shares the host process. +- ``max_retries`` (cluster-side, infra-failure retry) isn't emulated; + client-side retries via ``@task(retry_on=...)`` work end-to-end via + the same code path the real session uses. + +Deadlock note: the executor uses a default of 16 threads. A pipeline +whose in-flight depth exceeds the pool can deadlock (every thread +blocked on a data dep whose computation needs another thread). Pass +``LocalCluster(max_workers=N)`` for deeper graphs. +""" + +from __future__ import annotations + +import hashlib +import threading +import traceback +import uuid +from concurrent.futures import ThreadPoolExecutor +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any + +import anyio +import cloudpickle +from pymonik._internal._logging import get_logger +from armonik.common import TaskDefinition, TaskOptions + +from pymonik import context as ctx_mod +from pymonik import envelope as env_mod +from pymonik._internal.exec_cache import ExecCache, compute_cache_key, default_cache_dir +from pymonik._internal.refs import auto_spill, extract_deps, resolve_refs +from pymonik._internal.submit import normalise_calls, submit_many +from pymonik.context import WorkerContext +from pymonik.errors import TaskCancelled, TaskFailed +from pymonik.future import Future, FutureList +from pymonik.options import EMPTY, TaskOpts +from pymonik.task import Task, _current_session + +log = get_logger(__name__) + + +class _FakeTaskHandler: + """Minimal duck-type for ``armonik.worker.TaskHandler`` — what + :class:`WorkerContext` reads off it (``task_id`` / ``session_id``). + """ + + __slots__ = ("task_id", "session_id") + + def __init__(self, task_id: str, session_id: str) -> None: + self.task_id = task_id + self.session_id = session_id + + +class LocalCluster: + """Drop-in for ``PymonikClient`` that runs tasks in a thread pool. + + Use exactly like the real client:: + + with LocalCluster() as client: + with client.session(partition="local") as s: + assert add.spawn(2, 3).result() == 5 + + Or async:: + + async with LocalCluster() as client: + async with client.session_async(partition="local") as s: + assert await add.spawn(2, 3) == 5 + """ + + def __init__( + self, + *, + max_workers: int = 16, + cache: bool | str | Path | None = None, + ) -> None: + self._max_workers = max_workers + self._executor: ThreadPoolExecutor | None = None + self._cache: ExecCache | None + if cache is None or cache is False: + self._cache = None + elif cache is True: + self._cache = ExecCache(default_cache_dir()) + else: + self._cache = ExecCache(Path(cache)) + if self._cache is not None: + log.info("local exec cache enabled", root=str(self._cache.root)) + + # ---- sync lifecycle ---- + + def __enter__(self) -> "LocalCluster": + self._executor = ThreadPoolExecutor( + max_workers=self._max_workers, + thread_name_prefix="pymonik-local", + ) + log.info("local cluster started", max_workers=self._max_workers, mode="sync") + return self + + def __exit__(self, exc_type, exc, tb) -> None: + if self._executor is not None: + self._executor.shutdown(wait=True) + self._executor = None + + def session( + self, + *, + partition: str | list[str] | tuple[str, ...] = "local", + default_options: TaskOpts | None = None, + deps: list[str] | tuple[str, ...] | None = None, + isolate: bool | None = None, + index_url: str | None = None, + env: dict[str, str] | None = None, + ) -> "LocalSession": + merged = default_options or EMPTY + if ( + deps is not None + or isolate is not None + or index_url is not None + or env is not None + ): + merged = merged.merge( + TaskOpts( + deps=tuple(deps) if deps is not None else None, + isolate=isolate, + index_url=index_url, + env=dict(env) if env is not None else None, + ) + ) + return LocalSession( + self, + partition=partition, + default_options=merged, + cache=self._cache, + ) + + # ---- async lifecycle ---- + + async def __aenter__(self) -> "LocalCluster": + self._executor = ThreadPoolExecutor( + max_workers=self._max_workers, + thread_name_prefix="pymonik-local", + ) + log.info("local cluster started", max_workers=self._max_workers, mode="async") + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + if self._executor is not None: + await anyio.to_thread.run_sync(self._executor.shutdown) + self._executor = None + + @asynccontextmanager + async def session_async( + self, + *, + partition: str | list[str] | tuple[str, ...] = "local", + default_options: TaskOpts | None = None, + deps: list[str] | tuple[str, ...] | None = None, + isolate: bool | None = None, + index_url: str | None = None, + env: dict[str, str] | None = None, + ): + merged = default_options or EMPTY + if ( + deps is not None + or isolate is not None + or index_url is not None + or env is not None + ): + merged = merged.merge( + TaskOpts( + deps=tuple(deps) if deps is not None else None, + isolate=isolate, + index_url=index_url, + env=dict(env) if env is not None else None, + ) + ) + sess = LocalSession( + self, + partition=partition, + default_options=merged, + cache=self._cache, + ) + async with sess: + yield sess + + +class LocalSession: + """In-process equivalent of :class:`pymonik.session.Session`. + + Same submission API; futures are real :class:`pymonik.Future` instances + so ``.result()`` / ``await`` / ``.cancel()`` work with the same code. + Submission goes through the same shared pipeline the cluster session + uses, so envelope encoding and ref resolution are exercised here too. + """ + + def __init__( + self, + cluster: LocalCluster, + partition: str | list[str] | tuple[str, ...], + default_options: TaskOpts = EMPTY, + *, + cache: ExecCache | None = None, + ) -> None: + self._cluster = cluster + if isinstance(partition, str): + self._partitions: tuple[str, ...] = (partition,) + else: + parts = tuple(partition) + if not parts: + raise ValueError("partition list cannot be empty") + self._partitions = parts + self._partition = self._partitions[0] + self._default_opts = default_options + self._cache = cache + self._session_id = f"local-{uuid.uuid4().hex[:8]}" + + self._pending: dict[str, Future[Any]] = {} + self._cancel_events: dict[str, threading.Event] = {} + + # Three buckets of bytes addressable by result_id: + # _payloads — envelope bytes from upload_payloads + # _blob_bytes — blob.upload + auto-spill bytes + # _result_bytes — pickled return values from completed tasks + # _result_events signals "bytes are now in _result_bytes or + # _blob_bytes", so dispatcher threads waiting on a data dep can + # block efficiently. + self._payloads: dict[str, bytes] = {} + self._blob_bytes: dict[str, bytes] = {} + self._result_bytes: dict[str, bytes] = {} + self._result_events: dict[str, threading.Event] = {} + + self._lock = threading.Lock() + self._stop = threading.Event() + self._spill_threshold = 1 << 30 # ~1 GiB; effectively never spill locally + self._ctx_token: Any = None + + @property + def session_id(self) -> str: + return self._session_id + + @property + def partition(self) -> str: + return self._partition + + @property + def partitions(self) -> tuple[str, ...]: + return self._partitions + + # ---- context manager (sync) ---- + + def __enter__(self) -> "LocalSession": + self._ctx_token = _current_session.set(self) + return self + + def __exit__(self, exc_type, exc, tb) -> None: + try: + self._stop.set() + with self._lock: + pending = list(self._pending.values()) + self._pending.clear() + for fut in pending: + fut._resolve_error(TaskCancelled(fut.task_id)) + finally: + if self._ctx_token is not None: + _current_session.reset(self._ctx_token) + self._ctx_token = None + + # ---- context manager (async) ---- + + async def __aenter__(self) -> "LocalSession": + self._ctx_token = _current_session.set(self) + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + try: + self._stop.set() + with self._lock: + pending = list(self._pending.values()) + self._pending.clear() + for fut in pending: + fut._resolve_error(TaskCancelled(fut.task_id)) + finally: + if self._ctx_token is not None: + _current_session.reset(self._ctx_token) + self._ctx_token = None + + # ---- blob upload (in-memory, content-hash dedup like Session) ---- + + def _upload_blob(self, data: bytes) -> str: + from pymonik import blob as blob_mod + + h = blob_mod.content_hash(data) + rid = f"local-blob-{h[:16]}" + with self._lock: + if rid in self._blob_bytes: + return rid + self._blob_bytes[rid] = data + ev = self._result_events.setdefault(rid, threading.Event()) + ev.set() + return rid + + # ---- submission ---- + + def _submit_one( + self, + task: Task[Any, Any], + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + return self._submit_many(task, [(args, kwargs)])[0] + + def _submit_many( + self, + task: Task[Any, Any], + calls: list[Any], + ) -> FutureList[Any]: + normalised = normalise_calls(calls) + cached_hits, miss_idxs, keys = self._cache_classify(task, normalised) + + miss_calls = [normalised[i] for i in miss_idxs] + if miss_calls: + miss_futures = self._submit_through_pipeline(task, miss_calls) + else: + miss_futures = FutureList([]) + + out: list[Future[Any]] = [None] * len(normalised) # type: ignore[list-item] + for i, raw in cached_hits.items(): + out[i] = Future._new_cached(self, raw) + for j, idx in enumerate(miss_idxs): + fut = miss_futures[j] + if idx in keys: + fut._cache_key = keys[idx] + out[idx] = fut + return FutureList(out) + + def _cache_classify( + self, + task: Task[Any, Any], + normalised: list[tuple[tuple[Any, ...], dict[str, Any]]], + ) -> tuple[dict[int, bytes], list[int], dict[int, str]]: + if self._cache is None or task.opts.cache is not True: + return {}, list(range(len(normalised))), {} + + import pymonik + + fn_pickle_hash = hashlib.sha256(cloudpickle.dumps(task.func)).digest() + cached_hits: dict[int, bytes] = {} + miss_idxs: list[int] = [] + keys: dict[int, str] = {} + for i, (args, kwargs) in enumerate(normalised): + key = compute_cache_key( + pymonik_version=pymonik.__version__, + task_name=task.name, + function_pickle_hash=fn_pickle_hash, + args=args, + kwargs=kwargs, + ) + if key is None: + miss_idxs.append(i) + continue + try: + cached_hits[i] = self._cache.get_bytes(key) + log.info("cache hit (local)", task=task.name, key=key[:16]) + except KeyError: + miss_idxs.append(i) + keys[i] = key + if cached_hits: + log.info( + "cache batch summary (local)", + task=task.name, + hits=len(cached_hits), + misses=len(miss_idxs), + ) + return cached_hits, miss_idxs, keys + + def _submit_through_pipeline( + self, + task: Task[Any, Any], + calls: list[tuple[tuple[Any, ...], dict[str, Any]]], + ) -> FutureList[Any]: + from pymonik.future import MultiResultHandle + + backend = _LocalBackend(self) + multi_fields = task.multi_fields + + def make_future( + task_id: str, + output_ids: list[str], + _args: tuple[Any, ...], + _kwargs: dict[str, Any], + ) -> Any: + if multi_fields: + field_to_future = { + field: Future(self, task_id=task_id, result_id=oid) + for field, oid in zip(multi_fields, output_ids) + } + return MultiResultHandle(self, task_id, field_to_future) + return Future(self, task_id=task_id, result_id=output_ids[0]) + + def register_and_launch(output_ids: list[str], handle: Any) -> None: + with self._lock: + if multi_fields: + for field, oid in zip(multi_fields, output_ids): + self._pending[oid] = handle._field_to_future[field] + else: + self._pending[output_ids[0]] = handle + # The backend launches the dispatcher keyed by the *primary* + # output id (first of the group). The dispatcher then writes + # to all output ids in the group. + backend._launch_for(output_ids) + + return submit_many( + task=task, + calls=calls, + backend=backend, + blob_uploader=self._upload_blob, + spill_threshold=self._spill_threshold, + default_opts=self._default_opts, + partition=self._partition, + future_factory=make_future, + on_submitted=register_and_launch, + apply_retry_policy=True, + attempt=1, + ) + + # ---- retry path (same hooks Session uses) ---- + + def _schedule_retry(self, fut: Future[Any], *, attempt: int) -> None: + rs = fut._retry_state + assert rs is not None + task, args, kwargs, _max, _on, backoff_fn = rs + delay = max(0.0, float(backoff_fn(attempt - 1))) + log.info( + "task retrying (local)", + task=task.name, + attempt=attempt, + delay_s=round(delay, 3), + old_task_id=fut.task_id, + ) + + def _run(): + if delay > 0.0 and self._stop.wait(timeout=delay): + return + backend = _LocalBackend(self) + + def make_future(*_a, **_k): + raise AssertionError("future_factory not used with existing_future") + + def register_and_launch(output_ids: list[str], registered_fut: Any) -> None: + with self._lock: + self._pending[output_ids[0]] = registered_fut + backend._launch_for(output_ids) + + submit_many( + task=task, + calls=[(args, kwargs)], + backend=backend, + blob_uploader=self._upload_blob, + spill_threshold=self._spill_threshold, + default_opts=self._default_opts, + partition=self._partition, + future_factory=make_future, + on_submitted=register_and_launch, + apply_retry_policy=False, + existing_future=fut, + attempt=fut._retry_attempt + 1, + ) + + threading.Thread( + target=_run, + name=f"pymonik-local-retry-{fut.task_id[-8:]}", + daemon=True, + ).start() + + # ---- cancellation ---- + + def _cancel_future(self, fut: Future[Any]) -> None: + with self._lock: + self._pending.pop(fut.result_id, None) + ev = self._cancel_events.pop(fut.result_id, None) + # Wake any data-dep waiters with no bytes (they'll see the + # error path). + rev = self._result_events.get(fut.result_id) + if ev is not None: + ev.set() + if rev is not None: + rev.set() + fut._resolve_error(TaskCancelled(fut.task_id)) + + def cancel(self) -> None: + """Cancel this in-process session. Same shape as Session.cancel.""" + with self._lock: + pending = list(self._pending.values()) + self._pending.clear() + cancel_events = list(self._cancel_events.values()) + self._cancel_events.clear() + result_events = list(self._result_events.values()) + for ev in cancel_events: + ev.set() + for ev in result_events: + ev.set() + for fut in pending: + fut._resolve_error(TaskCancelled(fut.task_id)) + + # LocalCluster has no real session-lifecycle RPCs, but we mirror the + # cluster Session's verb set so the same code runs in both places. + # pause/resume/stop_submission are no-ops locally. + + def pause(self) -> None: # pragma: no cover - in-process no-op + log.info("local pause (no-op)", session=self.session_id) + + def resume(self) -> None: # pragma: no cover + log.info("local resume (no-op)", session=self.session_id) + + def stop_submission(self, *, client: bool = True, worker: bool = True) -> None: # pragma: no cover + log.info( + "local stop_submission (no-op)", + session=self.session_id, + client=client, + worker=worker, + ) + + # ---- dispatcher (worker-equivalent for one task) ---- + + def _write_result_bytes(self, output_id: str, pickled: bytes) -> None: + """Write resolved bytes for one output id and wake any waiters.""" + with self._lock: + self._result_bytes[output_id] = pickled + ev = self._result_events.get(output_id) + if ev is not None: + ev.set() + + def _submit_tail( + self, + promise: Any, + *, + expected_output_ids: list[str], + ) -> str: + """Submit a TailPromise to run with caller-supplied output ids. + + The local equivalent of :meth:`WorkerSession._submit_tail`. Builds + an envelope for the promise's task, registers it as a dispatch + keyed by the (parent's) primary output id — replacing the + parent's already-completed dispatch entry — and schedules. The + tail's dispatcher writes to ``expected_output_ids``; the parent's + future, registered under ``expected_output_ids[0]``, resolves + when the tail completes. + """ + from pymonik._internal._otel import inject_context + + task = promise._task + args = promise._args + kwargs = promise._kwargs + + deps: list[str] = [] + args_rewritten = tuple(extract_deps(a, deps) for a in args) + kwargs_rewritten = {k: extract_deps(v, deps) for k, v in kwargs.items()} + args_rewritten = tuple( + auto_spill( + a, deps, upload_blob=self._upload_blob, threshold=self._spill_threshold + ) + for a in args_rewritten + ) + kwargs_rewritten = { + k: auto_spill( + v, deps, upload_blob=self._upload_blob, threshold=self._spill_threshold + ) + for k, v in kwargs_rewritten.items() + } + + merged_opts = task.opts + env_dict = merged_opts.env or {} + env_spec_obj = None + if merged_opts.deps or env_dict: + from pymonik.envelope import EnvSpec + + env_spec_obj = EnvSpec( + deps=tuple(merged_opts.deps or ()), + isolate=merged_opts.isolate if merged_opts.isolate is not None else False, + index_url=merged_opts.index_url or "", + env=tuple(sorted(env_dict.items())), + ) + + carrier: dict[str, str] = {} + inject_context(carrier) + + from pymonik.envelope import TaskEnvelope + + envelope = TaskEnvelope( + function_pickle=cloudpickle.dumps(task.func), + args_pickle=cloudpickle.dumps((args_rewritten, kwargs_rewritten)), + func_name=task.name, + attempt=1, + env_spec=env_spec_obj, + otel_context=tuple(sorted(carrier.items())), + multi_fields=task.multi_fields or (), + ) + payload_bytes = env_mod.encode(envelope) + + new_task_id = f"local-tail-{uuid.uuid4().hex[:12]}" + cancel_ev = threading.Event() + with self._lock: + self._cancel_events[expected_output_ids[0]] = cancel_ev + + executor = self._cluster._executor + assert executor is not None, "LocalCluster is not running" + executor.submit( + self._dispatch, + new_task_id, + list(expected_output_ids), + payload_bytes, + sorted(set(deps)), + cancel_ev, + ) + log.info( + "tail submitted (local)", + func=task.name, + child_task=new_task_id, + expected_outputs=list(expected_output_ids), + ) + return new_task_id + + def _dispatch_result( + self, + *, + result: Any, + envelope: Any, + task_id: str, + output_ids: list[str], + fut: Any, + ) -> None: + """Map a user-function return onto local output writes / tail submits. + + Mirrors :func:`pymonik.worker._dispatch_result` for LocalCluster. + """ + from pymonik.future import MultiResultHandle as _MRH + from pymonik.multiresult import MultiResult, TailPromise + + multi_fields = envelope.multi_fields + + # ---- whole-task tail-call ---- + if isinstance(result, TailPromise): + child_task = result._task + child_multi = child_task.multi_fields or () + if multi_fields: + if child_multi != multi_fields: + fut._resolve_error( + TaskFailed( + task_id, + f"tail-called task {child_task.name!r} declares " + f"{list(child_multi)}, parent declares " + f"{list(multi_fields)}", + ) + ) + return + else: + if child_multi: + fut._resolve_error( + TaskFailed( + task_id, + f"tail-called task {child_task.name!r} is multi-output " + f"but parent is single-output", + ) + ) + return + self._submit_tail(result, expected_output_ids=output_ids) + return + + # ---- multi-output return ---- + if isinstance(result, MultiResult): + if not multi_fields: + fut._resolve_error( + TaskFailed( + task_id, + "function returned MultiResult but task wasn't declared " + "multi-output (decoration didn't extract a schema).", + ) + ) + return + returned = set(result.fields.keys()) + declared = set(multi_fields) + if returned != declared: + fut._resolve_error( + TaskFailed( + task_id, + f"MultiResult shape mismatch: declared {sorted(declared)}, " + f"returned {sorted(returned)}", + ) + ) + return + + field_to_oid = dict(zip(multi_fields, output_ids)) + for field, value in result.fields.items(): + oid = field_to_oid[field] + if isinstance(value, TailPromise): + if value._task.multi_fields: + fut._resolve_error( + TaskFailed( + task_id, + f"field {field!r} delegates to multi-output task " + f"{value._task.name!r}; not supported", + ) + ) + return + # Each per-field tail submits its own dispatch with + # only that output id; the field's Future is already + # registered under `oid`, so the tail dispatch + # resolves it when the child writes. + self._submit_tail(value, expected_output_ids=[oid]) + elif isinstance(value, Future): + fut._resolve_error( + TaskFailed( + task_id, + f"field {field!r} is a Future from .spawn() — " + f"use .tail() for delegation", + ) + ) + return + elif isinstance(value, _MRH): + fut._resolve_error( + TaskFailed( + task_id, + f"field {field!r} is a MultiResultHandle; nested " + f"per-field access isn't supported", + ) + ) + return + else: + pickled = cloudpickle.dumps(value) + self._write_result_bytes(oid, pickled) + field_fut = self._field_future_for(oid) + if field_fut is not None: + field_fut._resolve_ok(pickled) + return + + # ---- plain single-output return ---- + if multi_fields: + fut._resolve_error( + TaskFailed( + task_id, + f"task declared multi-output {list(multi_fields)} but " + f"returned {type(result).__name__} (expected MultiResult)", + ) + ) + return + + try: + pickled = cloudpickle.dumps(result) + except Exception as e: + fut._resolve_error( + TaskFailed(task_id, f"could not pickle result: {e!r}") + ) + return + + self._write_result_bytes(output_ids[0], pickled) + if self._cache is not None and fut._cache_key is not None: + try: + self._cache.put_bytes(fut._cache_key, pickled) + log.info( + "cache stored (local)", + task_id=fut.task_id, + key=fut._cache_key[:16], + bytes=len(pickled), + ) + except Exception as e: # noqa: BLE001 + log.warning("cache write failed", error=str(e)) + fut._resolve_ok(pickled) + + def _field_future_for(self, output_id: str) -> "Future[Any] | None": + """Look up the per-field Future registered for this output id.""" + with self._lock: + return self._pending.get(output_id) + + def _dispatch( + self, + task_id: str, + output_ids: list[str], + payload_bytes: bytes, + data_dep_ids: list[str], + cancel_ev: threading.Event, + ) -> None: + """Decode the envelope, resolve refs, run the function, write outputs. + + Mirrors :func:`pymonik.worker._process` minus the ArmoniK plumbing. + For multi-output tasks ``output_ids`` carries N ids in stable + sorted-field order; the worker writes each field's bytes to the + matching id. + """ + sess_token = _current_session.set(self) + spliced_path: str | None = None + prior_env: dict[str, str | None] | None = None + primary_output = output_ids[0] + + # Find the future registered for this dispatch's primary output. + with self._lock: + fut = self._pending.get(primary_output) + if fut is None: + log.warning("local dispatch: no future registered", output_id=primary_output) + _current_session.reset(sess_token) + return + + try: + # Build the data_deps dict by waiting for each upstream result. + data_deps: dict[str, bytes] = {} + for rid in data_dep_ids: + with self._lock: + ev = self._result_events.get(rid) + if ev is not None: + ev.wait() + with self._lock: + if rid in self._result_bytes: + data_deps[rid] = self._result_bytes[rid] + elif rid in self._blob_bytes: + data_deps[rid] = self._blob_bytes[rid] + else: + # Upstream cancelled or missing. + fut._resolve_error( + TaskFailed(task_id, f"upstream {rid} unavailable") + ) + return + + if cancel_ev.is_set(): + fut._resolve_error(TaskCancelled(task_id)) + return + + # Decode the envelope and resolve refs. + try: + envelope = env_mod.decode(payload_bytes) + except Exception as e: + fut._resolve_error( + TaskFailed(task_id, f"local envelope decode failed: {e!r}") + ) + return + + # If env_spec.deps + isolate=True, run via subprocess for fidelity + # with the worker. ``isolate=False`` falls through to the inline + # path after splicing the venv's site-packages into sys.path. + if ( + envelope.env_spec is not None + and envelope.env_spec.deps + and envelope.env_spec.isolate + ): + from pymonik._internal.subprocess_dispatch import run_in_subprocess + + try: + pickled = run_in_subprocess( + env_spec=envelope.env_spec, + envelope_bytes=payload_bytes, + data_deps=data_deps, + ) + except TaskFailed as e: + fut._resolve_error(e) + return + except Exception as e: + fut._resolve_error( + TaskFailed(task_id, f"local subprocess dispatch failed: {e!r}") + ) + return + # Subprocess path is single-output (multi-output is rejected + # upstream in worker._process for isolate=True because the + # subprocess child has no agent-sidecar channel). + self._write_result_bytes(output_ids[0], pickled) + if self._cache is not None and fut._cache_key is not None: + try: + self._cache.put_bytes(fut._cache_key, pickled) + except Exception as e: # noqa: BLE001 + log.warning("cache write failed", error=str(e)) + fut._resolve_ok(pickled) + return + + if envelope.env_spec is not None: + from pymonik._internal.env_builder import ( + apply_env_overlay, + ensure_env, + venv_site_packages, + ) + import sys as _sys + + try: + if envelope.env_spec.deps and not envelope.env_spec.isolate: + venv_dir = ensure_env(envelope.env_spec) + site = str(venv_site_packages(venv_dir)) + if site not in _sys.path: + _sys.path.insert(0, site) + spliced_path = site + if envelope.env_spec.env: + prior_env = apply_env_overlay(envelope.env_spec.env) + except Exception as e: + fut._resolve_error( + TaskFailed(task_id, f"local env build failed: {e!r}") + ) + return + + try: + func = cloudpickle.loads(envelope.function_pickle) + args, kwargs = cloudpickle.loads(envelope.args_pickle) + args = tuple(resolve_refs(a, data_deps) for a in args) + kwargs = {k: resolve_refs(v, data_deps) for k, v in kwargs.items()} + except Exception as e: + fut._resolve_error( + TaskFailed(task_id, f"local envelope decode failed: {e!r}") + ) + return + + # Worker-side context (logger, attempt, cancel hook). + fake_th = _FakeTaskHandler(task_id=task_id, session_id=self._session_id) + worker_ctx = WorkerContext( + fake_th, + attempt=envelope.attempt, + cancel_check=cancel_ev.is_set, + ) + ctx_token = ctx_mod._set(worker_ctx) + try: + try: + from pymonik._internal import _otel as _otel_mod + + with _otel_mod.use_extracted_context(dict(envelope.otel_context)): + with _otel_mod.start_span( + "pymonik.task.run", + attrs={ + "pymonik.func": envelope.func_name, + "pymonik.task_id": task_id, + "pymonik.attempt": envelope.attempt, + "pymonik.local": True, + }, + kind="server", + ): + result = func(*args, **kwargs) + except TaskCancelled: + fut._resolve_error(TaskCancelled(task_id)) + return + except Exception as e: + tb = traceback.format_exc() + fut._resolve_error( + TaskFailed(task_id, f"{type(e).__name__}: {e}\n{tb}") + ) + return + finally: + ctx_mod._reset(ctx_token) + + self._dispatch_result( + result=result, + envelope=envelope, + task_id=task_id, + output_ids=output_ids, + fut=fut, + ) + + finally: + if prior_env is not None: + from pymonik._internal.env_builder import restore_env_overlay + + restore_env_overlay(prior_env) + if spliced_path is not None: + import sys as _sys + + try: + _sys.path.remove(spliced_path) + except ValueError: + pass + with self._lock: + for oid in output_ids: + self._cancel_events.pop(oid, None) + _current_session.reset(sess_token) + + +class _LocalBackend: + """In-process SubmissionBackend. + + ``allocate_outputs`` mints synthetic ids, ``upload_payloads`` parks + bytes in the session's payload dict, and ``submit`` stashes + per-task dispatch parameters keyed by output id. The dispatcher is + fired by :meth:`LocalSession._submit_many`'s ``on_submitted`` hook + (via :meth:`_launch_for`) — that ordering ensures the future is in + ``self._pending`` before the dispatcher thread starts looking for it. + """ + + __slots__ = ("_s", "_dispatches") + + def __init__(self, session: LocalSession) -> None: + self._s = session + # primary_output_id -> (task_id, payload_bytes, data_dep_ids, all_output_ids) + self._dispatches: dict[str, tuple[str, bytes, list[str], list[str]]] = {} + + @property + def session_id(self) -> str: + return self._s.session_id + + @property + def allowed_partitions(self) -> tuple[str, ...] | None: + return None + + def allocate_outputs(self, names: list[str]) -> list[str]: + ids: list[str] = [] + with self._s._lock: + for _ in names: + rid = f"local-out-{uuid.uuid4().hex[:12]}" + self._s._result_events[rid] = threading.Event() + ids.append(rid) + return ids + + def upload_payloads(self, named_data: dict[str, bytes]) -> dict[str, str]: + out: dict[str, str] = {} + with self._s._lock: + for name, data in named_data.items(): + rid = f"local-pl-{uuid.uuid4().hex[:12]}" + self._s._payloads[rid] = data + out[name] = rid + return out + + def submit( + self, + definitions: list[TaskDefinition], + default_options: TaskOptions, + ) -> list[str]: + task_ids: list[str] = [] + for d in definitions: + tid = f"local-task-{uuid.uuid4().hex[:12]}" + task_ids.append(tid) + output_ids = list(d.expected_output_ids) + primary = output_ids[0] + with self._s._lock: + payload_bytes = self._s._payloads[d.payload_id] + self._dispatches[primary] = ( + tid, + payload_bytes, + list(d.data_dependencies), + output_ids, + ) + return task_ids + + def _launch_for(self, output_ids: list[str]) -> None: + """Schedule the dispatch job for the given output id group. + + Called from ``LocalSession._submit_many``'s ``on_submitted`` hook + after the future(s) are registered. ``output_ids[0]`` is the + primary key for ``_dispatches``; the dispatcher writes to all of + ``output_ids``. + """ + primary = output_ids[0] + dispatch = self._dispatches.pop(primary, None) + if dispatch is None: + return + task_id, payload_bytes, data_dep_ids, all_output_ids = dispatch + cancel_ev = threading.Event() + with self._s._lock: + for oid in all_output_ids: + self._s._cancel_events[oid] = cancel_ev + executor = self._s._cluster._executor + assert executor is not None, "LocalCluster is not running" + executor.submit( + self._s._dispatch, + task_id, + all_output_ids, + payload_bytes, + data_dep_ids, + cancel_ev, + ) diff --git a/src/pymonik/worker.py b/src/pymonik/worker.py new file mode 100644 index 0000000..3e02ba7 --- /dev/null +++ b/src/pymonik/worker.py @@ -0,0 +1,395 @@ +"""Worker entrypoint. + +Baked into the PymoniK worker image. On task arrival: + +1. Decode the msgspec envelope from ``task_handler.payload``. +2. Unpickle the function and ``(args, kwargs)``. +3. Walk args/kwargs, replacing any ``FutureRef`` with the downloaded bytes + of the corresponding result from ``task_handler.data_dependencies``. +4. Open a ``WorkerContext`` (reachable via ``pymonik.current()``) and a + worker-side ``WorkerSession`` (so ``task.spawn(...)`` from inside the + user function submits via the agent sidecar). +5. Call the function. +6. If it returns a ``Future``, treat that as a tail call — the referenced + task has been submitted with ``expected_output_ids=[our_own_output]`` + and is now ArmoniK's responsibility to deliver. Otherwise, pickle the + return value and send it as the expected output. + +Errors surface as ``Output(error_message)`` so ArmoniK marks the task as +failed and the client raises ``TaskFailed``. +""" + +from __future__ import annotations + +import contextvars +import traceback +from typing import Any + +import cloudpickle +from pymonik._internal._logging import get_logger +from armonik.common import Output +from armonik.worker import TaskHandler, armonik_worker + +from pymonik import context as ctx_mod +from pymonik import envelope as env_mod +from pymonik._internal import _otel +from pymonik._internal.refs import resolve_refs +from pymonik.context import WorkerContext +from pymonik.errors import TaskCancelled +from pymonik.future import Future, MultiResultHandle +from pymonik.worker_session import WorkerSession + +log = get_logger(__name__) + + +# Populated by the patched ``ArmoniKWorker.Process`` (see :func:`_patch_process`), +# read inside :func:`_process` to give WorkerContext a handle on the gRPC +# server context so user code can observe cancellation. +_grpc_ctx_var: contextvars.ContextVar[Any] = contextvars.ContextVar( + "_pymonik_grpc_ctx", default=None +) + + +def _patch_process() -> None: + """Wrap ``ArmoniKWorker.Process`` so the gRPC context reaches our dispatcher. + + Upstream ``armonik.worker.ArmoniKWorker.Process`` accepts (request, + context) from the gRPC server but only passes ``request`` through to + our processor (via ``TaskHandler``). We need ``context.is_active()`` + for cancellation; this wrapper stashes the context in a + :class:`contextvars.ContextVar` for the duration of the call. Idempotent. + """ + from armonik.worker.worker import ArmoniKWorker + + if getattr(ArmoniKWorker.Process, "_pymonik_patched", False): + return + _original = ArmoniKWorker.Process + + def _patched(self, request, context): + token = _grpc_ctx_var.set(context) + try: + return _original(self, request, context) + finally: + _grpc_ctx_var.reset(token) + + _patched._pymonik_patched = True # type: ignore[attr-defined] + ArmoniKWorker.Process = _patched # type: ignore[method-assign] + + +def _dispatch_result( + result: Any, + *, + envelope: env_mod.TaskEnvelope, + task_handler: TaskHandler, + parent_output_ids: list[str], + session: "WorkerSession", +) -> Output: + """Map a user-function return value onto ArmoniK output writes / submits. + + Three return shapes are valid: + + - ``TailPromise`` — whole-task tail-call. Submit the child with the + parent's full set of expected output ids. Parent task returns + ``Output()`` (the child writes everything). + - ``MultiResult`` — multi-output. Each field is either a plain value + (cloudpickled and written directly) or a ``TailPromise`` (per-field + delegation: submit a child task with that field's output id). + - Anything else — single-output, write the cloudpickled value to + ``parent_output_ids[0]``. + + Returns the ``Output`` to hand back to the agent. + """ + from pymonik.multiresult import MultiResult, TailPromise + + multi_fields: tuple[str, ...] = envelope.multi_fields + + # ---- whole-task tail-call ---- + if isinstance(result, TailPromise): + child_task = result._task + child_multi: tuple[str, ...] = child_task.multi_fields or () + if multi_fields: + # Parent declares N outputs; child must match the schema. + if child_multi != multi_fields: + return Output( + f"worker error: tail-called task {child_task.name!r} declares " + f"fields {list(child_multi)} but parent declares " + f"{list(multi_fields)} — shapes must match for whole-task " + f"tail-call." + ) + session._submit_tail(result, expected_output_ids=parent_output_ids) + else: + # Parent is single-output; child must be too. + if child_multi: + return Output( + f"worker error: tail-called task {child_task.name!r} is " + f"multi-output ({list(child_multi)}) but parent is " + f"single-output. Wrap the call in a multi-output parent " + f"or pick a single-output child." + ) + session._submit_tail(result, expected_output_ids=parent_output_ids) + log.info( + "task tail-called", + task_id=task_handler.task_id, + child_func=child_task.name, + ) + return Output() + + # ---- multi-output return ---- + if isinstance(result, MultiResult): + if not multi_fields: + return Output( + "worker error: function returned MultiResult but task wasn't " + "declared multi-output (decoration didn't extract a field " + "schema). Construct MultiResult with literal kwargs in the " + "task body, or pass outputs=(...) to the @task decorator." + ) + returned = set(result.fields.keys()) + declared = set(multi_fields) + if returned != declared: + missing = declared - returned + extra = returned - declared + details = [] + if missing: + details.append(f"missing {sorted(missing)}") + if extra: + details.append(f"extra {sorted(extra)}") + return Output( + f"worker error: MultiResult shape mismatch ({', '.join(details)}). " + f"Declared: {sorted(declared)}; returned: {sorted(returned)}." + ) + + field_to_oid = dict(zip(multi_fields, parent_output_ids)) + pending_writes: dict[str, bytes] = {} + + for field, value in result.fields.items(): + oid = field_to_oid[field] + if isinstance(value, TailPromise): + # Per-field delegation. The promise's task must be + # single-output (no nested multi-result). + if value._task.multi_fields: + return Output( + f"worker error: field {field!r} delegates to " + f"{value._task.name!r} which is multi-output. " + f"Per-field tail-call requires a single-output task; " + f"forward via a passthrough task instead." + ) + session._submit_tail(value, expected_output_ids=[oid]) + elif isinstance(value, Future): + return Output( + f"worker error: field {field!r} is a Future from .spawn(). " + f"To delegate this field, use .tail() instead — " + f"MultiResult({field}=other.tail(args), ...)." + ) + elif isinstance(value, MultiResultHandle): + return Output( + f"worker error: field {field!r} is a MultiResultHandle. " + f"Per-field nested multi-output access isn't supported; " + f"insert a passthrough single-output task to forward " + f"the specific field." + ) + else: + pending_writes[oid] = cloudpickle.dumps(value) + + if pending_writes: + task_handler.send_results(pending_writes) + log.info( + "task completed (multi)", + task_id=task_handler.task_id, + func=envelope.func_name, + fields=list(multi_fields), + ) + return Output() + + # ---- plain single-output return ---- + if multi_fields: + return Output( + f"worker error: task declared multi-output fields {list(multi_fields)} " + f"but returned a {type(result).__name__} (expected MultiResult)." + ) + task_handler.send_results({parent_output_ids[0]: cloudpickle.dumps(result)}) + log.info("task completed", task_id=task_handler.task_id, func=envelope.func_name) + return Output() + + +def _process(task_handler: TaskHandler) -> Output: + try: + envelope = env_mod.decode(task_handler.payload) + log.info( + "task received", + task_id=task_handler.task_id, + session_id=task_handler.session_id, + func=envelope.func_name, + envelope_version=envelope.version, + data_deps=len(task_handler.data_dependencies or {}), + deps=list(envelope.env_spec.deps) if envelope.env_spec else None, + isolate=envelope.env_spec.isolate if envelope.env_spec else None, + env_keys=[k for k, _ in envelope.env_spec.env] if envelope.env_spec else None, + ) + + if not task_handler.expected_results: + return Output("worker error: no expected_results on the task") + parent_output_ids = list(task_handler.expected_results) + is_multi = bool(envelope.multi_fields) + + if is_multi and len(parent_output_ids) != len(envelope.multi_fields): + return Output( + f"worker error: envelope declares {len(envelope.multi_fields)} " + f"output fields but task has {len(parent_output_ids)} " + f"expected_output_ids" + ) + + data_deps: dict[str, bytes] = dict(task_handler.data_dependencies or {}) + + # Subprocess path: deps declared AND isolation requested. The child + # runs the full pipeline against the env's interpreter; env vars + # are applied to the child's environment by run_in_subprocess. + # Note: subprocess path doesn't (yet) support TailPromise / MultiResult + # — those need agent-sidecar access, which the child doesn't have. + if ( + envelope.env_spec is not None + and envelope.env_spec.deps + and envelope.env_spec.isolate + ): + from pymonik._internal.subprocess_dispatch import run_in_subprocess + + if is_multi: + return Output( + "worker error: multi-output tasks aren't supported " + "with isolate=True (subprocess can't access the agent " + "sidecar). Use isolate=False or move to a baked image." + ) + result_pickle = run_in_subprocess( + env_spec=envelope.env_spec, + envelope_bytes=task_handler.payload, + data_deps=data_deps, + ) + task_handler.send_results({parent_output_ids[0]: result_pickle}) + log.info( + "task completed (subprocess)", + task_id=task_handler.task_id, + func=envelope.func_name, + ) + return Output() + + # All other paths run inline in the worker process; they may need + # to splice a venv into sys.path (deps + !isolate) and/or overlay + # env vars. Compute the overlay once, restore in finally. + import sys as _sys + from pymonik._internal.env_builder import ( + apply_env_overlay, + ensure_env, + restore_env_overlay, + venv_site_packages, + ) + + spliced_path: str | None = None + prior_env: dict[str, str | None] | None = None + if envelope.env_spec is not None: + if envelope.env_spec.deps: + venv_dir = ensure_env(envelope.env_spec) + site = str(venv_site_packages(venv_dir)) + if site not in _sys.path: + _sys.path.insert(0, site) + spliced_path = site + if envelope.env_spec.env: + prior_env = apply_env_overlay(envelope.env_spec.env) + + try: + func = cloudpickle.loads(envelope.function_pickle) + args, kwargs = cloudpickle.loads(envelope.args_pickle) + args = tuple(resolve_refs(a, data_deps) for a in args) + kwargs = {k: resolve_refs(v, data_deps) for k, v in kwargs.items()} + + worker_ctx = WorkerContext( + task_handler, + grpc_context=_grpc_ctx_var.get(), + attempt=envelope.attempt, + ) + session = WorkerSession( + task_handler, parent_output_ids=parent_output_ids + ) + + from pymonik.task import _current_session as _cs + + ctx_token = ctx_mod._set(worker_ctx) + sess_token = _cs.set(session) + try: + # Re-attach the trace context the client injected, then + # open a worker-side span so the user function runs under + # a span that's a child of pymonik.submit. + otel_carrier = dict(envelope.otel_context) + with _otel.use_extracted_context(otel_carrier): + with _otel.start_span( + "pymonik.task.run", + attrs={ + "pymonik.func": envelope.func_name, + "pymonik.task_id": task_handler.task_id, + "pymonik.attempt": envelope.attempt, + }, + kind="server", + ): + result: Any = func(*args, **kwargs) + finally: + _cs.reset(sess_token) + ctx_mod._reset(ctx_token) + finally: + if prior_env is not None: + restore_env_overlay(prior_env) + if spliced_path is not None: + try: + _sys.path.remove(spliced_path) + except ValueError: + pass + + return _dispatch_result( + result, + envelope=envelope, + task_handler=task_handler, + parent_output_ids=parent_output_ids, + session=session, + ) + + except TaskCancelled as e: + # Cooperative cancellation via ``pymonik.current().cancel_if_requested()``. + # The cluster already has the task marked CANCELLING; our return is + # mostly cosmetic (the agent's gRPC call is likely already dead). + log.info("task cancelled cooperatively", task_id=task_handler.task_id) + return Output(f"cancelled: {e}") + + except Exception as e: + tb = traceback.format_exc() + log.error("task failed", task_id=task_handler.task_id, error=str(e)) + return Output(f"{type(e).__name__}: {e}\n{tb}") + + +def run() -> None: + """Bound to the ``pymonik-worker`` console script in pyproject.toml. + + Workers always log — operators rely on the polling-agent → k8s + pipeline to surface what each pod is doing. The library default + (silent) doesn't fit a long-running worker, so we explicitly call + :func:`pymonik.enable_logging` here. Override the level via + ``PYMONIK_WORKER_LOG_LEVEL`` env var. + + OTel: same auto-detect rule as on the client (env vars present → + enabled). Workers typically inherit ``OTEL_EXPORTER_OTLP_ENDPOINT`` + from their pod env so they export to the same collector as the + client. + """ + import os + + from pymonik._internal._logging import enable_logging + + enable_logging(level=os.getenv("PYMONIK_WORKER_LOG_LEVEL", "INFO")) + _otel.setup(service_name=os.getenv("OTEL_SERVICE_NAME", "pymonik-worker")) + _patch_process() + + @armonik_worker() + def processor(task_handler: TaskHandler) -> Output: + return _process(task_handler) + + processor.run() + + +if __name__ == "__main__": + run() diff --git a/src/pymonik/worker_session.py b/src/pymonik/worker_session.py new file mode 100644 index 0000000..db7e8ab --- /dev/null +++ b/src/pymonik/worker_session.py @@ -0,0 +1,275 @@ +"""WorkerSession — what ``task.spawn(...)`` and ``task.tail(...)`` use from +*inside* a worker task. + +Routes through the agent sidecar (``task_handler.create_results_metadata`` +/ ``create_results`` / ``submit_tasks``) instead of the control plane, +since workers don't have a control-plane channel. + +Two submission paths: + +- **Regular spawn** — fresh output result_id; the child produces its own + result. Use when you want to continue and maybe pass the future to + another spawn. +- **Tail-call** — the worker dispatcher binds a returned ``TailPromise`` + to one of the parent's expected output ids and submits via + :meth:`WorkerSession._submit_tail`. The child writes directly to the + parent's output id, which ArmoniK delivers to whoever was awaiting + the parent's result. + +Intermediate futures created by ``.spawn()`` carry only ``result_id`` / +``task_id`` — the worker has no poller, so ``.result()`` on them raises. +They're useful for passing into further ``.spawn()`` calls (creating +data_dependencies edges inside the DAG). + +Submission for ``.spawn()`` goes through +:func:`pymonik._internal.submit.submit_many`. Tail-call submissions are +single, output-id-pinned, and bypass the pipeline's allocation step — +:meth:`_submit_tail` does its own envelope build + create_results + +submit_tasks. +""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING, Any + +import cloudpickle +from armonik.common import TaskDefinition, TaskOptions + +from pymonik import blob as blob_mod +from pymonik import envelope as env_mod +from pymonik._internal._logging import get_logger +from pymonik._internal._otel import current_trace_id_hex, inject_context +from pymonik._internal.refs import auto_spill, extract_deps +from pymonik._internal.submit import submit_many +from pymonik.envelope import EnvSpec, TaskEnvelope +from pymonik.errors import PymonikError +from pymonik.future import Future, FutureList, MultiResultHandle +from pymonik.options import EMPTY + +# Same default as the client-side session; see session._DEFAULT_SPILL_THRESHOLD. +_DEFAULT_SPILL_THRESHOLD = 256 * 1024 + +if TYPE_CHECKING: + from armonik.worker import TaskHandler + from pymonik.multiresult import TailPromise + from pymonik.task import Task + +log = get_logger(__name__) + + +class WorkerSession: + """Session facade that submits via the agent sidecar. + + Installed as the ``_current_session`` ContextVar for the duration of a + @task function's execution on a worker. + """ + + __slots__ = ("_th", "_parent_output_ids", "_blob_cache", "_spill_threshold") + + def __init__( + self, + task_handler: "TaskHandler", + *, + parent_output_ids: list[str], + ) -> None: + self._th = task_handler + self._parent_output_ids = parent_output_ids + self._blob_cache: dict[str, str] = {} + self._spill_threshold = _DEFAULT_SPILL_THRESHOLD + + @property + def session_id(self) -> str: + return self._th.session_id + + @property + def parent_output_ids(self) -> list[str]: + return self._parent_output_ids + + def _upload_blob(self, data: bytes) -> str: + """Upload via the agent sidecar; dedup within this worker's invocation.""" + h = blob_mod.content_hash(data) + cached = self._blob_cache.get(h) + if cached is not None: + return cached + name = f"{self.session_id}__blob__{h[:16]}" + result_map = self._th.create_results(results_data={name: data}) + rid = result_map[name].result_id + self._blob_cache[h] = rid + log.info("blob uploaded (worker)", hash=h[:16], size=len(data), result_id=rid) + return rid + + # ---- submission ---- + + def _submit_one( + self, + task: "Task[Any, Any]", + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + """Eager spawn — submits with fresh output id(s). + + Returns ``Future`` for single-output tasks, ``MultiResultHandle`` + for multi-output. Both are worker-stub flavoured (can't be + awaited; pass to other ``.spawn()``s or ignore). + """ + return self._submit_many(task, [(args, kwargs)])[0] + + def _submit_many( + self, + task: "Task[Any, Any]", + calls: list[Any], + ) -> FutureList[Any]: + """Submit N invocations via the shared pipeline.""" + backend = _AgentBackend(self) + multi_fields = task.multi_fields + + def make_stub( + task_id: str, + output_ids: list[str], + _args: tuple[Any, ...], + _kwargs: dict[str, Any], + ) -> Any: + if multi_fields: + field_to_future = { + field: Future._new_worker_stub( + session=self, task_id=task_id, result_id=oid + ) + for field, oid in zip(multi_fields, output_ids) + } + return MultiResultHandle(self, task_id, field_to_future) + return Future._new_worker_stub( + session=self, task_id=task_id, result_id=output_ids[0] + ) + + return submit_many( + task=task, + calls=calls, + backend=backend, + blob_uploader=self._upload_blob, + spill_threshold=self._spill_threshold, + default_opts=EMPTY, + partition="", + future_factory=make_stub, + apply_retry_policy=False, + attempt=1, + ) + + def _submit_tail( + self, + promise: "TailPromise[Any]", + *, + expected_output_ids: list[str], + ) -> str: + """Submit a tail-call promise with caller-supplied output ids. + + Bypasses :func:`submit_many` because the parent already owns the + output ids. Submits via the agent sidecar exactly the same way + :class:`_AgentBackend` does, but without going through the + allocate-outputs step. + + Returns the new task id (mostly for logging — parent doesn't + await the child). + """ + task = promise._task + args = promise._args + kwargs = promise._kwargs + + deps: list[str] = [] + args_rewritten = tuple(extract_deps(a, deps) for a in args) + kwargs_rewritten = {k: extract_deps(v, deps) for k, v in kwargs.items()} + args_rewritten = tuple( + auto_spill(a, deps, upload_blob=self._upload_blob, threshold=self._spill_threshold) + for a in args_rewritten + ) + kwargs_rewritten = { + k: auto_spill(v, deps, upload_blob=self._upload_blob, threshold=self._spill_threshold) + for k, v in kwargs_rewritten.items() + } + + merged_opts = task.opts # workers don't carry session defaults + + env_dict = merged_opts.env or {} + env_spec: EnvSpec | None = None + if merged_opts.deps or env_dict: + env_spec = EnvSpec( + deps=tuple(merged_opts.deps or ()), + isolate=merged_opts.isolate if merged_opts.isolate is not None else False, + index_url=merged_opts.index_url or "", + env=tuple(sorted(env_dict.items())), + ) + + traceparent_carrier: dict[str, str] = {} + inject_context(traceparent_carrier) + + envelope = TaskEnvelope( + function_pickle=cloudpickle.dumps(task.func), + args_pickle=cloudpickle.dumps((args_rewritten, kwargs_rewritten)), + func_name=task.name, + attempt=1, + env_spec=env_spec, + otel_context=tuple(sorted(traceparent_carrier.items())), + multi_fields=task.multi_fields or (), + ) + + payload_name = f"{self.session_id}__pl__{task.name}__tail__{uuid.uuid4()}" + result_map = self._th.create_results( + results_data={payload_name: env_mod.encode(envelope)} + ) + payload_id = result_map[payload_name].result_id + + per_task_options = merged_opts.to_armonik(default_partition="") + + definition = TaskDefinition( + payload_id=payload_id, + expected_output_ids=expected_output_ids, + data_dependencies=sorted(set(deps)), + ) + + submitted = self._th.submit_tasks( + tasks=[definition], default_task_options=per_task_options + ) + new_task_id = submitted[0].id + log.info( + "tail submitted", + func=task.name, + child_task=new_task_id, + expected_outputs=expected_output_ids, + trace_id=current_trace_id_hex(), + ) + return new_task_id + + +class _AgentBackend: + """SubmissionBackend that routes through the agent sidecar TaskHandler.""" + + __slots__ = ("_ws",) + + def __init__(self, ws: WorkerSession) -> None: + self._ws = ws + + @property + def session_id(self) -> str: + return self._ws.session_id + + @property + def allowed_partitions(self) -> tuple[str, ...] | None: + return None + + def allocate_outputs(self, names: list[str]) -> list[str]: + m = self._ws._th.create_results_metadata(result_names=names) + return [m[n].result_id for n in names] + + def upload_payloads(self, named_data: dict[str, bytes]) -> dict[str, str]: + m = self._ws._th.create_results(results_data=named_data) + return {n: r.result_id for n, r in m.items()} + + def submit( + self, + definitions: list[TaskDefinition], + default_options: TaskOptions, + ) -> list[str]: + submitted = self._ws._th.submit_tasks( + tasks=definitions, default_task_options=default_options + ) + return [s.id for s in submitted] diff --git a/test_client/.python-version b/test_client/.python-version deleted file mode 100644 index 56d91d3..0000000 --- a/test_client/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.10.12 diff --git a/test_client/README.md b/test_client/README.md deleted file mode 100644 index 4809e43..0000000 --- a/test_client/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Test Client - -This project contains multiple example projects that work with an editable install of the PymoniK version in master. These could both serve as examples and also as a way to test-drive PymoniK during development. - -# TODO: - -- Remove the UV project structure, and instead just use uv scripts with dependencies for each test client. diff --git a/test_client/adaptive_vector_addition.py b/test_client/adaptive_vector_addition.py deleted file mode 100644 index 6de2f83..0000000 --- a/test_client/adaptive_vector_addition.py +++ /dev/null @@ -1,71 +0,0 @@ -import numpy as np -from pymonik import Pymonik, task - -# Define a threshold for vector size. -VECTOR_SIZE_THRESHOLD = 512 - -@task -def aggregate_results(result_1, result_2) -> np.ndarray: - return np.concatenate([result_1, result_2]) - -@task -def vec_add(a: np.ndarray, b: np.ndarray) -> np.ndarray: - """ - Adds two numpy vectors. If vectors are larger than VECTOR_SIZE_THRESHOLD, - it splits them into chunks and invokes itself as subtasks. - The results are then aggregated by the aggregate_results task. - """ - if not isinstance(a, np.ndarray) or not isinstance(b, np.ndarray): - raise TypeError("Inputs must be numpy arrays.") - - if a.shape != b.shape: - raise ValueError("Input vectors must have the same shape.") - - if a.size > VECTOR_SIZE_THRESHOLD: - # Split vectors into two chunks, ideally you'd have a chunk size and you'd split into miltiple chunks. - # A half-way split was chosen here to highlight subtasking. - mid_point = a.size // 2 - a1, a2 = np.split(a, [mid_point]) - b1, b2 = np.split(b, [mid_point]) - - # Invoke vec_add as subtasks for each chunk - # Use delegate=True for subtasking as shown in the example - result_handle1 = vec_add.invoke(a1, b1) - result_handle2 = vec_add.invoke(a2, b2) - - # Aggregate the results using the aggregate_results task - # Pass the result handles directly - return aggregate_results.invoke(result_handle1, result_handle2, delegate=True) - else: - # If vectors are small enough, perform addition directly - return a + b - -if __name__ == "__main__": - # Ensure you have an ArmoniK cluster running and accessible - # Set endpoint and partition name accordingly - # The environment needs numpy - - with Pymonik(endpoint="localhost:5001", partition="pymonik", environment={"pip": ["numpy"]}): - # Create large sample vectors - vector_size = 4096 # Example size larger than threshold - vec_a = np.arange(vector_size) - vec_b = np.arange(vector_size) * 2 - - print(f"Invoking vec_add with vector size: {vector_size}") - # Invoke the main task - final_result_handle = vec_add.invoke(vec_a, vec_b) - - # Wait for the final result and retrieve it - try: - final_result = final_result_handle.wait().get() - print(f"Result of vec_add task: {final_result}") - print(f"Expected result starts with: {np.arange(vector_size) + np.arange(vector_size) * 2}") - # Verify a small part of the result - # print(f"First 10 elements: {final_result[:10]}") - # print(f"Expected first 10: {(vec_a + vec_b)[:10]}") - assert np.array_equal(final_result, vec_a + vec_b) - print("Verification successful!") - except Exception as e: - print(f"An error occurred: {e}") - - print("PymoniK client finished.") diff --git a/test_client/estimate_pi.py b/test_client/estimate_pi.py deleted file mode 100644 index 08c6a75..0000000 --- a/test_client/estimate_pi.py +++ /dev/null @@ -1,35 +0,0 @@ -import random -from pymonik import Pymonik, task - -@task -def estimate_pi_partial(num_samples: int) -> tuple[int, int]: - """Generates random points and counts those inside the unit circle.""" - points_in_circle = 0 - for _ in range(num_samples): - x, y = random.random(), random.random() - if x*x + y*y <= 1.0: - points_in_circle += 1 - return (points_in_circle, num_samples) - -@task -def sum_results(x): - """Sums the samples to get an estimation of PI.""" - total_points_in_circle = sum(res[0] for res in x) - total_samples = sum(res[1] for res in x) - return 4.0 * total_points_in_circle / total_samples - -if __name__ == "__main__": - num_tasks = 100 # Number of parallel tasks - samples_per_task = 20000 # Samples per task - - with Pymonik(): - print(f"Submitting {num_tasks} parallel tasks for Pi estimation...") - - results = estimate_pi_partial.map_invoke([(samples_per_task,) for _ in range(num_tasks)]) - final_result = sum_results.invoke(results) - print("Waiting for all tasks to complete...") - final_result = final_result.wait().get() # TODO: streaming results - - # Calculate final Pi estimate - print(f"Estimated value of Pi: {final_result}") - diff --git a/test_client/lambda_tasks.py b/test_client/lambda_tasks.py deleted file mode 100644 index c6337a6..0000000 --- a/test_client/lambda_tasks.py +++ /dev/null @@ -1,12 +0,0 @@ -from pymonik import Pymonik, Task - -if __name__ == "__main__": - pymonik = Pymonik(endpoint="localhost:5001", partition="pymonik") - print("PymoniK client running..") - with pymonik: - try: - my_task = Task(lambda a, b: a+b, func_name="add") - result = my_task.invoke(1, 2).wait().get() - print(f"Result of add task: {result}") - except Exception as e: - print(f"Error: {e}") diff --git a/test_client/materialize_test.py b/test_client/materialize_test.py deleted file mode 100644 index b661b47..0000000 --- a/test_client/materialize_test.py +++ /dev/null @@ -1,207 +0,0 @@ -from pymonik import Pymonik, task, materialize, Materialize -import os -from pathlib import Path - -# === EXAMPLE 1: Basic file materialization === - -@task -def process_config_file(config_mat: Materialize): - """Process a configuration file that was materialized in the worker.""" - config_path = Path(config_mat.worker_path) - - if not config_path.exists(): - return f"Error: Config file not found at {config_path}" - - # Read and process the config file - with open(config_path, 'r') as f: - config_content = f.read() - - return f"Processed config from {config_path}: {len(config_content)} characters" - - -# === EXAMPLE 2: Directory materialization === - -@task -def process_dataset(dataset_mat: Materialize): - """Process a dataset directory that was materialized in the worker.""" - dataset_path = Path(dataset_mat.worker_path) - - if not dataset_path.exists(): - return f"Error: Dataset directory not found at {dataset_path}" - - # Count files in the dataset - file_count = sum(1 for _ in dataset_path.rglob('*') if _.is_file()) - total_size = sum(f.stat().st_size for f in dataset_path.rglob('*') if f.is_file()) - - return { - "dataset_path": str(dataset_path), - "file_count": file_count, - "total_size_bytes": total_size, - "hash": dataset_mat.content_hash - } - - -# === EXAMPLE 3: Multiple materializations === - -@task -def compare_datasets(dataset1_mat: Materialize, dataset2_mat: Materialize): - """Compare two materialized datasets.""" - - results = {} - - for name, mat in [("dataset1", dataset1_mat), ("dataset2", dataset2_mat)]: - dataset_path = Path(mat.worker_path) - - if dataset_path.exists(): - file_count = sum(1 for _ in dataset_path.rglob('*') if _.is_file()) - results[name] = { - "path": str(dataset_path), - "files": file_count, - "hash": mat.content_hash, - "exists": True - } - else: - results[name] = {"exists": False, "path": str(dataset_path)} - - return results - - -# === EXAMPLE 4: Using materialization with context === - -@task(require_context=True) -def advanced_file_processing(ctx, data_mat: Materialize, output_dir: str): - """Advanced processing with access to context for logging.""" - - ctx.logger.info(f"Processing materialized data: {data_mat.source_path}") - ctx.logger.info(f"Worker path: {data_mat.worker_path}") - ctx.logger.info(f"Content hash: {data_mat.content_hash}") - - data_path = Path(data_mat.worker_path) - - if not data_path.exists(): - ctx.logger.error(f"Data not found at {data_path}") - return {"success": False, "error": "Data not materialized"} - - # Process the data (example: copy files to output directory) - output_path = Path(output_dir) - output_path.mkdir(parents=True, exist_ok=True) - - processed_files = [] - if data_path.is_file(): - # Single file - import shutil - output_file = output_path / data_path.name - shutil.copy2(data_path, output_file) - processed_files.append(str(output_file)) - ctx.logger.info(f"Copied file to {output_file}") - else: - # Directory - import shutil - for file_path in data_path.rglob('*'): - if file_path.is_file(): - rel_path = file_path.relative_to(data_path) - output_file = output_path / rel_path - output_file.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(file_path, output_file) - processed_files.append(str(output_file)) - ctx.logger.info(f"Copied {len(processed_files)} files to {output_path}") - - return { - "success": True, - "input_hash": data_mat.content_hash, - "processed_files": processed_files, - "output_directory": str(output_path) - } - - -# === CLIENT USAGE EXAMPLES === - -if __name__ == "__main__": - - # Setup - create some test files/directories - test_dir = Path("./test_materials") - test_dir.mkdir(exist_ok=True) - - # Create a test config file - config_file = test_dir / "config.txt" - with open(config_file, "w") as f: - f.write("debug=true\nmax_workers=10\ntimeout=302\n") - - # Create a test dataset directory - dataset_dir = test_dir / "dataset" - dataset_dir.mkdir(exist_ok=True) - for i in range(5): - data_file = dataset_dir / f"data_{i}.txt" - with open(data_file, "w") as f: - f.write(f"Sample data file {i}\n" * (i + 1)) - - # Create a subdirectory in dataset - subdir = dataset_dir / "subdir" - subdir.mkdir(exist_ok=True) - (subdir / "nested_file.txt").write_text("Nested content") - - with Pymonik(endpoint="localhost:5001") as pk: - - print("=== Creating Materialize objects ===") - - # Example 1: Materialize a single config file - config_mat = materialize(config_file, "/tmp/worker_config.txt") - print(f"Config file hash: {config_mat.content_hash}") - - # Upload the materialize object - config_mat = pk.upload_materialize(config_mat, force_upload=True) - print(f"Config uploaded with result_id: {config_mat.result_id}") - - # Example 2: Materialize a directory (will be zipped) - dataset_mat = materialize(dataset_dir, "/tmp/worker_dataset") - print(f"Dataset directory hash: {dataset_mat.content_hash}") - - # Upload the dataset - dataset_mat = pk.upload_materialize(dataset_mat, force_upload=True) - print(f"Dataset uploaded with result_id: {dataset_mat.result_id}") - - # Example 3: Create another dataset for comparison - dataset2_dir = test_dir / "dataset2" - dataset2_dir.mkdir(exist_ok=True) - (dataset2_dir / "different_file.txt").write_text("Different content") - - dataset2_mat = materialize(dataset2_dir, "/tmp/worker_dataset2") - dataset2_mat = pk.upload_materialize(dataset2_mat, force_upload=True) - - print("\n=== Running tasks with materialized content ===") - - # Test 1: Process config file - result1 = process_config_file.invoke(config_mat).wait().get() - print(f"Config processing result: {result1}") - - # Test 2: Process dataset - result2 = process_dataset.invoke(dataset_mat).wait().get() - print(f"Dataset processing result: {result2}") - - # Test 3: Compare datasets - result3 = compare_datasets.invoke(dataset_mat, dataset2_mat).wait().get() - print(f"Dataset comparison result: {result3}") - - # Test 4: Advanced processing with context - result4 = advanced_file_processing.invoke( - dataset_mat, "/tmp/worker_output" - ).wait().get() - print(f"Advanced processing result: {result4}") - - print("\n=== Testing deduplication ===") - - # Create the same config file again - should reuse existing upload - config_file_2 = test_dir / "config_copy.txt" - with open(config_file_2, "w") as f: - f.write("debug=true\nmax_workers=10\ntimeout=300\n") # Same content - - config_mat_2 = materialize(config_file_2, "/tmp/worker_config_2.txt") - print(f"Second config hash: {config_mat_2.content_hash}") - print(f"Hashes match: {config_mat.content_hash == config_mat_2.content_hash}") - - # This should reuse the existing upload - config_mat_2 = pk.upload_materialize(config_mat_2) - print(f"Second config result_id: {config_mat_2.result_id}") - print(f"Result IDs match: {config_mat.result_id == config_mat_2.result_id}") - - print("\n=== All tests completed ===") diff --git a/test_client/pyproject.toml b/test_client/pyproject.toml deleted file mode 100644 index ae41e86..0000000 --- a/test_client/pyproject.toml +++ /dev/null @@ -1,14 +0,0 @@ -[project] -name = "test-client" -version = "0.1.0" -description = "Add your description here" -readme = "README.md" -requires-python = ">=3.10.12" -dependencies = [ - "debugpy>=1.8.14", - "numpy>=2.2.5", - "pymonik", -] - -[tool.uv.sources] -pymonik = { path = "../pymonik", editable = true } diff --git a/test_client/retrieve_object_test.py b/test_client/retrieve_object_test.py deleted file mode 100644 index 54428cc..0000000 --- a/test_client/retrieve_object_test.py +++ /dev/null @@ -1,101 +0,0 @@ - -from pymonik import Pymonik, PymonikContext, task - -@task(require_context=True) -def test_retrieve_with_unpickling(ctx: PymonikContext, result_id: str): - """Test retrieving and automatically unpickling an object.""" - ctx.logger.info(f"Testing retrieve_object with auto_unpickle=True for {result_id}") - - # This will retrieve and unpickle the object automatically - obj = ctx.retrieve_object(result_id) - - if obj is not None: - ctx.logger.info(f"Successfully retrieved and unpickled object: {obj}") - ctx.logger.info(f"Object type: {type(obj)}") - return f"Success: {obj}" - else: - ctx.logger.error("Failed to retrieve/unpickle object") - return "Failed" - -@task(require_context=True) -def test_retrieve_without_unpickling(ctx: PymonikContext, result_id: str): - """Test retrieving an object without unpickling.""" - ctx.logger.info(f"Testing retrieve_object with auto_unpickle=False for {result_id}") - - # Just retrieve the file, don't unpickle - success = ctx.retrieve_object(result_id, auto_unpickle=False) - - if success: - object_path = ctx.get_object_path(result_id) - ctx.logger.info(f"Successfully retrieved object to {object_path}") - - # Manually check file size - file_size = object_path.stat().st_size - ctx.logger.info(f"Retrieved file size: {file_size} bytes") - return f"Success: Retrieved {file_size} bytes to {object_path}" - else: - ctx.logger.error("Failed to retrieve object") - return "Failed" - -@task(require_context=True) -def test_check_exists(ctx: PymonikContext, result_id: str): - """Test the check_exists and force_retrieve functionality.""" - ctx.logger.info(f"Testing existence checking for {result_id}") - - # Check if object exists locally first - exists_initially = ctx.object_exists_locally(result_id) - ctx.logger.info(f"Object exists locally initially: {exists_initially}") - - # First retrieval (should actually retrieve from ArmoniK) - obj1 = ctx.retrieve_object(result_id, check_exists=True) - ctx.logger.info(f"First retrieval result: {obj1}") - - # Second retrieval (should use local copy) - obj2 = ctx.retrieve_object(result_id, check_exists=True) - ctx.logger.info(f"Second retrieval result (should be from cache): {obj2}") - - # Force retrieval (should retrieve from ArmoniK even though local copy exists) - obj3 = ctx.retrieve_object(result_id, check_exists=True, force_retrieve=True) - ctx.logger.info(f"Force retrieval result: {obj3}") - - return { - "existed_initially": exists_initially, - "first_retrieval": str(obj1), - "second_retrieval": str(obj2), - "force_retrieval": str(obj3), - "all_match": obj1 == obj2 == obj3 - } - -if __name__ == "__main__": - with Pymonik("localhost:5001") as pk: - # Create test objects - simple_obj = "Hello PymoniK!" - complex_obj = { - "data": list(range(100)), - "metadata": {"created_by": "test", "version": 1.0}, - "nested": {"deep": {"value": 42}} - } - - # Upload objects to ArmoniK - simple_handle = pk.put(simple_obj, "simple_test_obj") - complex_handle = pk.put(complex_obj, "complex_test_obj") - - print(f"Simple object result ID: {simple_handle.result_id}") - print(f"Complex object result ID: {complex_handle.result_id}") - - # Test 1: Basic retrieve with unpickling - print("\n=== Test 1: Retrieve with unpickling ===") - result1 = test_retrieve_with_unpickling.invoke(simple_handle.result_id).wait().get() - print(f"Result: {result1}") - - # Test 2: Retrieve without unpickling - print("\n=== Test 2: Retrieve without unpickling ===") - result2 = test_retrieve_without_unpickling.invoke(complex_handle.result_id).wait().get() - print(f"Result: {result2}") - - # Test 3: Test existence checking and caching - print("\n=== Test 3: Existence checking and caching ===") - result3 = test_check_exists.invoke(simple_handle.result_id).wait().get() - print(f"Result: {result3}") - - print("\n=== All tests completed ===") diff --git a/test_client/subtasking.py b/test_client/subtasking.py deleted file mode 100644 index 3b6e9a5..0000000 --- a/test_client/subtasking.py +++ /dev/null @@ -1,29 +0,0 @@ -from pymonik import Pymonik, task - -@task -def add_one(a:int) -> int: - """ - A simple task that adds one to an integer. - """ - return a + 1 - -@task -def add(a: int, b: int) -> int: - """ - A simple task that adds two integers. - """ - if a <= 0: - return b - b_plus_one = add_one.invoke(b) - return add.invoke(a-1, b_plus_one, delegate=True) - - -if __name__ == "__main__": - pymonik = Pymonik(endpoint="localhost:5001", partition="pymonik") - print("PymoniK client running..") - with pymonik: - # try: - result = add.invoke(8, 2).wait().get() - print(f"Result of add task: {result}") - # except Exception as e: - # print(f"Error: {e}") diff --git a/test_client/task_options.py b/test_client/task_options.py deleted file mode 100644 index 23efdf9..0000000 --- a/test_client/task_options.py +++ /dev/null @@ -1,81 +0,0 @@ -from datetime import timedelta -import numpy as np -from pymonik import Pymonik, task -import debugpy -from armonik.common.objects import TaskOptions - -# debugpy.listen(("localhost", 5678)) -# debugpy.wait_for_client() -# Define a threshold for vector size. -VECTOR_SIZE_THRESHOLD = 512 - -@task -def aggregate_results(result_1, result_2) -> np.ndarray: - return np.concatenate([result_1, result_2]) - -@task(require_context=True) -def vec_add(ctx, a: np.ndarray, b: np.ndarray, **kwargs) -> np.ndarray: - """ - Adds two numpy vectors. If vectors are larger than VECTOR_SIZE_THRESHOLD, - it splits them into chunks and invokes itself as subtasks. - The results are then aggregated by the aggregate_results task. - """ - - partition_name = ctx.task_handler.task_options.partition_id - ctx.logger.info(f"Executing vec add in partition {partition_name}") - - # if not isinstance(a, np.ndarray) or not isinstance(b, np.ndarray): - # raise TypeError("Inputs must be numpy arrays.") - - if a.shape != b.shape: - raise ValueError("Input vectors must have the same shape.") - - if a.size > VECTOR_SIZE_THRESHOLD: - # Split vectors into two chunks, ideally you'd have a chunk size and you'd split into miltiple chunks. - # A half-way split was chosen here to highlight subtasking. - mid_point = a.size // 2 - a1, a2 = np.split(a, [mid_point]) - b1, b2 = np.split(b, [mid_point]) - - # Invoke vec_add as subtasks for each chunk - # Use delegate=True for subtasking as shown in the example - result_handle1 = vec_add.invoke(a1, b1, pmk_partition_id="pymonik2") - result_handle2 = vec_add.invoke(a2, b2, pmk_partition_id="pymonik2") - - # Aggregate the results using the aggregate_results task - # Pass the result handles directly - return aggregate_results.invoke(result_handle1, result_handle2, delegate=True) - else: - # If vectors are small enough, perform addition directly - return a + b - -if __name__ == "__main__": - # Ensure you have an ArmoniK cluster running and accessible - # Set endpoint and partition name accordingly - # The environment needs numpy - - - with Pymonik(endpoint="localhost:5001", partition=["pymonik", "pymonik2"], environment={"pip": ["numpy"]}) as pymonik: - # Create large sample vectors - vector_size = 4096 # Example size larger than threshold - vec_a = np.arange(vector_size) - vec_b = np.arange(vector_size) * 2 - - print(f"Invoking vec_add with vector size: {vector_size}") - # Invoke the main task - final_result_handle = vec_add.invoke(vec_a, vec_b, task_options= TaskOptions(partition_id="pymonik2", max_duration=timedelta(seconds=12), priority=1, max_retries=2)) - - # Wait for the final result and retrieve it - try: - final_result = final_result_handle.wait().get() - print(f"Result of vec_add task: {final_result}") - print(f"Expected result starts with: {np.arange(vector_size) + np.arange(vector_size) * 2}") - # Verify a small part of the result - # print(f"First 10 elements: {final_result[:10]}") - # print(f"Expected first 10: {(vec_a + vec_b)[:10]}") - assert np.array_equal(final_result, vec_a + vec_b) - print("Verification successful!") - except Exception as e: - print(f"An error occurred: {e}") - - print("PymoniK client finished.") diff --git a/test_client/uploading_objects.py b/test_client/uploading_objects.py deleted file mode 100644 index 9f05d2c..0000000 --- a/test_client/uploading_objects.py +++ /dev/null @@ -1,16 +0,0 @@ -from pymonik import Pymonik, task - -@task -def add_one(x): - return x + 1 - -if __name__ == "__main__": - pymonik = Pymonik(endpoint="localhost:5001", partition="pymonik") - print("PymoniK client running..") - with pymonik: - try: - ref = pymonik.put(41) - result = add_one.invoke(ref).wait().get() - print(f"Result of add_one task: {result}") - except Exception as e: - print(f"Error: {e}") diff --git a/test_client/uv.lock b/test_client/uv.lock deleted file mode 100644 index ad2156f..0000000 --- a/test_client/uv.lock +++ /dev/null @@ -1,423 +0,0 @@ -version = 1 -revision = 2 -requires-python = ">=3.10.12" - -[[package]] -name = "armonik" -version = "3.25.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "deprecation" }, - { name = "grpcio" }, - { name = "grpcio-tools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8e/22/6299b9be5da13e6a909a3dc285e1b4b1b4990da337e598d4863acdc723aa/armonik-3.25.0.tar.gz", hash = "sha256:5fd5114da313b279d28fccdcb4cd1e6e1d0d302f5cf01b83d2ebd888ef1982c9", size = 89599, upload-time = "2025-03-06T14:39:21.779Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/2d/03ba8cad0f8bbe0850cdbee37bc904a70a005552c0091f904a0244223dc4/armonik-3.25.0-py3-none-any.whl", hash = "sha256:2c4f4428264bcf0b4016599441eeaad13f8947cfd3e3398b5a75ef4fe4d70483", size = 148256, upload-time = "2025-03-06T14:39:20.077Z" }, -] - -[[package]] -name = "cffi" -version = "1.17.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/07/f44ca684db4e4f08a3fdc6eeb9a0d15dc6883efc7b8c90357fdbf74e186c/cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14", size = 182191, upload-time = "2024-09-04T20:43:30.027Z" }, - { url = "https://files.pythonhosted.org/packages/08/fd/cc2fedbd887223f9f5d170c96e57cbf655df9831a6546c1727ae13fa977a/cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67", size = 178592, upload-time = "2024-09-04T20:43:32.108Z" }, - { url = "https://files.pythonhosted.org/packages/de/cc/4635c320081c78d6ffc2cab0a76025b691a91204f4aa317d568ff9280a2d/cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382", size = 426024, upload-time = "2024-09-04T20:43:34.186Z" }, - { url = "https://files.pythonhosted.org/packages/b6/7b/3b2b250f3aab91abe5f8a51ada1b717935fdaec53f790ad4100fe2ec64d1/cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702", size = 448188, upload-time = "2024-09-04T20:43:36.286Z" }, - { url = "https://files.pythonhosted.org/packages/d3/48/1b9283ebbf0ec065148d8de05d647a986c5f22586b18120020452fff8f5d/cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3", size = 455571, upload-time = "2024-09-04T20:43:38.586Z" }, - { url = "https://files.pythonhosted.org/packages/40/87/3b8452525437b40f39ca7ff70276679772ee7e8b394934ff60e63b7b090c/cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6", size = 436687, upload-time = "2024-09-04T20:43:40.084Z" }, - { url = "https://files.pythonhosted.org/packages/8d/fb/4da72871d177d63649ac449aec2e8a29efe0274035880c7af59101ca2232/cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17", size = 446211, upload-time = "2024-09-04T20:43:41.526Z" }, - { url = "https://files.pythonhosted.org/packages/ab/a0/62f00bcb411332106c02b663b26f3545a9ef136f80d5df746c05878f8c4b/cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8", size = 461325, upload-time = "2024-09-04T20:43:43.117Z" }, - { url = "https://files.pythonhosted.org/packages/36/83/76127035ed2e7e27b0787604d99da630ac3123bfb02d8e80c633f218a11d/cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e", size = 438784, upload-time = "2024-09-04T20:43:45.256Z" }, - { url = "https://files.pythonhosted.org/packages/21/81/a6cd025db2f08ac88b901b745c163d884641909641f9b826e8cb87645942/cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be", size = 461564, upload-time = "2024-09-04T20:43:46.779Z" }, - { url = "https://files.pythonhosted.org/packages/f8/fe/4d41c2f200c4a457933dbd98d3cf4e911870877bd94d9656cc0fcb390681/cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c", size = 171804, upload-time = "2024-09-04T20:43:48.186Z" }, - { url = "https://files.pythonhosted.org/packages/d1/b6/0b0f5ab93b0df4acc49cae758c81fe4e5ef26c3ae2e10cc69249dfd8b3ab/cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15", size = 181299, upload-time = "2024-09-04T20:43:49.812Z" }, - { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264, upload-time = "2024-09-04T20:43:51.124Z" }, - { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651, upload-time = "2024-09-04T20:43:52.872Z" }, - { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259, upload-time = "2024-09-04T20:43:56.123Z" }, - { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200, upload-time = "2024-09-04T20:43:57.891Z" }, - { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235, upload-time = "2024-09-04T20:44:00.18Z" }, - { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721, upload-time = "2024-09-04T20:44:01.585Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242, upload-time = "2024-09-04T20:44:03.467Z" }, - { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999, upload-time = "2024-09-04T20:44:05.023Z" }, - { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242, upload-time = "2024-09-04T20:44:06.444Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604, upload-time = "2024-09-04T20:44:08.206Z" }, - { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727, upload-time = "2024-09-04T20:44:09.481Z" }, - { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400, upload-time = "2024-09-04T20:44:10.873Z" }, - { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, - { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, - { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, - { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, - { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, - { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, - { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, - { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, - { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, - { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989, upload-time = "2024-09-04T20:44:28.956Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802, upload-time = "2024-09-04T20:44:30.289Z" }, - { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792, upload-time = "2024-09-04T20:44:32.01Z" }, - { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893, upload-time = "2024-09-04T20:44:33.606Z" }, - { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810, upload-time = "2024-09-04T20:44:35.191Z" }, - { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200, upload-time = "2024-09-04T20:44:36.743Z" }, - { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447, upload-time = "2024-09-04T20:44:38.492Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358, upload-time = "2024-09-04T20:44:40.046Z" }, - { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469, upload-time = "2024-09-04T20:44:41.616Z" }, - { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475, upload-time = "2024-09-04T20:44:43.733Z" }, - { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, -] - -[[package]] -name = "cloudpickle" -version = "3.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/39/069100b84d7418bc358d81669d5748efb14b9cceacd2f9c75f550424132f/cloudpickle-3.1.1.tar.gz", hash = "sha256:b216fa8ae4019d5482a8ac3c95d8f6346115d8835911fd4aefd1a445e4242c64", size = 22113, upload-time = "2025-01-14T17:02:05.085Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/e8/64c37fadfc2816a7701fa8a6ed8d87327c7d54eacfbfb6edab14a2f2be75/cloudpickle-3.1.1-py3-none-any.whl", hash = "sha256:c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e", size = 20992, upload-time = "2025-01-14T17:02:02.417Z" }, -] - -[[package]] -name = "cryptography" -version = "44.0.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cd/25/4ce80c78963834b8a9fd1cc1266be5ed8d1840785c0f2e1b73b8d128d505/cryptography-44.0.2.tar.gz", hash = "sha256:c63454aa261a0cf0c5b4718349629793e9e634993538db841165b3df74f37ec0", size = 710807, upload-time = "2025-03-02T00:01:37.692Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/ef/83e632cfa801b221570c5f58c0369db6fa6cef7d9ff859feab1aae1a8a0f/cryptography-44.0.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:efcfe97d1b3c79e486554efddeb8f6f53a4cdd4cf6086642784fa31fc384e1d7", size = 6676361, upload-time = "2025-03-02T00:00:06.528Z" }, - { url = "https://files.pythonhosted.org/packages/30/ec/7ea7c1e4c8fc8329506b46c6c4a52e2f20318425d48e0fe597977c71dbce/cryptography-44.0.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29ecec49f3ba3f3849362854b7253a9f59799e3763b0c9d0826259a88efa02f1", size = 3952350, upload-time = "2025-03-02T00:00:09.537Z" }, - { url = "https://files.pythonhosted.org/packages/27/61/72e3afdb3c5ac510330feba4fc1faa0fe62e070592d6ad00c40bb69165e5/cryptography-44.0.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc821e161ae88bfe8088d11bb39caf2916562e0a2dc7b6d56714a48b784ef0bb", size = 4166572, upload-time = "2025-03-02T00:00:12.03Z" }, - { url = "https://files.pythonhosted.org/packages/26/e4/ba680f0b35ed4a07d87f9e98f3ebccb05091f3bf6b5a478b943253b3bbd5/cryptography-44.0.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3c00b6b757b32ce0f62c574b78b939afab9eecaf597c4d624caca4f9e71e7843", size = 3958124, upload-time = "2025-03-02T00:00:14.518Z" }, - { url = "https://files.pythonhosted.org/packages/9c/e8/44ae3e68c8b6d1cbc59040288056df2ad7f7f03bbcaca6b503c737ab8e73/cryptography-44.0.2-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7bdcd82189759aba3816d1f729ce42ffded1ac304c151d0a8e89b9996ab863d5", size = 3678122, upload-time = "2025-03-02T00:00:17.212Z" }, - { url = "https://files.pythonhosted.org/packages/27/7b/664ea5e0d1eab511a10e480baf1c5d3e681c7d91718f60e149cec09edf01/cryptography-44.0.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4973da6ca3db4405c54cd0b26d328be54c7747e89e284fcff166132eb7bccc9c", size = 4191831, upload-time = "2025-03-02T00:00:19.696Z" }, - { url = "https://files.pythonhosted.org/packages/2a/07/79554a9c40eb11345e1861f46f845fa71c9e25bf66d132e123d9feb8e7f9/cryptography-44.0.2-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:4e389622b6927d8133f314949a9812972711a111d577a5d1f4bee5e58736b80a", size = 3960583, upload-time = "2025-03-02T00:00:22.488Z" }, - { url = "https://files.pythonhosted.org/packages/bb/6d/858e356a49a4f0b591bd6789d821427de18432212e137290b6d8a817e9bf/cryptography-44.0.2-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:f514ef4cd14bb6fb484b4a60203e912cfcb64f2ab139e88c2274511514bf7308", size = 4191753, upload-time = "2025-03-02T00:00:25.038Z" }, - { url = "https://files.pythonhosted.org/packages/b2/80/62df41ba4916067fa6b125aa8c14d7e9181773f0d5d0bd4dcef580d8b7c6/cryptography-44.0.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1bc312dfb7a6e5d66082c87c34c8a62176e684b6fe3d90fcfe1568de675e6688", size = 4079550, upload-time = "2025-03-02T00:00:26.929Z" }, - { url = "https://files.pythonhosted.org/packages/f3/cd/2558cc08f7b1bb40683f99ff4327f8dcfc7de3affc669e9065e14824511b/cryptography-44.0.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b721b8b4d948b218c88cb8c45a01793483821e709afe5f622861fc6182b20a7", size = 4298367, upload-time = "2025-03-02T00:00:28.735Z" }, - { url = "https://files.pythonhosted.org/packages/71/59/94ccc74788945bc3bd4cf355d19867e8057ff5fdbcac781b1ff95b700fb1/cryptography-44.0.2-cp37-abi3-win32.whl", hash = "sha256:51e4de3af4ec3899d6d178a8c005226491c27c4ba84101bfb59c901e10ca9f79", size = 2772843, upload-time = "2025-03-02T00:00:30.592Z" }, - { url = "https://files.pythonhosted.org/packages/ca/2c/0d0bbaf61ba05acb32f0841853cfa33ebb7a9ab3d9ed8bb004bd39f2da6a/cryptography-44.0.2-cp37-abi3-win_amd64.whl", hash = "sha256:c505d61b6176aaf982c5717ce04e87da5abc9a36a5b39ac03905c4aafe8de7aa", size = 3209057, upload-time = "2025-03-02T00:00:33.393Z" }, - { url = "https://files.pythonhosted.org/packages/9e/be/7a26142e6d0f7683d8a382dd963745e65db895a79a280a30525ec92be890/cryptography-44.0.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:8e0ddd63e6bf1161800592c71ac794d3fb8001f2caebe0966e77c5234fa9efc3", size = 6677789, upload-time = "2025-03-02T00:00:36.009Z" }, - { url = "https://files.pythonhosted.org/packages/06/88/638865be7198a84a7713950b1db7343391c6066a20e614f8fa286eb178ed/cryptography-44.0.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81276f0ea79a208d961c433a947029e1a15948966658cf6710bbabb60fcc2639", size = 3951919, upload-time = "2025-03-02T00:00:38.581Z" }, - { url = "https://files.pythonhosted.org/packages/d7/fc/99fe639bcdf58561dfad1faa8a7369d1dc13f20acd78371bb97a01613585/cryptography-44.0.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a1e657c0f4ea2a23304ee3f964db058c9e9e635cc7019c4aa21c330755ef6fd", size = 4167812, upload-time = "2025-03-02T00:00:42.934Z" }, - { url = "https://files.pythonhosted.org/packages/53/7b/aafe60210ec93d5d7f552592a28192e51d3c6b6be449e7fd0a91399b5d07/cryptography-44.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6210c05941994290f3f7f175a4a57dbbb2afd9273657614c506d5976db061181", size = 3958571, upload-time = "2025-03-02T00:00:46.026Z" }, - { url = "https://files.pythonhosted.org/packages/16/32/051f7ce79ad5a6ef5e26a92b37f172ee2d6e1cce09931646eef8de1e9827/cryptography-44.0.2-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d1c3572526997b36f245a96a2b1713bf79ce99b271bbcf084beb6b9b075f29ea", size = 3679832, upload-time = "2025-03-02T00:00:48.647Z" }, - { url = "https://files.pythonhosted.org/packages/78/2b/999b2a1e1ba2206f2d3bca267d68f350beb2b048a41ea827e08ce7260098/cryptography-44.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b042d2a275c8cee83a4b7ae30c45a15e6a4baa65a179a0ec2d78ebb90e4f6699", size = 4193719, upload-time = "2025-03-02T00:00:51.397Z" }, - { url = "https://files.pythonhosted.org/packages/72/97/430e56e39a1356e8e8f10f723211a0e256e11895ef1a135f30d7d40f2540/cryptography-44.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:d03806036b4f89e3b13b6218fefea8d5312e450935b1a2d55f0524e2ed7c59d9", size = 3960852, upload-time = "2025-03-02T00:00:53.317Z" }, - { url = "https://files.pythonhosted.org/packages/89/33/c1cf182c152e1d262cac56850939530c05ca6c8d149aa0dcee490b417e99/cryptography-44.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c7362add18b416b69d58c910caa217f980c5ef39b23a38a0880dfd87bdf8cd23", size = 4193906, upload-time = "2025-03-02T00:00:56.49Z" }, - { url = "https://files.pythonhosted.org/packages/e1/99/87cf26d4f125380dc674233971069bc28d19b07f7755b29861570e513650/cryptography-44.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8cadc6e3b5a1f144a039ea08a0bdb03a2a92e19c46be3285123d32029f40a922", size = 4081572, upload-time = "2025-03-02T00:00:59.995Z" }, - { url = "https://files.pythonhosted.org/packages/b3/9f/6a3e0391957cc0c5f84aef9fbdd763035f2b52e998a53f99345e3ac69312/cryptography-44.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6f101b1f780f7fc613d040ca4bdf835c6ef3b00e9bd7125a4255ec574c7916e4", size = 4298631, upload-time = "2025-03-02T00:01:01.623Z" }, - { url = "https://files.pythonhosted.org/packages/e2/a5/5bc097adb4b6d22a24dea53c51f37e480aaec3465285c253098642696423/cryptography-44.0.2-cp39-abi3-win32.whl", hash = "sha256:3dc62975e31617badc19a906481deacdeb80b4bb454394b4098e3f2525a488c5", size = 2773792, upload-time = "2025-03-02T00:01:04.133Z" }, - { url = "https://files.pythonhosted.org/packages/33/cf/1f7649b8b9a3543e042d3f348e398a061923ac05b507f3f4d95f11938aa9/cryptography-44.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:5f6f90b72d8ccadb9c6e311c775c8305381db88374c65fa1a68250aa8a9cb3a6", size = 3210957, upload-time = "2025-03-02T00:01:06.987Z" }, - { url = "https://files.pythonhosted.org/packages/99/10/173be140714d2ebaea8b641ff801cbcb3ef23101a2981cbf08057876f89e/cryptography-44.0.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:af4ff3e388f2fa7bff9f7f2b31b87d5651c45731d3e8cfa0944be43dff5cfbdb", size = 3396886, upload-time = "2025-03-02T00:01:09.51Z" }, - { url = "https://files.pythonhosted.org/packages/2f/b4/424ea2d0fce08c24ede307cead3409ecbfc2f566725d4701b9754c0a1174/cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0529b1d5a0105dd3731fa65680b45ce49da4d8115ea76e9da77a875396727b41", size = 3892387, upload-time = "2025-03-02T00:01:11.348Z" }, - { url = "https://files.pythonhosted.org/packages/28/20/8eaa1a4f7c68a1cb15019dbaad59c812d4df4fac6fd5f7b0b9c5177f1edd/cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7ca25849404be2f8e4b3c59483d9d3c51298a22c1c61a0e84415104dacaf5562", size = 4109922, upload-time = "2025-03-02T00:01:13.934Z" }, - { url = "https://files.pythonhosted.org/packages/11/25/5ed9a17d532c32b3bc81cc294d21a36c772d053981c22bd678396bc4ae30/cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:268e4e9b177c76d569e8a145a6939eca9a5fec658c932348598818acf31ae9a5", size = 3895715, upload-time = "2025-03-02T00:01:16.895Z" }, - { url = "https://files.pythonhosted.org/packages/63/31/2aac03b19c6329b62c45ba4e091f9de0b8f687e1b0cd84f101401bece343/cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:9eb9d22b0a5d8fd9925a7764a054dca914000607dff201a24c791ff5c799e1fa", size = 4109876, upload-time = "2025-03-02T00:01:18.751Z" }, - { url = "https://files.pythonhosted.org/packages/99/ec/6e560908349843718db1a782673f36852952d52a55ab14e46c42c8a7690a/cryptography-44.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2bf7bf75f7df9715f810d1b038870309342bff3069c5bd8c6b96128cb158668d", size = 3131719, upload-time = "2025-03-02T00:01:21.269Z" }, - { url = "https://files.pythonhosted.org/packages/d6/d7/f30e75a6aa7d0f65031886fa4a1485c2fbfe25a1896953920f6a9cfe2d3b/cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:909c97ab43a9c0c0b0ada7a1281430e4e5ec0458e6d9244c0e821bbf152f061d", size = 3887513, upload-time = "2025-03-02T00:01:22.911Z" }, - { url = "https://files.pythonhosted.org/packages/9c/b4/7a494ce1032323ca9db9a3661894c66e0d7142ad2079a4249303402d8c71/cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:96e7a5e9d6e71f9f4fca8eebfd603f8e86c5225bb18eb621b2c1e50b290a9471", size = 4107432, upload-time = "2025-03-02T00:01:24.701Z" }, - { url = "https://files.pythonhosted.org/packages/45/f8/6b3ec0bc56123b344a8d2b3264a325646d2dcdbdd9848b5e6f3d37db90b3/cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d1b3031093a366ac767b3feb8bcddb596671b3aaff82d4050f984da0c248b615", size = 3891421, upload-time = "2025-03-02T00:01:26.335Z" }, - { url = "https://files.pythonhosted.org/packages/57/ff/f3b4b2d007c2a646b0f69440ab06224f9cf37a977a72cdb7b50632174e8a/cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:04abd71114848aa25edb28e225ab5f268096f44cf0127f3d36975bdf1bdf3390", size = 4107081, upload-time = "2025-03-02T00:01:28.938Z" }, -] - -[[package]] -name = "debugpy" -version = "1.8.14" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/75/087fe07d40f490a78782ff3b0a30e3968936854105487decdb33446d4b0e/debugpy-1.8.14.tar.gz", hash = "sha256:7cd287184318416850aa8b60ac90105837bb1e59531898c07569d197d2ed5322", size = 1641444, upload-time = "2025-04-10T19:46:10.981Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/df/156df75a41aaebd97cee9d3870fe68f8001b6c1c4ca023e221cfce69bece/debugpy-1.8.14-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:93fee753097e85623cab1c0e6a68c76308cd9f13ffdf44127e6fab4fbf024339", size = 2076510, upload-time = "2025-04-10T19:46:13.315Z" }, - { url = "https://files.pythonhosted.org/packages/69/cd/4fc391607bca0996db5f3658762106e3d2427beaef9bfd363fd370a3c054/debugpy-1.8.14-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d937d93ae4fa51cdc94d3e865f535f185d5f9748efb41d0d49e33bf3365bd79", size = 3559614, upload-time = "2025-04-10T19:46:14.647Z" }, - { url = "https://files.pythonhosted.org/packages/1a/42/4e6d2b9d63e002db79edfd0cb5656f1c403958915e0e73ab3e9220012eec/debugpy-1.8.14-cp310-cp310-win32.whl", hash = "sha256:c442f20577b38cc7a9aafecffe1094f78f07fb8423c3dddb384e6b8f49fd2987", size = 5208588, upload-time = "2025-04-10T19:46:16.233Z" }, - { url = "https://files.pythonhosted.org/packages/97/b1/cc9e4e5faadc9d00df1a64a3c2d5c5f4b9df28196c39ada06361c5141f89/debugpy-1.8.14-cp310-cp310-win_amd64.whl", hash = "sha256:f117dedda6d969c5c9483e23f573b38f4e39412845c7bc487b6f2648df30fe84", size = 5241043, upload-time = "2025-04-10T19:46:17.768Z" }, - { url = "https://files.pythonhosted.org/packages/67/e8/57fe0c86915671fd6a3d2d8746e40485fd55e8d9e682388fbb3a3d42b86f/debugpy-1.8.14-cp311-cp311-macosx_14_0_universal2.whl", hash = "sha256:1b2ac8c13b2645e0b1eaf30e816404990fbdb168e193322be8f545e8c01644a9", size = 2175064, upload-time = "2025-04-10T19:46:19.486Z" }, - { url = "https://files.pythonhosted.org/packages/3b/97/2b2fd1b1c9569c6764ccdb650a6f752e4ac31be465049563c9eb127a8487/debugpy-1.8.14-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf431c343a99384ac7eab2f763980724834f933a271e90496944195318c619e2", size = 3132359, upload-time = "2025-04-10T19:46:21.192Z" }, - { url = "https://files.pythonhosted.org/packages/c0/ee/b825c87ed06256ee2a7ed8bab8fb3bb5851293bf9465409fdffc6261c426/debugpy-1.8.14-cp311-cp311-win32.whl", hash = "sha256:c99295c76161ad8d507b413cd33422d7c542889fbb73035889420ac1fad354f2", size = 5133269, upload-time = "2025-04-10T19:46:23.047Z" }, - { url = "https://files.pythonhosted.org/packages/d5/a6/6c70cd15afa43d37839d60f324213843174c1d1e6bb616bd89f7c1341bac/debugpy-1.8.14-cp311-cp311-win_amd64.whl", hash = "sha256:7816acea4a46d7e4e50ad8d09d963a680ecc814ae31cdef3622eb05ccacf7b01", size = 5158156, upload-time = "2025-04-10T19:46:24.521Z" }, - { url = "https://files.pythonhosted.org/packages/d9/2a/ac2df0eda4898f29c46eb6713a5148e6f8b2b389c8ec9e425a4a1d67bf07/debugpy-1.8.14-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:8899c17920d089cfa23e6005ad9f22582fd86f144b23acb9feeda59e84405b84", size = 2501268, upload-time = "2025-04-10T19:46:26.044Z" }, - { url = "https://files.pythonhosted.org/packages/10/53/0a0cb5d79dd9f7039169f8bf94a144ad3efa52cc519940b3b7dde23bcb89/debugpy-1.8.14-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6bb5c0dcf80ad5dbc7b7d6eac484e2af34bdacdf81df09b6a3e62792b722826", size = 4221077, upload-time = "2025-04-10T19:46:27.464Z" }, - { url = "https://files.pythonhosted.org/packages/f8/d5/84e01821f362327bf4828728aa31e907a2eca7c78cd7c6ec062780d249f8/debugpy-1.8.14-cp312-cp312-win32.whl", hash = "sha256:281d44d248a0e1791ad0eafdbbd2912ff0de9eec48022a5bfbc332957487ed3f", size = 5255127, upload-time = "2025-04-10T19:46:29.467Z" }, - { url = "https://files.pythonhosted.org/packages/33/16/1ed929d812c758295cac7f9cf3dab5c73439c83d9091f2d91871e648093e/debugpy-1.8.14-cp312-cp312-win_amd64.whl", hash = "sha256:5aa56ef8538893e4502a7d79047fe39b1dae08d9ae257074c6464a7b290b806f", size = 5297249, upload-time = "2025-04-10T19:46:31.538Z" }, - { url = "https://files.pythonhosted.org/packages/4d/e4/395c792b243f2367d84202dc33689aa3d910fb9826a7491ba20fc9e261f5/debugpy-1.8.14-cp313-cp313-macosx_14_0_universal2.whl", hash = "sha256:329a15d0660ee09fec6786acdb6e0443d595f64f5d096fc3e3ccf09a4259033f", size = 2485676, upload-time = "2025-04-10T19:46:32.96Z" }, - { url = "https://files.pythonhosted.org/packages/ba/f1/6f2ee3f991327ad9e4c2f8b82611a467052a0fb0e247390192580e89f7ff/debugpy-1.8.14-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f920c7f9af409d90f5fd26e313e119d908b0dd2952c2393cd3247a462331f15", size = 4217514, upload-time = "2025-04-10T19:46:34.336Z" }, - { url = "https://files.pythonhosted.org/packages/79/28/b9d146f8f2dc535c236ee09ad3e5ac899adb39d7a19b49f03ac95d216beb/debugpy-1.8.14-cp313-cp313-win32.whl", hash = "sha256:3784ec6e8600c66cbdd4ca2726c72d8ca781e94bce2f396cc606d458146f8f4e", size = 5254756, upload-time = "2025-04-10T19:46:36.199Z" }, - { url = "https://files.pythonhosted.org/packages/e0/62/a7b4a57013eac4ccaef6977966e6bec5c63906dd25a86e35f155952e29a1/debugpy-1.8.14-cp313-cp313-win_amd64.whl", hash = "sha256:684eaf43c95a3ec39a96f1f5195a7ff3d4144e4a18d69bb66beeb1a6de605d6e", size = 5297119, upload-time = "2025-04-10T19:46:38.141Z" }, - { url = "https://files.pythonhosted.org/packages/97/1a/481f33c37ee3ac8040d3d51fc4c4e4e7e61cb08b8bc8971d6032acc2279f/debugpy-1.8.14-py2.py3-none-any.whl", hash = "sha256:5cd9a579d553b6cb9759a7908a41988ee6280b961f24f63336835d9418216a20", size = 5256230, upload-time = "2025-04-10T19:46:54.077Z" }, -] - -[[package]] -name = "deprecation" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788, upload-time = "2020-04-20T14:23:38.738Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178, upload-time = "2020-04-20T14:23:36.581Z" }, -] - -[[package]] -name = "grpcio" -version = "1.62.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/31/2e/1e2cd0edeaaeaae0ab9df2615492725d0f2f689d3f9aa3b088356af7a584/grpcio-1.62.3.tar.gz", hash = "sha256:4439bbd759636e37b66841117a66444b454937e27f0125205d2d117d7827c643", size = 26297933, upload-time = "2024-08-06T00:32:51.086Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/86/f1aa615ee551e8b4f59b0d1189a09e16eefb3d243487115ab7be56eecbec/grpcio-1.62.3-cp310-cp310-linux_armv7l.whl", hash = "sha256:13571a5b868dcc308a55d36669a2d17d9dcd6ec8335213f6c49cc68da7305abe", size = 4766342, upload-time = "2024-08-06T00:20:32.775Z" }, - { url = "https://files.pythonhosted.org/packages/c5/63/ee244c4b64f0e71cef5314f9fa1d120c072e33c2e4c545dc75bd1af2a5c5/grpcio-1.62.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:f5def814c5a4c90c8fe389c526ab881f4a28b7e239b23ed8e02dd02934dfaa1a", size = 9991627, upload-time = "2024-08-06T00:20:35.662Z" }, - { url = "https://files.pythonhosted.org/packages/70/69/23bd58a27c472221fc340dd08eee2becf1a2c9d27d00e279c78a6b6f53cc/grpcio-1.62.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:7349cd7445ac65fbe1b744dcab9cc1ec02dae2256941a2e67895926cbf7422b4", size = 5290817, upload-time = "2024-08-06T00:20:38.718Z" }, - { url = "https://files.pythonhosted.org/packages/74/3a/875be32d9d1516398049ebc39cc6e7620d50d807093ce624f0469cee5e51/grpcio-1.62.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:646c14e9f3356d3f34a65b58b0f8d08daa741ba1d4fcd4966b79407543332154", size = 5829374, upload-time = "2024-08-06T00:20:41.631Z" }, - { url = "https://files.pythonhosted.org/packages/58/2f/f3fc773270cf17e7ca076c1f6435278f58641d475a25cdeea5b2d8d4845b/grpcio-1.62.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:807176971c504c598976f5a9ea62363cffbbbb6c7509d9808c2342b020880fa2", size = 5549649, upload-time = "2024-08-06T00:20:44.562Z" }, - { url = "https://files.pythonhosted.org/packages/b4/dc/deb8b1da1fa6111c3f44253433faf977678dea7dd381ce397ee33a1b4d8c/grpcio-1.62.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:43670a25b752b7ed960fcec3db50ae5886dc0df897269b3f5119cde9b731745f", size = 6113730, upload-time = "2024-08-06T00:20:47.107Z" }, - { url = "https://files.pythonhosted.org/packages/9d/e4/d3556f073563cea4aabfa340b08f462e8a748c7190f34a3467442d72ac48/grpcio-1.62.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:668211f3699bbee4deaf1d6e6b8df59328bf63f077bf2dc9b8bfa4a17df4a279", size = 5779201, upload-time = "2024-08-06T00:20:50.606Z" }, - { url = "https://files.pythonhosted.org/packages/15/c3/bf758db22525e1e3cd541f9bbfd33b248cf6866678a1285127cf5b6ec6a0/grpcio-1.62.3-cp310-cp310-win32.whl", hash = "sha256:216740723fc5971429550c374a0c039723b9d4dcaf7ba05227b7e0a500b06417", size = 3170437, upload-time = "2024-08-06T00:20:53.245Z" }, - { url = "https://files.pythonhosted.org/packages/d4/2f/1a4710440cc9e94a8d38af6dce0e670803a029ebc0f904929079a1c7ba58/grpcio-1.62.3-cp310-cp310-win_amd64.whl", hash = "sha256:b708401ede2c4cb8943e8a713988fcfe6cbea105b07cd7fa7c8a9f137c22bddb", size = 3730887, upload-time = "2024-08-06T00:20:56.018Z" }, - { url = "https://files.pythonhosted.org/packages/f5/3a/c3da1df7d55cfe481b02221d8e22e603f43fdf1646f2c02e7d69370d5e4b/grpcio-1.62.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:c8bb1a7aa82af6c7713cdf9dcb8f4ea1024ac7ce82bb0a0a82a49aea5237da34", size = 4774831, upload-time = "2024-08-06T00:20:58.985Z" }, - { url = "https://files.pythonhosted.org/packages/c8/12/c769a65437081cce5c7aece3fcc0a0e9e5293d85455cbdc9dd4edc9c56b9/grpcio-1.62.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:57823dc7299c4f258ae9c32fd327d29f729d359c34d7612b36e48ed45b3ab8d0", size = 10019810, upload-time = "2024-08-06T00:21:02.31Z" }, - { url = "https://files.pythonhosted.org/packages/18/08/b2a2c66f183240e55ca99a0dd85c2c2ad1cb0846e7ad628900843a49a155/grpcio-1.62.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:1de3d04d9a4ec31ebe848ae1fe61e4cbc367fb9495cbf6c54368e60609a998d9", size = 5294405, upload-time = "2024-08-06T00:21:05.586Z" }, - { url = "https://files.pythonhosted.org/packages/fe/ac/6e523ccaf068dc022de3cc798f539bd070fd45e4db953241cff23fd867a6/grpcio-1.62.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:325c56ce94d738c31059cf91376f625d3effdff8f85c96660a5fd6395d5a707f", size = 5830738, upload-time = "2024-08-06T00:21:08.512Z" }, - { url = "https://files.pythonhosted.org/packages/78/a0/3a1c81854f76d8c1462533e18cc754b9d3e434234678cb2273b1bff5885c/grpcio-1.62.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c175b252d063af388523a397dbe8edbc4319761f5ee892a8a0f5890acc067362", size = 5547176, upload-time = "2024-08-06T00:21:11.374Z" }, - { url = "https://files.pythonhosted.org/packages/7b/1a/9462e3c81429c4b299a7222df8cd3c1f84625eecc353967d5ddfec43f8f2/grpcio-1.62.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:25cd75dc73c5269932413e517be778640402f18cf9a81147e68645bd8af18ab0", size = 6116809, upload-time = "2024-08-06T00:21:13.99Z" }, - { url = "https://files.pythonhosted.org/packages/40/6c/99f922adeb6ca65b6f84f6bf1032f5b94c042eef65ab9b13d438819e9205/grpcio-1.62.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1b85d35a7d9638c03321dfe466645b87e23c30df1266f9e04bbb5f44e7579a9", size = 5779755, upload-time = "2024-08-06T00:21:17.029Z" }, - { url = "https://files.pythonhosted.org/packages/ae/b7/d48ff1a5f6ad59748df7717a3f101679e0e1b9004f5b6efa96831c2b847c/grpcio-1.62.3-cp311-cp311-win32.whl", hash = "sha256:6be243f3954b0ca709f56f9cae926c84ac96e1cce19844711e647a1f1db88b99", size = 3167821, upload-time = "2024-08-06T00:21:20.522Z" }, - { url = "https://files.pythonhosted.org/packages/9f/90/32ba95836adb03dd04a393f1b9a8bde7919e9f08ee5fd94dc77351f9305f/grpcio-1.62.3-cp311-cp311-win_amd64.whl", hash = "sha256:e9ffdb7bc9ccd56ec201aec3eab3432e1e820335b5a16ad2b37e094218dcd7a6", size = 3729044, upload-time = "2024-08-06T00:21:23.542Z" }, - { url = "https://files.pythonhosted.org/packages/33/3f/23748407c2fd739e983c366b805aeb86ed57c718f2619aa3a5856594ed67/grpcio-1.62.3-cp312-cp312-linux_armv7l.whl", hash = "sha256:4c9c1502c76cadbf2e145061b63af077b08d5677afcef91970d6db87b30e2f8b", size = 4733041, upload-time = "2024-08-06T00:21:26.788Z" }, - { url = "https://files.pythonhosted.org/packages/de/27/0f85db1ad84569c8c2f82c0d473b84dd09f8fe5e053298b1f35935b92d62/grpcio-1.62.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:abfe64811177e681edc81d9d9d1bd23edc5f599bd9846650864769264ace30cd", size = 9978073, upload-time = "2024-08-06T00:21:29.381Z" }, - { url = "https://files.pythonhosted.org/packages/4d/d1/df712e7f5cdd2676fde7a3459783f18dd8b6b8c6a201774551d431cfa50c/grpcio-1.62.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:3737e5ef0aa0fcdfeaf3b4ecc1a6be78b494549b28aec4b7f61b5dc357f7d8be", size = 5233910, upload-time = "2024-08-06T00:21:32.391Z" }, - { url = "https://files.pythonhosted.org/packages/12/b1/9e28e35382a4f29def23f7cbf5414a667d2249ce83eaf7024d31f88b0399/grpcio-1.62.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:940459d81685549afdfe13a6de102c52ea4cdda093477baa53056884aadf7c48", size = 5766972, upload-time = "2024-08-06T00:21:35.743Z" }, - { url = "https://files.pythonhosted.org/packages/82/2e/43218874d1852af1ea9801a2be62cc596ddd45984e7adba0fb9f66393c81/grpcio-1.62.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac9783d5679c8da612465168c820fd0b916e70ec5496c840bddba0be7f2d124c", size = 5492246, upload-time = "2024-08-06T00:21:38.978Z" }, - { url = "https://files.pythonhosted.org/packages/7a/59/8d83b5b52cf9a655633e36e7953899901fc93aefd15d3e1ff8129a7ef30e/grpcio-1.62.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:c95a0b76a44c548e6bd8c5f7dbecf89c77e2e16d3965be817b57769c4a30bea2", size = 6062547, upload-time = "2024-08-06T00:21:44.445Z" }, - { url = "https://files.pythonhosted.org/packages/99/8c/cf726cbee9a3e636adecc94a55136c72da8c36422c8c0173e0e3be535665/grpcio-1.62.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b097347441b86a8c3ad9579abaf5e5f7f82b1d74a898f47360433b2bca0e4536", size = 5729178, upload-time = "2024-08-06T00:21:48.757Z" }, - { url = "https://files.pythonhosted.org/packages/ea/a5/3a24d6d05da642e9d94902aa5c681ce9afd6f6af079d05a1d6d3aaa20cd6/grpcio-1.62.3-cp312-cp312-win32.whl", hash = "sha256:3fb7d966a976d762a31346353a19fce4afcffbeda3027dd563bc8cb521fcf799", size = 3156979, upload-time = "2024-08-06T00:21:51.846Z" }, - { url = "https://files.pythonhosted.org/packages/d7/9d/dc29922afbd0bb2616a14241508e6ee871b35f783a6b2e7104b44f82a2c6/grpcio-1.62.3-cp312-cp312-win_amd64.whl", hash = "sha256:454a6aed4ebd56198d37e1f3be6f1c70838e33dd62d1e2cea12f2bcb08efecc5", size = 3711573, upload-time = "2024-08-06T00:21:55.032Z" }, -] - -[[package]] -name = "grpcio-tools" -version = "1.62.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "grpcio" }, - { name = "protobuf" }, - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/54/fa/b69bd8040eafc09b88bb0ec0fea59e8aacd1a801e688af087cead213b0d0/grpcio-tools-1.62.3.tar.gz", hash = "sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833", size = 4538520, upload-time = "2024-08-06T00:37:11.035Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/eb/eb0a3aa9480c3689d31fd2ad536df6a828e97a60f667c8a93d05bdf07150/grpcio_tools-1.62.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2f968b049c2849540751ec2100ab05e8086c24bead769ca734fdab58698408c1", size = 5117556, upload-time = "2024-08-06T00:30:32.892Z" }, - { url = "https://files.pythonhosted.org/packages/f3/fb/8be3dda485f7fab906bfa02db321c3ecef953a87cdb5f6572ca08b187bcb/grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:0a8c0c4724ae9c2181b7dbc9b186df46e4f62cb18dc184e46d06c0ebeccf569e", size = 2719330, upload-time = "2024-08-06T00:30:36.113Z" }, - { url = "https://files.pythonhosted.org/packages/63/de/6978f8d10066e240141cd63d1fbfc92818d96bb53427074f47a8eda921e1/grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5782883a27d3fae8c425b29a9d3dcf5f47d992848a1b76970da3b5a28d424b26", size = 3070818, upload-time = "2024-08-06T00:30:38.797Z" }, - { url = "https://files.pythonhosted.org/packages/74/34/bb8f816893fc73fd6d830e895e8638d65d13642bb7a434f9175c5ca7da11/grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3d812daffd0c2d2794756bd45a353f89e55dc8f91eb2fc840c51b9f6be62667", size = 2804993, upload-time = "2024-08-06T00:30:41.72Z" }, - { url = "https://files.pythonhosted.org/packages/78/60/b2198d7db83293cdb9760fc083f077c73e4c182da06433b3b157a1567d06/grpcio_tools-1.62.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b47d0dda1bdb0a0ba7a9a6de88e5a1ed61f07fad613964879954961e36d49193", size = 3684915, upload-time = "2024-08-06T00:30:43.996Z" }, - { url = "https://files.pythonhosted.org/packages/61/20/56dbdc4ecb14d42a03cd164ff45e6e84572bbe61ee59c50c39f4d556a8d5/grpcio_tools-1.62.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ca246dffeca0498be9b4e1ee169b62e64694b0f92e6d0be2573e65522f39eea9", size = 3297482, upload-time = "2024-08-06T00:30:46.451Z" }, - { url = "https://files.pythonhosted.org/packages/4a/dc/e417a313c905744ce8cedf1e1edd81c41dc45ff400ae1c45080e18f26712/grpcio_tools-1.62.3-cp310-cp310-win32.whl", hash = "sha256:6a56d344b0bab30bf342a67e33d386b0b3c4e65868ffe93c341c51e1a8853ca5", size = 909793, upload-time = "2024-08-06T00:30:49.465Z" }, - { url = "https://files.pythonhosted.org/packages/d9/69/75e7ebfd8d755d3e7be5c6d1aa6d13220f5bba3a98965e4b50c329046777/grpcio_tools-1.62.3-cp310-cp310-win_amd64.whl", hash = "sha256:710fecf6a171dcbfa263a0a3e7070e0df65ba73158d4c539cec50978f11dad5d", size = 1052459, upload-time = "2024-08-06T00:30:51.98Z" }, - { url = "https://files.pythonhosted.org/packages/23/52/2dfe0a46b63f5ebcd976570aa5fc62f793d5a8b169e211c6a5aede72b7ae/grpcio_tools-1.62.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:703f46e0012af83a36082b5f30341113474ed0d91e36640da713355cd0ea5d23", size = 5147623, upload-time = "2024-08-06T00:30:54.894Z" }, - { url = "https://files.pythonhosted.org/packages/f0/2e/29fdc6c034e058482e054b4a3c2432f84ff2e2765c1342d4f0aa8a5c5b9a/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:7cc83023acd8bc72cf74c2edbe85b52098501d5b74d8377bfa06f3e929803492", size = 2719538, upload-time = "2024-08-06T00:30:57.928Z" }, - { url = "https://files.pythonhosted.org/packages/f9/60/abe5deba32d9ec2c76cdf1a2f34e404c50787074a2fee6169568986273f1/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ff7d58a45b75df67d25f8f144936a3e44aabd91afec833ee06826bd02b7fbe7", size = 3070964, upload-time = "2024-08-06T00:31:00.267Z" }, - { url = "https://files.pythonhosted.org/packages/bc/ad/e2b066684c75f8d9a48508cde080a3a36618064b9cadac16d019ca511444/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f2483ea232bd72d98a6dc6d7aefd97e5bc80b15cd909b9e356d6f3e326b6e43", size = 2805003, upload-time = "2024-08-06T00:31:02.565Z" }, - { url = "https://files.pythonhosted.org/packages/9c/3f/59bf7af786eae3f9d24ee05ce75318b87f541d0950190ecb5ffb776a1a58/grpcio_tools-1.62.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:962c84b4da0f3b14b3cdb10bc3837ebc5f136b67d919aea8d7bb3fd3df39528a", size = 3685154, upload-time = "2024-08-06T00:31:05.339Z" }, - { url = "https://files.pythonhosted.org/packages/f1/79/4dd62478b91e27084c67b35a2316ce8a967bd8b6cb8d6ed6c86c3a0df7cb/grpcio_tools-1.62.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8ad0473af5544f89fc5a1ece8676dd03bdf160fb3230f967e05d0f4bf89620e3", size = 3297942, upload-time = "2024-08-06T00:31:08.456Z" }, - { url = "https://files.pythonhosted.org/packages/b8/cb/86449ecc58bea056b52c0b891f26977afc8c4464d88c738f9648da941a75/grpcio_tools-1.62.3-cp311-cp311-win32.whl", hash = "sha256:db3bc9fa39afc5e4e2767da4459df82b095ef0cab2f257707be06c44a1c2c3e5", size = 910231, upload-time = "2024-08-06T00:31:11.464Z" }, - { url = "https://files.pythonhosted.org/packages/45/a4/9736215e3945c30ab6843280b0c6e1bff502910156ea2414cd77fbf1738c/grpcio_tools-1.62.3-cp311-cp311-win_amd64.whl", hash = "sha256:e0898d412a434e768a0c7e365acabe13ff1558b767e400936e26b5b6ed1ee51f", size = 1052496, upload-time = "2024-08-06T00:31:13.665Z" }, - { url = "https://files.pythonhosted.org/packages/2a/a5/d6887eba415ce318ae5005e8dfac3fa74892400b54b6d37b79e8b4f14f5e/grpcio_tools-1.62.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:d102b9b21c4e1e40af9a2ab3c6d41afba6bd29c0aa50ca013bf85c99cdc44ac5", size = 5147690, upload-time = "2024-08-06T00:31:16.436Z" }, - { url = "https://files.pythonhosted.org/packages/8a/7c/3cde447a045e83ceb4b570af8afe67ffc86896a2fe7f59594dc8e5d0a645/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:0a52cc9444df978438b8d2332c0ca99000521895229934a59f94f37ed896b133", size = 2720538, upload-time = "2024-08-06T00:31:18.905Z" }, - { url = "https://files.pythonhosted.org/packages/88/07/f83f2750d44ac4f06c07c37395b9c1383ef5c994745f73c6bfaf767f0944/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141d028bf5762d4a97f981c501da873589df3f7e02f4c1260e1921e565b376fa", size = 3071571, upload-time = "2024-08-06T00:31:21.684Z" }, - { url = "https://files.pythonhosted.org/packages/37/74/40175897deb61e54aca716bc2e8919155b48f33aafec8043dda9592d8768/grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47a5c093ab256dec5714a7a345f8cc89315cb57c298b276fa244f37a0ba507f0", size = 2806207, upload-time = "2024-08-06T00:31:24.208Z" }, - { url = "https://files.pythonhosted.org/packages/ec/ee/d8de915105a217cbcb9084d684abdc032030dcd887277f2ef167372287fe/grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d", size = 3685815, upload-time = "2024-08-06T00:31:26.917Z" }, - { url = "https://files.pythonhosted.org/packages/fd/d9/4360a6c12be3d7521b0b8c39e5d3801d622fbb81cc2721dbd3eee31e28c8/grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e02d7c1a02e3814c94ba0cfe43d93e872c758bd8fd5c2797f894d0c49b4a1dfc", size = 3298378, upload-time = "2024-08-06T00:31:30.401Z" }, - { url = "https://files.pythonhosted.org/packages/29/3b/7cdf4a9e5a3e0a35a528b48b111355cd14da601413a4f887aa99b6da468f/grpcio_tools-1.62.3-cp312-cp312-win32.whl", hash = "sha256:b881fd9505a84457e9f7e99362eeedd86497b659030cf57c6f0070df6d9c2b9b", size = 910416, upload-time = "2024-08-06T00:31:33.118Z" }, - { url = "https://files.pythonhosted.org/packages/6c/66/dd3ec249e44c1cc15e902e783747819ed41ead1336fcba72bf841f72c6e9/grpcio_tools-1.62.3-cp312-cp312-win_amd64.whl", hash = "sha256:11c625eebefd1fd40a228fc8bae385e448c7e32a6ae134e43cf13bbc23f902b7", size = 1052856, upload-time = "2024-08-06T00:31:36.519Z" }, -] - -[[package]] -name = "numpy" -version = "2.2.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/b2/ce4b867d8cd9c0ee84938ae1e6a6f7926ebf928c9090d036fc3c6a04f946/numpy-2.2.5.tar.gz", hash = "sha256:a9c0d994680cd991b1cb772e8b297340085466a6fe964bc9d4e80f5e2f43c291", size = 20273920, upload-time = "2025-04-19T23:27:42.561Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/4e/3d9e6d16237c2aa5485695f0626cbba82f6481efca2e9132368dea3b885e/numpy-2.2.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f4a922da1729f4c40932b2af4fe84909c7a6e167e6e99f71838ce3a29f3fe26", size = 21252117, upload-time = "2025-04-19T22:31:01.142Z" }, - { url = "https://files.pythonhosted.org/packages/38/e4/db91349d4079cd15c02ff3b4b8882a529991d6aca077db198a2f2a670406/numpy-2.2.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b6f91524d31b34f4a5fee24f5bc16dcd1491b668798b6d85585d836c1e633a6a", size = 14424615, upload-time = "2025-04-19T22:31:24.873Z" }, - { url = "https://files.pythonhosted.org/packages/f8/59/6e5b011f553c37b008bd115c7ba7106a18f372588fbb1b430b7a5d2c41ce/numpy-2.2.5-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:19f4718c9012e3baea91a7dba661dcab2451cda2550678dc30d53acb91a7290f", size = 5428691, upload-time = "2025-04-19T22:31:33.998Z" }, - { url = "https://files.pythonhosted.org/packages/a2/58/d5d70ebdac82b3a6ddf409b3749ca5786636e50fd64d60edb46442af6838/numpy-2.2.5-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:eb7fd5b184e5d277afa9ec0ad5e4eb562ecff541e7f60e69ee69c8d59e9aeaba", size = 6965010, upload-time = "2025-04-19T22:31:45.281Z" }, - { url = "https://files.pythonhosted.org/packages/dc/a8/c290394be346d4e7b48a40baf292626fd96ec56a6398ace4c25d9079bc6a/numpy-2.2.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6413d48a9be53e183eb06495d8e3b006ef8f87c324af68241bbe7a39e8ff54c3", size = 14369885, upload-time = "2025-04-19T22:32:06.557Z" }, - { url = "https://files.pythonhosted.org/packages/c2/70/fed13c70aabe7049368553e81d7ca40f305f305800a007a956d7cd2e5476/numpy-2.2.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7451f92eddf8503c9b8aa4fe6aa7e87fd51a29c2cfc5f7dbd72efde6c65acf57", size = 16418372, upload-time = "2025-04-19T22:32:31.716Z" }, - { url = "https://files.pythonhosted.org/packages/04/ab/c3c14f25ddaecd6fc58a34858f6a93a21eea6c266ba162fa99f3d0de12ac/numpy-2.2.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0bcb1d057b7571334139129b7f941588f69ce7c4ed15a9d6162b2ea54ded700c", size = 15883173, upload-time = "2025-04-19T22:32:55.106Z" }, - { url = "https://files.pythonhosted.org/packages/50/18/f53710a19042911c7aca824afe97c203728a34b8cf123e2d94621a12edc3/numpy-2.2.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:36ab5b23915887543441efd0417e6a3baa08634308894316f446027611b53bf1", size = 18206881, upload-time = "2025-04-19T22:33:22.08Z" }, - { url = "https://files.pythonhosted.org/packages/6b/ec/5b407bab82f10c65af5a5fe754728df03f960fd44d27c036b61f7b3ef255/numpy-2.2.5-cp310-cp310-win32.whl", hash = "sha256:422cc684f17bc963da5f59a31530b3936f57c95a29743056ef7a7903a5dbdf88", size = 6609852, upload-time = "2025-04-19T22:33:33.357Z" }, - { url = "https://files.pythonhosted.org/packages/b6/f5/467ca8675c7e6c567f571d8db942cc10a87588bd9e20a909d8af4171edda/numpy-2.2.5-cp310-cp310-win_amd64.whl", hash = "sha256:e4f0b035d9d0ed519c813ee23e0a733db81ec37d2e9503afbb6e54ccfdee0fa7", size = 12944922, upload-time = "2025-04-19T22:33:53.192Z" }, - { url = "https://files.pythonhosted.org/packages/f5/fb/e4e4c254ba40e8f0c78218f9e86304628c75b6900509b601c8433bdb5da7/numpy-2.2.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c42365005c7a6c42436a54d28c43fe0e01ca11eb2ac3cefe796c25a5f98e5e9b", size = 21256475, upload-time = "2025-04-19T22:34:24.174Z" }, - { url = "https://files.pythonhosted.org/packages/81/32/dd1f7084f5c10b2caad778258fdaeedd7fbd8afcd2510672811e6138dfac/numpy-2.2.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:498815b96f67dc347e03b719ef49c772589fb74b8ee9ea2c37feae915ad6ebda", size = 14461474, upload-time = "2025-04-19T22:34:46.578Z" }, - { url = "https://files.pythonhosted.org/packages/0e/65/937cdf238ef6ac54ff749c0f66d9ee2b03646034c205cea9b6c51f2f3ad1/numpy-2.2.5-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:6411f744f7f20081b1b4e7112e0f4c9c5b08f94b9f086e6f0adf3645f85d3a4d", size = 5426875, upload-time = "2025-04-19T22:34:56.281Z" }, - { url = "https://files.pythonhosted.org/packages/25/17/814515fdd545b07306eaee552b65c765035ea302d17de1b9cb50852d2452/numpy-2.2.5-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:9de6832228f617c9ef45d948ec1cd8949c482238d68b2477e6f642c33a7b0a54", size = 6969176, upload-time = "2025-04-19T22:35:07.518Z" }, - { url = "https://files.pythonhosted.org/packages/e5/32/a66db7a5c8b5301ec329ab36d0ecca23f5e18907f43dbd593c8ec326d57c/numpy-2.2.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:369e0d4647c17c9363244f3468f2227d557a74b6781cb62ce57cf3ef5cc7c610", size = 14374850, upload-time = "2025-04-19T22:35:31.347Z" }, - { url = "https://files.pythonhosted.org/packages/ad/c9/1bf6ada582eebcbe8978f5feb26584cd2b39f94ededeea034ca8f84af8c8/numpy-2.2.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:262d23f383170f99cd9191a7c85b9a50970fe9069b2f8ab5d786eca8a675d60b", size = 16430306, upload-time = "2025-04-19T22:35:57.573Z" }, - { url = "https://files.pythonhosted.org/packages/6a/f0/3f741863f29e128f4fcfdb99253cc971406b402b4584663710ee07f5f7eb/numpy-2.2.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aa70fdbdc3b169d69e8c59e65c07a1c9351ceb438e627f0fdcd471015cd956be", size = 15884767, upload-time = "2025-04-19T22:36:22.245Z" }, - { url = "https://files.pythonhosted.org/packages/98/d9/4ccd8fd6410f7bf2d312cbc98892e0e43c2fcdd1deae293aeb0a93b18071/numpy-2.2.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37e32e985f03c06206582a7323ef926b4e78bdaa6915095ef08070471865b906", size = 18219515, upload-time = "2025-04-19T22:36:49.822Z" }, - { url = "https://files.pythonhosted.org/packages/b1/56/783237243d4395c6dd741cf16eeb1a9035ee3d4310900e6b17e875d1b201/numpy-2.2.5-cp311-cp311-win32.whl", hash = "sha256:f5045039100ed58fa817a6227a356240ea1b9a1bc141018864c306c1a16d4175", size = 6607842, upload-time = "2025-04-19T22:37:01.624Z" }, - { url = "https://files.pythonhosted.org/packages/98/89/0c93baaf0094bdaaaa0536fe61a27b1dce8a505fa262a865ec142208cfe9/numpy-2.2.5-cp311-cp311-win_amd64.whl", hash = "sha256:b13f04968b46ad705f7c8a80122a42ae8f620536ea38cf4bdd374302926424dd", size = 12949071, upload-time = "2025-04-19T22:37:21.098Z" }, - { url = "https://files.pythonhosted.org/packages/e2/f7/1fd4ff108cd9d7ef929b8882692e23665dc9c23feecafbb9c6b80f4ec583/numpy-2.2.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ee461a4eaab4f165b68780a6a1af95fb23a29932be7569b9fab666c407969051", size = 20948633, upload-time = "2025-04-19T22:37:52.4Z" }, - { url = "https://files.pythonhosted.org/packages/12/03/d443c278348371b20d830af155ff2079acad6a9e60279fac2b41dbbb73d8/numpy-2.2.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ec31367fd6a255dc8de4772bd1658c3e926d8e860a0b6e922b615e532d320ddc", size = 14176123, upload-time = "2025-04-19T22:38:15.058Z" }, - { url = "https://files.pythonhosted.org/packages/2b/0b/5ca264641d0e7b14393313304da48b225d15d471250376f3fbdb1a2be603/numpy-2.2.5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:47834cde750d3c9f4e52c6ca28a7361859fcaf52695c7dc3cc1a720b8922683e", size = 5163817, upload-time = "2025-04-19T22:38:24.885Z" }, - { url = "https://files.pythonhosted.org/packages/04/b3/d522672b9e3d28e26e1613de7675b441bbd1eaca75db95680635dd158c67/numpy-2.2.5-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2c1a1c6ccce4022383583a6ded7bbcda22fc635eb4eb1e0a053336425ed36dfa", size = 6698066, upload-time = "2025-04-19T22:38:35.782Z" }, - { url = "https://files.pythonhosted.org/packages/a0/93/0f7a75c1ff02d4b76df35079676b3b2719fcdfb39abdf44c8b33f43ef37d/numpy-2.2.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d75f338f5f79ee23548b03d801d28a505198297534f62416391857ea0479571", size = 14087277, upload-time = "2025-04-19T22:38:57.697Z" }, - { url = "https://files.pythonhosted.org/packages/b0/d9/7c338b923c53d431bc837b5b787052fef9ae68a56fe91e325aac0d48226e/numpy-2.2.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a801fef99668f309b88640e28d261991bfad9617c27beda4a3aec4f217ea073", size = 16135742, upload-time = "2025-04-19T22:39:22.689Z" }, - { url = "https://files.pythonhosted.org/packages/2d/10/4dec9184a5d74ba9867c6f7d1e9f2e0fb5fe96ff2bf50bb6f342d64f2003/numpy-2.2.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:abe38cd8381245a7f49967a6010e77dbf3680bd3627c0fe4362dd693b404c7f8", size = 15581825, upload-time = "2025-04-19T22:39:45.794Z" }, - { url = "https://files.pythonhosted.org/packages/80/1f/2b6fcd636e848053f5b57712a7d1880b1565eec35a637fdfd0a30d5e738d/numpy-2.2.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5a0ac90e46fdb5649ab6369d1ab6104bfe5854ab19b645bf5cda0127a13034ae", size = 17899600, upload-time = "2025-04-19T22:40:13.427Z" }, - { url = "https://files.pythonhosted.org/packages/ec/87/36801f4dc2623d76a0a3835975524a84bd2b18fe0f8835d45c8eae2f9ff2/numpy-2.2.5-cp312-cp312-win32.whl", hash = "sha256:0cd48122a6b7eab8f06404805b1bd5856200e3ed6f8a1b9a194f9d9054631beb", size = 6312626, upload-time = "2025-04-19T22:40:25.223Z" }, - { url = "https://files.pythonhosted.org/packages/8b/09/4ffb4d6cfe7ca6707336187951992bd8a8b9142cf345d87ab858d2d7636a/numpy-2.2.5-cp312-cp312-win_amd64.whl", hash = "sha256:ced69262a8278547e63409b2653b372bf4baff0870c57efa76c5703fd6543282", size = 12645715, upload-time = "2025-04-19T22:40:44.528Z" }, - { url = "https://files.pythonhosted.org/packages/e2/a0/0aa7f0f4509a2e07bd7a509042967c2fab635690d4f48c6c7b3afd4f448c/numpy-2.2.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:059b51b658f4414fff78c6d7b1b4e18283ab5fa56d270ff212d5ba0c561846f4", size = 20935102, upload-time = "2025-04-19T22:41:16.234Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e4/a6a9f4537542912ec513185396fce52cdd45bdcf3e9d921ab02a93ca5aa9/numpy-2.2.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:47f9ed103af0bc63182609044b0490747e03bd20a67e391192dde119bf43d52f", size = 14191709, upload-time = "2025-04-19T22:41:38.472Z" }, - { url = "https://files.pythonhosted.org/packages/be/65/72f3186b6050bbfe9c43cb81f9df59ae63603491d36179cf7a7c8d216758/numpy-2.2.5-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:261a1ef047751bb02f29dfe337230b5882b54521ca121fc7f62668133cb119c9", size = 5149173, upload-time = "2025-04-19T22:41:47.823Z" }, - { url = "https://files.pythonhosted.org/packages/e5/e9/83e7a9432378dde5802651307ae5e9ea07bb72b416728202218cd4da2801/numpy-2.2.5-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4520caa3807c1ceb005d125a75e715567806fed67e315cea619d5ec6e75a4191", size = 6684502, upload-time = "2025-04-19T22:41:58.689Z" }, - { url = "https://files.pythonhosted.org/packages/ea/27/b80da6c762394c8ee516b74c1f686fcd16c8f23b14de57ba0cad7349d1d2/numpy-2.2.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d14b17b9be5f9c9301f43d2e2a4886a33b53f4e6fdf9ca2f4cc60aeeee76372", size = 14084417, upload-time = "2025-04-19T22:42:19.897Z" }, - { url = "https://files.pythonhosted.org/packages/aa/fc/ebfd32c3e124e6a1043e19c0ab0769818aa69050ce5589b63d05ff185526/numpy-2.2.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ba321813a00e508d5421104464510cc962a6f791aa2fca1c97b1e65027da80d", size = 16133807, upload-time = "2025-04-19T22:42:44.433Z" }, - { url = "https://files.pythonhosted.org/packages/bf/9b/4cc171a0acbe4666f7775cfd21d4eb6bb1d36d3a0431f48a73e9212d2278/numpy-2.2.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4cbdef3ddf777423060c6f81b5694bad2dc9675f110c4b2a60dc0181543fac7", size = 15575611, upload-time = "2025-04-19T22:43:09.928Z" }, - { url = "https://files.pythonhosted.org/packages/a3/45/40f4135341850df48f8edcf949cf47b523c404b712774f8855a64c96ef29/numpy-2.2.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:54088a5a147ab71a8e7fdfd8c3601972751ded0739c6b696ad9cb0343e21ab73", size = 17895747, upload-time = "2025-04-19T22:43:36.983Z" }, - { url = "https://files.pythonhosted.org/packages/f8/4c/b32a17a46f0ffbde8cc82df6d3daeaf4f552e346df143e1b188a701a8f09/numpy-2.2.5-cp313-cp313-win32.whl", hash = "sha256:c8b82a55ef86a2d8e81b63da85e55f5537d2157165be1cb2ce7cfa57b6aef38b", size = 6309594, upload-time = "2025-04-19T22:47:10.523Z" }, - { url = "https://files.pythonhosted.org/packages/13/ae/72e6276feb9ef06787365b05915bfdb057d01fceb4a43cb80978e518d79b/numpy-2.2.5-cp313-cp313-win_amd64.whl", hash = "sha256:d8882a829fd779f0f43998e931c466802a77ca1ee0fe25a3abe50278616b1471", size = 12638356, upload-time = "2025-04-19T22:47:30.253Z" }, - { url = "https://files.pythonhosted.org/packages/79/56/be8b85a9f2adb688e7ded6324e20149a03541d2b3297c3ffc1a73f46dedb/numpy-2.2.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:e8b025c351b9f0e8b5436cf28a07fa4ac0204d67b38f01433ac7f9b870fa38c6", size = 20963778, upload-time = "2025-04-19T22:44:09.251Z" }, - { url = "https://files.pythonhosted.org/packages/ff/77/19c5e62d55bff507a18c3cdff82e94fe174957bad25860a991cac719d3ab/numpy-2.2.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8dfa94b6a4374e7851bbb6f35e6ded2120b752b063e6acdd3157e4d2bb922eba", size = 14207279, upload-time = "2025-04-19T22:44:31.383Z" }, - { url = "https://files.pythonhosted.org/packages/75/22/aa11f22dc11ff4ffe4e849d9b63bbe8d4ac6d5fae85ddaa67dfe43be3e76/numpy-2.2.5-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:97c8425d4e26437e65e1d189d22dff4a079b747ff9c2788057bfb8114ce1e133", size = 5199247, upload-time = "2025-04-19T22:44:40.361Z" }, - { url = "https://files.pythonhosted.org/packages/4f/6c/12d5e760fc62c08eded0394f62039f5a9857f758312bf01632a81d841459/numpy-2.2.5-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:352d330048c055ea6db701130abc48a21bec690a8d38f8284e00fab256dc1376", size = 6711087, upload-time = "2025-04-19T22:44:51.188Z" }, - { url = "https://files.pythonhosted.org/packages/ef/94/ece8280cf4218b2bee5cec9567629e61e51b4be501e5c6840ceb593db945/numpy-2.2.5-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b4c0773b6ada798f51f0f8e30c054d32304ccc6e9c5d93d46cb26f3d385ab19", size = 14059964, upload-time = "2025-04-19T22:45:12.451Z" }, - { url = "https://files.pythonhosted.org/packages/39/41/c5377dac0514aaeec69115830a39d905b1882819c8e65d97fc60e177e19e/numpy-2.2.5-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55f09e00d4dccd76b179c0f18a44f041e5332fd0e022886ba1c0bbf3ea4a18d0", size = 16121214, upload-time = "2025-04-19T22:45:37.734Z" }, - { url = "https://files.pythonhosted.org/packages/db/54/3b9f89a943257bc8e187145c6bc0eb8e3d615655f7b14e9b490b053e8149/numpy-2.2.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:02f226baeefa68f7d579e213d0f3493496397d8f1cff5e2b222af274c86a552a", size = 15575788, upload-time = "2025-04-19T22:46:01.908Z" }, - { url = "https://files.pythonhosted.org/packages/b1/c4/2e407e85df35b29f79945751b8f8e671057a13a376497d7fb2151ba0d290/numpy-2.2.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c26843fd58f65da9491165072da2cccc372530681de481ef670dcc8e27cfb066", size = 17893672, upload-time = "2025-04-19T22:46:28.585Z" }, - { url = "https://files.pythonhosted.org/packages/29/7e/d0b44e129d038dba453f00d0e29ebd6eaf2f06055d72b95b9947998aca14/numpy-2.2.5-cp313-cp313t-win32.whl", hash = "sha256:1a161c2c79ab30fe4501d5a2bbfe8b162490757cf90b7f05be8b80bc02f7bb8e", size = 6377102, upload-time = "2025-04-19T22:46:39.949Z" }, - { url = "https://files.pythonhosted.org/packages/63/be/b85e4aa4bf42c6502851b971f1c326d583fcc68227385f92089cf50a7b45/numpy-2.2.5-cp313-cp313t-win_amd64.whl", hash = "sha256:d403c84991b5ad291d3809bace5e85f4bbf44a04bdc9a88ed2bb1807b3360bb8", size = 12750096, upload-time = "2025-04-19T22:47:00.147Z" }, - { url = "https://files.pythonhosted.org/packages/35/e4/5ef5ef1d4308f96961198b2323bfc7c7afb0ccc0d623b01c79bc87ab496d/numpy-2.2.5-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b4ea7e1cff6784e58fe281ce7e7f05036b3e1c89c6f922a6bfbc0a7e8768adbe", size = 21083404, upload-time = "2025-04-19T22:48:01.605Z" }, - { url = "https://files.pythonhosted.org/packages/a3/5f/bde9238e8e977652a16a4b114ed8aa8bb093d718c706eeecb5f7bfa59572/numpy-2.2.5-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:d7543263084a85fbc09c704b515395398d31d6395518446237eac219eab9e55e", size = 6828578, upload-time = "2025-04-19T22:48:13.118Z" }, - { url = "https://files.pythonhosted.org/packages/ef/7f/813f51ed86e559ab2afb6a6f33aa6baf8a560097e25e4882a938986c76c2/numpy-2.2.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0255732338c4fdd00996c0421884ea8a3651eea555c3a56b84892b66f696eb70", size = 16234796, upload-time = "2025-04-19T22:48:37.102Z" }, - { url = "https://files.pythonhosted.org/packages/68/67/1175790323026d3337cc285cc9c50eca637d70472b5e622529df74bb8f37/numpy-2.2.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d2e3bdadaba0e040d1e7ab39db73e0afe2c74ae277f5614dad53eadbecbbb169", size = 12859001, upload-time = "2025-04-19T22:48:57.665Z" }, -] - -[[package]] -name = "packaging" -version = "25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, -] - -[[package]] -name = "protobuf" -version = "4.25.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/63/84fdeac1f03864c2b8b9f0b7fe711c4af5f95759ee281d2026530086b2f5/protobuf-4.25.7.tar.gz", hash = "sha256:28f65ae8c14523cc2c76c1e91680958700d3eac69f45c96512c12c63d9a38807", size = 380612, upload-time = "2025-04-24T02:56:58.685Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/ed/9a58076cfb8edc237c92617f1d3744660e9b4457d54f3c2fdf1a4bbae5c7/protobuf-4.25.7-cp310-abi3-win32.whl", hash = "sha256:dc582cf1a73a6b40aa8e7704389b8d8352da616bc8ed5c6cc614bdd0b5ce3f7a", size = 392457, upload-time = "2025-04-24T02:56:40.798Z" }, - { url = "https://files.pythonhosted.org/packages/28/b3/e00870528029fe252cf3bd6fa535821c276db3753b44a4691aee0d52ff9e/protobuf-4.25.7-cp310-abi3-win_amd64.whl", hash = "sha256:cd873dbddb28460d1706ff4da2e7fac175f62f2a0bebc7b33141f7523c5a2399", size = 413446, upload-time = "2025-04-24T02:56:44.199Z" }, - { url = "https://files.pythonhosted.org/packages/60/1d/f450a193f875a20099d4492d2c1cb23091d65d512956fb1e167ee61b4bf0/protobuf-4.25.7-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4c899f09b0502eb39174c717ccf005b844ea93e31137c167ddcacf3e09e49610", size = 394248, upload-time = "2025-04-24T02:56:45.75Z" }, - { url = "https://files.pythonhosted.org/packages/c8/b8/ea88e9857484a0618c74121618b9e620fc50042de43cdabbebe1b93a83e0/protobuf-4.25.7-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:6d2f5dede3d112e573f0e5f9778c0c19d9f9e209727abecae1d39db789f522c6", size = 293717, upload-time = "2025-04-24T02:56:47.427Z" }, - { url = "https://files.pythonhosted.org/packages/a7/81/d0b68e9a9a76804113b6dedc6fffed868b97048bbe6f1bedc675bdb8523c/protobuf-4.25.7-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:d41fb7ae72a25fcb79b2d71e4247f0547a02e8185ed51587c22827a87e5736ed", size = 294636, upload-time = "2025-04-24T02:56:48.976Z" }, - { url = "https://files.pythonhosted.org/packages/17/d7/1e7c80cb2ea2880cfe38580dcfbb22b78b746640c9c13fc3337a6967dc4c/protobuf-4.25.7-py3-none-any.whl", hash = "sha256:e9d969f5154eaeab41404def5dcf04e62162178f4b9de98b2d3c1c70f5f84810", size = 156468, upload-time = "2025-04-24T02:56:56.957Z" }, -] - -[[package]] -name = "pycparser" -version = "2.22" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload-time = "2024-03-30T13:22:22.564Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, -] - -[[package]] -name = "pymonik" -source = { editable = "../pymonik" } -dependencies = [ - { name = "armonik" }, - { name = "cloudpickle" }, - { name = "grpcio" }, - { name = "pyyaml" }, -] - -[package.metadata] -requires-dist = [ - { name = "armonik", specifier = ">=3.25.0" }, - { name = "cloudpickle", specifier = ">=3.1.1" }, - { name = "grpcio" }, - { name = "pyyaml", specifier = ">=6.0.2" }, -] - -[package.metadata.requires-dev] -dev = [{ name = "ruff", specifier = ">=0.11.6" }] - -[[package]] -name = "pyyaml" -version = "6.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" }, - { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" }, - { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" }, - { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" }, - { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" }, - { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" }, - { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" }, - { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" }, - { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" }, - { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" }, - { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" }, - { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" }, - { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" }, - { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" }, - { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" }, - { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" }, - { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" }, - { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, - { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, - { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, - { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" }, - { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" }, - { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" }, - { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" }, - { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" }, - { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, - { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" }, - { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" }, - { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" }, - { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" }, - { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" }, - { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" }, - { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" }, - { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" }, - { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, -] - -[[package]] -name = "setuptools" -version = "79.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bb/71/b6365e6325b3290e14957b2c3a804a529968c77a049b2ed40c095f749707/setuptools-79.0.1.tar.gz", hash = "sha256:128ce7b8f33c3079fd1b067ecbb4051a66e8526e7b65f6cec075dfc650ddfa88", size = 1367909, upload-time = "2025-04-23T22:20:59.241Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/6d/b4752b044bf94cb802d88a888dc7d288baaf77d7910b7dedda74b5ceea0c/setuptools-79.0.1-py3-none-any.whl", hash = "sha256:e147c0549f27767ba362f9da434eab9c5dc0045d5304feb602a0af001089fc51", size = 1256281, upload-time = "2025-04-23T22:20:56.768Z" }, -] - -[[package]] -name = "test-client" -version = "0.1.0" -source = { virtual = "." } -dependencies = [ - { name = "debugpy" }, - { name = "numpy" }, - { name = "pymonik" }, -] - -[package.metadata] -requires-dist = [ - { name = "debugpy", specifier = ">=1.8.14" }, - { name = "numpy", specifier = ">=2.2.5" }, - { name = "pymonik", editable = "../pymonik" }, -] diff --git a/test_client/worker_cache.py b/test_client/worker_cache.py deleted file mode 100644 index 6e9f85f..0000000 --- a/test_client/worker_cache.py +++ /dev/null @@ -1,53 +0,0 @@ -from time import sleep -from pymonik import Pymonik, PymonikContext, task -import cloudpickle as pickle -import os - -def find_file(filename, search_path=None): - """ - Search for a file by name in the file system. - - Args: - filename (str): Name of the file to search for - search_path (str, optional): Root directory to start search from. - Defaults to current working directory. - - Returns: - str or None: Full path to the file if found, None if not found - """ - if search_path is None: - search_path = os.getcwd() - - for root, dirs, files in os.walk(search_path): - if filename in files: - return os.path.join(root, filename) - - return None - -# TODO: Instead of require_context, let's not fuck with the function's signature -# instead, ContextManager, and `ctx = get_context()` in the function body when needed. -@task(require_context=True) -def my_task(ctx: PymonikContext, result_id): - - if ctx.retrieve_object(result_id): - ctx.logger.info("Found the object !") - path = find_file(result_id, "/") - ctx.logger.info(f"GetObjectPath({result_id}) == {ctx.get_object_path(result_id)}") - ctx.logger.info(f"find_file({result_id}) == {path}") - with open(str(path), "rb") as fh: - contents = fh.read() - unpickled_contents = pickle.loads(contents) - ctx.logger.info(f"File contents (binary) = {contents}") - ctx.logger.info(f"File contents (unpickled) = {unpickled_contents}") - return path == str(ctx.get_object_path(result_id)) - else: - ctx.logger.info("Couldn't find the object !") - return None - - -if __name__ == "__main__": - with Pymonik("localhost:5001") as pk: - handle = pk.put("H"*100, "my_obj").wait() - print(f"Result id = {handle}") - res = my_task.invoke(handle.result_id).wait().get() - print(res) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_attach_session.py b/tests/test_attach_session.py new file mode 100644 index 0000000..5b9bed9 --- /dev/null +++ b/tests/test_attach_session.py @@ -0,0 +1,116 @@ +"""``client.session(attach_to=session_id)`` — pick up an existing session. + +The attach path doesn't issue ``create_session`` and doesn't issue +``close_session`` on exit. We don't have a real cluster in unit tests, +so these checks operate on a `Session` constructed with a stubbed +client/channel — enough to verify the contract that: + +- ``_open_resources`` skips ``create_session`` and uses the supplied + id verbatim. +- ``_close_resources`` skips ``close_session``. +- ``Session.session_id`` returns the attached id. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from pymonik.session import Session + + +class _Channel: + def close(self): + pass + + +class _StubClient: + def __init__(self): + self._channel = _Channel() + + +@pytest.fixture +def stub_session(monkeypatch): + """Patch the armonik client classes the Session constructs at open. + + Returns ``(session, sessions_mock)`` so tests can assert on RPC calls. + """ + from pymonik import session as session_mod + + sessions_mock = MagicMock() + sessions_mock.create_session = MagicMock( + return_value="created-fresh-id" + ) + sessions_mock.close_session = MagicMock() + + monkeypatch.setattr(session_mod, "ArmoniKSessions", lambda _ch: sessions_mock) + monkeypatch.setattr(session_mod, "ArmoniKTasks", lambda _ch: MagicMock()) + monkeypatch.setattr(session_mod, "ArmoniKResults", lambda _ch: MagicMock()) + monkeypatch.setattr(session_mod, "ArmoniKEvents", lambda _ch: MagicMock()) + + yield sessions_mock + + +def test_attach_skips_create_and_close(stub_session): + """The attach path doesn't create or close the session.""" + sess = Session( + client=_StubClient(), # type: ignore[arg-type] + partition="pymonik", + attach_to="existing-session-abc", + use_events=False, # avoid needing a real events stream + ) + + sess._open_resources() + try: + assert sess.session_id == "existing-session-abc" + stub_session.create_session.assert_not_called() + finally: + sess._stop.set() + sess._close_resources() + + stub_session.close_session.assert_not_called() + + +def test_create_path_unchanged(stub_session): + """Without attach_to, behaviour is unchanged: create on open, close on exit.""" + sess = Session( + client=_StubClient(), # type: ignore[arg-type] + partition="pymonik", + use_events=False, + ) + + sess._open_resources() + try: + assert sess.session_id == "created-fresh-id" + stub_session.create_session.assert_called_once() + finally: + sess._stop.set() + sess._close_resources() + + stub_session.close_session.assert_called_once_with("created-fresh-id") + + +def test_attach_keeps_partition_validation(stub_session): + """Per-task partition validation still runs against the supplied + partition list (the cluster-side declaration was at create time; + we trust the user's list for client-side checks).""" + sess = Session( + client=_StubClient(), # type: ignore[arg-type] + partition=["pymonik", "gpu"], + attach_to="existing-session-xyz", + use_events=False, + ) + assert sess.partitions == ("pymonik", "gpu") + assert sess.partition == "pymonik" + + +def test_client_session_passes_attach_to(stub_session): + """``client.session(attach_to=...)`` propagates to the Session.""" + from pymonik import PymonikClient + + client = PymonikClient(endpoint="grpcs://test:5001") + # Build the session without entering the client (no real channel). + client._channel = _Channel() # type: ignore[assignment] + sess = client.session(partition="pymonik", attach_to="abc-123") + assert sess._attach_to == "abc-123" diff --git a/tests/test_directory_materialize.py b/tests/test_directory_materialize.py new file mode 100644 index 0000000..3bda134 --- /dev/null +++ b/tests/test_directory_materialize.py @@ -0,0 +1,81 @@ +"""Directory ``materialize()`` — zip on client, unzip on worker.""" + +from __future__ import annotations + +import io +import zipfile +from pathlib import Path + +from pymonik import Materialize, task +from pymonik.blob import _zip_directory +from pymonik.testing import LocalCluster + + +@task +def list_files_under(p: Path) -> list[str]: + return sorted(str(f.relative_to(p)) for f in p.rglob("*") if f.is_file()) + + +@task +def read_text_at(p: Path, rel: str) -> str: + return (p / rel).read_text() + + +@task +def read_file(p: Path) -> str: + return p.read_text() + + +def _make_tree(root: Path) -> None: + root.mkdir(parents=True, exist_ok=True) + (root / "a.txt").write_text("alpha") + (root / "sub").mkdir() + (root / "sub" / "b.txt").write_text("beta") + (root / "sub" / "c.txt").write_text("gamma") + + +def test_zip_directory_is_deterministic(tmp_path): + """Same content → same SHA, regardless of filesystem walk order.""" + a = tmp_path / "a" + b = tmp_path / "b" + _make_tree(a) + _make_tree(b) + assert _zip_directory(a) == _zip_directory(b) + + +def test_zip_directory_round_trip(tmp_path): + src = tmp_path / "tree" + _make_tree(src) + data = _zip_directory(src) + # It's a real zip. + with zipfile.ZipFile(io.BytesIO(data)) as zf: + names = sorted(zf.namelist()) + assert names == ["a.txt", "sub/b.txt", "sub/c.txt"] + + +def test_directory_materialize_end_to_end(tmp_path): + src = tmp_path / "assets" + _make_tree(src) + target = tmp_path / "worker_assets" + + with LocalCluster() as client: + with client.session() as s: + handle = Materialize.__new__(Materialize) # placeholder + mat = __import__("pymonik").blob.materialize(src, at=str(target)) + assert mat.is_dir is True + files = list_files_under.spawn(mat).result(timeout=30) + assert files == ["a.txt", "sub/b.txt", "sub/c.txt"] + txt = read_text_at.spawn(mat, "sub/b.txt").result(timeout=30) + assert txt == "beta" + + +def test_file_materialize_still_works(tmp_path): + src = tmp_path / "config.toml" + src.write_text("[ok]\n") + target = tmp_path / "worker_config.toml" + + with LocalCluster() as client: + with client.session() as s: + mat = __import__("pymonik").blob.materialize(src, at=str(target)) + assert mat.is_dir is False + assert read_file.spawn(mat).result(timeout=30) == "[ok]\n" diff --git a/tests/test_env_id.py b/tests/test_env_id.py new file mode 100644 index 0000000..f357632 --- /dev/null +++ b/tests/test_env_id.py @@ -0,0 +1,55 @@ +"""Hashing rules for ``compute_env_id``. + +Two clients submitting the *same* deps must land in the *same* venv. +That contract is what makes /cache/internal sharing work, so the +canonicalisation rules are tested explicitly here. +""" + +from __future__ import annotations + +from pymonik._internal.env_builder import canonical_deps, compute_env_id +from pymonik.envelope import EnvSpec + + +def test_canonical_deps_strip_lower_dedup_sort(): + assert canonical_deps([" numpy ", "POLARS", "numpy", ""]) == ("numpy", "polars") + + +def test_env_id_is_order_independent(): + a = EnvSpec(deps=("numpy", "polars", "scikit-learn==1.5.*")) + b = EnvSpec(deps=("scikit-learn==1.5.*", "polars", "numpy")) + assert compute_env_id(a) == compute_env_id(b) + + +def test_env_id_is_case_independent_for_names(): + a = EnvSpec(deps=("NumPy",)) + b = EnvSpec(deps=("numpy",)) + assert compute_env_id(a) == compute_env_id(b) + + +def test_env_id_changes_with_specifier(): + a = EnvSpec(deps=("numpy>=2",)) + b = EnvSpec(deps=("numpy",)) + assert compute_env_id(a) != compute_env_id(b) + + +def test_env_id_changes_with_index_url(): + base = EnvSpec(deps=("numpy",)) + other = EnvSpec(deps=("numpy",), index_url="https://my.private.index/") + assert compute_env_id(base) != compute_env_id(other) + + +def test_env_id_independent_of_isolate_flag(): + """``isolate`` is a dispatch-mode toggle, not part of the env identity: + a session that splices and a session that subprocesses should reuse + the same venv on disk.""" + a = EnvSpec(deps=("numpy",), isolate=True) + b = EnvSpec(deps=("numpy",), isolate=False) + assert compute_env_id(a) == compute_env_id(b) + + +def test_env_id_short_and_hexlike(): + spec = EnvSpec(deps=("numpy",)) + h = compute_env_id(spec) + assert len(h) == 32 + assert all(c in "0123456789abcdef" for c in h) diff --git a/tests/test_env_variables.py b/tests/test_env_variables.py new file mode 100644 index 0000000..53a987e --- /dev/null +++ b/tests/test_env_variables.py @@ -0,0 +1,65 @@ +"""``env`` parameter on session / @task / .with_options. + +Covers: +- env without deps → no venv, just env vars applied +- env merges key-wise across session ← @task ← .with_options +- env participates in env_id (different env → different venv when deps present) +- worker-side env restoration after the task runs +""" + +from __future__ import annotations + +import os + +import pytest + +from pymonik import task +from pymonik._internal.env_builder import compute_env_id +from pymonik.envelope import EnvSpec +from pymonik.options import EMPTY, TaskOpts +from pymonik.testing import LocalCluster + + +@task +def read_env(name: str) -> str | None: + return os.environ.get(name) + + +def test_env_only_no_deps_applies_at_call_time(tmp_path, monkeypatch): + monkeypatch.setenv("PYMONIK_ENVS_ROOT", str(tmp_path / "envs")) + monkeypatch.delenv("PMK_TEST_KEY", raising=False) + with LocalCluster() as client: + with client.session(env={"PMK_TEST_KEY": "from_session"}) as s: + assert read_env.spawn("PMK_TEST_KEY").result(timeout=30) == "from_session" + # Restored after the task finishes. + assert os.environ.get("PMK_TEST_KEY") is None + + +def test_env_per_task_overrides_session(tmp_path, monkeypatch): + monkeypatch.setenv("PYMONIK_ENVS_ROOT", str(tmp_path / "envs")) + monkeypatch.delenv("PMK_TEST_KEY", raising=False) + overridden = read_env.with_options(env={"PMK_TEST_KEY": "from_task"}) + with LocalCluster() as client: + with client.session(env={"PMK_TEST_KEY": "from_session"}) as s: + assert overridden.spawn("PMK_TEST_KEY").result(timeout=30) == "from_task" + + +def test_env_merge_keywise(): + a = TaskOpts(env={"A": "1", "B": "1"}) + b = TaskOpts(env={"B": "2", "C": "3"}) + merged = a.merge(b) + assert merged.env == {"A": "1", "B": "2", "C": "3"} + + +def test_env_changes_env_id_when_deps_present(): + base = EnvSpec(deps=("numpy",)) + with_env = EnvSpec(deps=("numpy",), env=(("FOO", "1"),)) + assert compute_env_id(base) != compute_env_id(with_env) + + +def test_env_id_stable_for_same_env_unsorted(): + """``submit_many`` always sorts before building the spec, but verify + the hash itself doesn't depend on the input order.""" + a = EnvSpec(deps=("numpy",), env=(("A", "1"), ("B", "2"))) + b = EnvSpec(deps=("numpy",), env=(("B", "2"), ("A", "1"))) + assert compute_env_id(a) == compute_env_id(b) diff --git a/tests/test_envelope_env_spec.py b/tests/test_envelope_env_spec.py new file mode 100644 index 0000000..d69c350 --- /dev/null +++ b/tests/test_envelope_env_spec.py @@ -0,0 +1,40 @@ +"""Wire round-trip for ``EnvSpec`` on the envelope.""" + +from __future__ import annotations + +from pymonik.envelope import EnvSpec, TaskEnvelope, decode, encode + + +def _roundtrip(env: TaskEnvelope) -> TaskEnvelope: + return decode(encode(env)) + + +def test_envelope_without_env_spec_default_none(): + env = TaskEnvelope(function_pickle=b"f", args_pickle=b"a", func_name="t") + rt = _roundtrip(env) + assert rt.env_spec is None + + +def test_envelope_with_env_spec_roundtrips(): + spec = EnvSpec(deps=("numpy>=2", "polars"), isolate=True, index_url="") + env = TaskEnvelope( + function_pickle=b"f", + args_pickle=b"a", + func_name="t", + env_spec=spec, + ) + rt = _roundtrip(env) + assert rt.env_spec is not None + assert rt.env_spec.deps == ("numpy>=2", "polars") + assert rt.env_spec.isolate is True + assert rt.env_spec.index_url == "" + + +def test_envelope_with_isolate_false_roundtrips(): + spec = EnvSpec(deps=("numpy",), isolate=False) + env = TaskEnvelope( + function_pickle=b"f", args_pickle=b"a", func_name="t", env_spec=spec + ) + rt = _roundtrip(env) + assert rt.env_spec is not None + assert rt.env_spec.isolate is False diff --git a/tests/test_map_starmap.py b/tests/test_map_starmap.py new file mode 100644 index 0000000..816dca5 --- /dev/null +++ b/tests/test_map_starmap.py @@ -0,0 +1,84 @@ +"""``map`` and ``starmap`` follow Python stdlib semantics. + +- ``map(f, *iterables)`` zips its iterables and applies the task to + each tuple — like ``builtins.map``. +- ``starmap(f, args_iter)`` takes an iterable of tuples and unpacks + each as positional args — like ``itertools.starmap``. + +The two are different shapes; using the wrong one is the kind of bug +the type checker won't catch (everything is ``Iterable[Any]`` in the +end), so the tests here pin the public surface. +""" + +from __future__ import annotations + +import pytest + +from pymonik import task +from pymonik.testing import LocalCluster + + +@task +def square(x: int) -> int: + return x * x + + +@task +def add(a: int, b: int) -> int: + return a + b + + +@task +def add3(a: int, b: int, c: int) -> int: + return a + b + c + + +def test_map_single_iterable(): + """One iterable, one positional arg per task.""" + with LocalCluster() as client: + with client.session() as s: + futs = square.map([1, 2, 3, 4]) + assert futs.results(timeout=10) == [1, 4, 9, 16] + + +def test_map_parallel_iterables(): + """Two iterables zipped, two args per task.""" + with LocalCluster() as client: + with client.session() as s: + futs = add.map([1, 3, 5], [2, 4, 6]) + assert futs.results(timeout=10) == [3, 7, 11] + + +def test_map_three_parallel_iterables(): + with LocalCluster() as client: + with client.session() as s: + futs = add3.map([1, 1, 1], [2, 2, 2], [3, 4, 5]) + assert futs.results(timeout=10) == [6, 7, 8] + + +def test_map_zip_stops_at_shortest(): + """Like Python's map: shortest iterable wins.""" + with LocalCluster() as client: + with client.session() as s: + futs = add.map([1, 2, 3, 4, 5], [10, 20]) + assert futs.results(timeout=10) == [11, 22] + + +def test_map_with_no_iterables_raises(): + with LocalCluster() as client: + with client.session() as s: + with pytest.raises(TypeError, match="at least one iterable"): + square.map() + + +def test_starmap_unpacks_tuples(): + with LocalCluster() as client: + with client.session() as s: + futs = add.starmap([(1, 2), (3, 4), (5, 6)]) + assert futs.results(timeout=10) == [3, 7, 11] + + +def test_local_call_unchanged(): + """The decorated function still works as a plain function.""" + assert add(2, 3) == 5 + assert square(7) == 49 diff --git a/tests/test_multi_partition.py b/tests/test_multi_partition.py new file mode 100644 index 0000000..9bc2cbd --- /dev/null +++ b/tests/test_multi_partition.py @@ -0,0 +1,86 @@ +"""Multi-partition routing on ``client.session(partition=[...])``.""" + +from __future__ import annotations + +import pytest + +from pymonik import task +from pymonik.errors import PymonikError +from pymonik.testing import LocalCluster + + +@task +def add(a: int, b: int) -> int: + return a + b + + +def test_session_accepts_partition_string(): + with LocalCluster() as client: + with client.session(partition="cpu") as s: + assert s.partitions == ("cpu",) + assert s.partition == "cpu" + + +def test_session_accepts_partition_list(): + with LocalCluster() as client: + with client.session(partition=["cpu", "gpu", "io"]) as s: + assert s.partitions == ("cpu", "gpu", "io") + assert s.partition == "cpu" # first is default + + +def test_empty_partition_list_rejected(): + with LocalCluster() as client: + with pytest.raises(ValueError, match="partition list cannot be empty"): + client.session(partition=[]) + + +def test_per_task_partition_within_set_succeeds(): + """LocalBackend has no partition constraint, so any selection is fine + in-process. Cluster-level enforcement happens via the ``allowed_partitions`` + backend hook on the real ``Session._ClientBackend``.""" + with LocalCluster() as client: + with client.session(partition=["cpu", "gpu"]) as s: + override = add.with_options(partition="gpu") + assert override.spawn(2, 3).result(timeout=10) == 5 + + +def test_per_task_partition_outside_set_rejected_by_cluster_backend(): + """The local backend reports ``allowed_partitions=None`` so it doesn't + enforce; this test reaches into the submission pipeline directly to + verify the validation logic with a backend that *does* report a set. + """ + from armonik.common import TaskDefinition, TaskOptions + + from pymonik._internal.submit import submit_many + from pymonik.future import Future + + class _StrictBackend: + @property + def session_id(self) -> str: + return "test" + + @property + def allowed_partitions(self) -> tuple[str, ...]: + return ("cpu",) + + def allocate_outputs(self, names): + raise AssertionError("should not be called — validation runs first") + + def upload_payloads(self, named): + raise AssertionError("should not be called") + + def submit(self, defs, opts): + raise AssertionError("should not be called") + + bad = add.with_options(partition="gpu") + with pytest.raises(PymonikError, match="partition 'gpu'"): + submit_many( + task=bad, + calls=[((1, 2), {})], + backend=_StrictBackend(), + blob_uploader=lambda b: "ignored", + spill_threshold=1024, + default_opts=type(bad.opts)(), + partition="cpu", + future_factory=lambda *a, **k: AssertionError("unreached"), + ) diff --git a/tests/test_otel.py b/tests/test_otel.py new file mode 100644 index 0000000..73d7507 --- /dev/null +++ b/tests/test_otel.py @@ -0,0 +1,171 @@ +"""OTel integration. + +Two regimes to verify: + +1. **OTel not installed / disabled** — every helper is a no-op, the + submission pipeline runs unchanged. The most important property is + "pymonik works without ``opentelemetry-api`` even being importable", + which is hard to test from inside a process where it *is* importable + — we settle for "all helpers no-op when ``setup(force=False)``". + +2. **OTel installed and enabled** — spans are produced for the documented + call sites; the worker can extract the trace context the client put + in the envelope; nesting works. +""" + +from __future__ import annotations + +import pytest + +from pymonik import task +from pymonik._internal import _otel +from pymonik.envelope import TaskEnvelope, decode, encode + +# Skip when the OTel API isn't installed — the no-op behaviour is exercised +# by every other test in the suite (which run with otel disabled). +pytest.importorskip("opentelemetry") + +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry import trace + + +@pytest.fixture(scope="module") +def _provider_and_exporter(): + """OTel only allows one TracerProvider per process; install once.""" + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + trace.set_tracer_provider(provider) + _otel._initialised = False + _otel._enabled = False + _otel.setup(force=True) + yield exporter + + +@pytest.fixture +def in_memory_exporter(_provider_and_exporter): + """Per-test handle. Clears spans between tests.""" + _provider_and_exporter.clear() + yield _provider_and_exporter + + +def test_no_otel_helpers_are_noop_when_disabled(): + """Force-disabled state: start_span yields None, inject is no-op. + + Run BEFORE the in_memory_exporter fixture installs a provider — once + installed it's irreversible per process. We achieve that by ordering + via name (this test starts with "test_no_otel" → first lex order). + """ + saved_init, saved_enabled = _otel._initialised, _otel._enabled + _otel._initialised = False + _otel._enabled = False + try: + _otel.setup(force=False) + with _otel.start_span("pymonik.test") as span: + assert span is None + carrier: dict[str, str] = {} + _otel.inject_context(carrier) + assert carrier == {} + assert _otel.current_trace_id_hex() is None + finally: + _otel._initialised, _otel._enabled = saved_init, saved_enabled + + +def test_start_span_emits_attributes(in_memory_exporter): + with _otel.start_span( + "pymonik.test", attrs={"pymonik.thing": 42}, kind="client" + ) as span: + assert span is not None + spans = in_memory_exporter.get_finished_spans() + assert len(spans) == 1 + s = spans[0] + assert s.name == "pymonik.test" + assert s.attributes is not None + assert s.attributes.get("pymonik.thing") == 42 + + +def test_inject_extract_round_trip(in_memory_exporter): + """Client injects trace context into a carrier; worker-equivalent + extracts it and a span opened there is a child of the original.""" + with _otel.start_span("client.parent") as parent: + carrier: dict[str, str] = {} + _otel.inject_context(carrier) + assert "traceparent" in carrier # W3C default propagator key + + # Simulate the worker: clear the current span, then re-attach via + # the carrier and open a child span. + with _otel.use_extracted_context(carrier): + with _otel.start_span("worker.child") as child: + assert child is not None + + spans = {s.name: s for s in in_memory_exporter.get_finished_spans()} + assert "client.parent" in spans + assert "worker.child" in spans + # The child's parent span context matches the parent's span id. + parent_span = spans["client.parent"] + child_span = spans["worker.child"] + assert child_span.parent is not None + assert child_span.parent.trace_id == parent_span.context.trace_id + assert child_span.parent.span_id == parent_span.context.span_id + + +def test_envelope_carries_otel_context_round_trip(): + env = TaskEnvelope( + function_pickle=b"f", + args_pickle=b"a", + func_name="t", + otel_context=(("traceparent", "00-abcd-ef01-01"),), + ) + rt = decode(encode(env)) + assert rt.otel_context == (("traceparent", "00-abcd-ef01-01"),) + + +def test_submit_pipeline_injects_context(in_memory_exporter): + """End-to-end: submit_many through LocalCluster → worker dispatch. + Verify the envelope produced carries trace context, the worker-side + span is a child of the client-side submit span.""" + from pymonik.testing import LocalCluster + + @task + def echo(x: int) -> int: + return x + + with LocalCluster() as client: + with client.session() as s: + assert echo.spawn(7).result(timeout=15) == 7 + + spans_by_name = {s.name: s for s in in_memory_exporter.get_finished_spans()} + assert "pymonik.submit" in spans_by_name + assert "pymonik.task.run" in spans_by_name + submit = spans_by_name["pymonik.submit"] + run = spans_by_name["pymonik.task.run"] + # Same trace, run is a descendant of submit. + assert run.context.trace_id == submit.context.trace_id + assert run.parent is not None + assert run.parent.span_id == submit.context.span_id + + +def test_submit_span_has_useful_attributes(in_memory_exporter): + from pymonik.testing import LocalCluster + + @task + def add(a: int, b: int) -> int: + return a + b + + with LocalCluster() as client: + with client.session(partition="cpu") as s: + add.map(range(4), range(1, 5)).results(timeout=15) + + submit_spans = [ + s for s in in_memory_exporter.get_finished_spans() if s.name == "pymonik.submit" + ] + assert len(submit_spans) == 1 + attrs = submit_spans[0].attributes + assert attrs is not None + assert attrs.get("pymonik.func") == "add" + assert attrs.get("pymonik.count") == 4 + assert attrs.get("pymonik.partition") == "cpu" diff --git a/tests/test_polling_options.py b/tests/test_polling_options.py new file mode 100644 index 0000000..51468b1 --- /dev/null +++ b/tests/test_polling_options.py @@ -0,0 +1,43 @@ +"""``polling_interval`` / ``polling_chunk`` constructor knobs propagate.""" + +from __future__ import annotations + +from pymonik import PymonikClient + + +def test_polling_defaults(): + # No connection — just check the kwargs landed on the instance. + c = PymonikClient(endpoint="grpcs://test:5001") + assert c._polling_interval == 0.5 + assert c._polling_chunk == 500 + + +def test_polling_overrides(): + c = PymonikClient( + endpoint="grpcs://test:5001", + polling_interval=2.0, + polling_chunk=100, + ) + assert c._polling_interval == 2.0 + assert c._polling_chunk == 100 + + +def test_session_inherits_client_polling_settings(monkeypatch): + """Session reads the polling kwargs from the client so a single + ``PymonikClient(polling_interval=...)`` configures every session.""" + from pymonik.session import Session + + c = PymonikClient( + endpoint="grpcs://test:5001", + polling_interval=3.5, + polling_chunk=42, + ) + # Pure constructor; doesn't connect. + s = Session( + c, + partition="x", + polling_interval=c._polling_interval, + polling_chunk=c._polling_chunk, + ) + assert s._polling_interval == 3.5 + assert s._polling_chunk == 42 diff --git a/tests/test_runtime_deps_local.py b/tests/test_runtime_deps_local.py new file mode 100644 index 0000000..4491482 --- /dev/null +++ b/tests/test_runtime_deps_local.py @@ -0,0 +1,125 @@ +"""End-to-end runtime-deps tests via ``LocalCluster``. + +These tests exercise the real env_builder + subprocess dispatcher against +a real ``uv``. They're slow-ish (the first install takes 10-30s) so we +mark them ``slow`` and skip if ``uv`` is missing on the box. + +Run only the fast suite: + uv run pytest tests/ -m "not slow" + +Run with these: + uv run pytest tests/test_runtime_deps_local.py -v +""" + +from __future__ import annotations + +import shutil +import sys + +import pytest + +from pymonik import task +from pymonik.testing import LocalCluster + +pytestmark = [ + pytest.mark.skipif( + shutil.which("uv") is None, + reason="uv not on PATH; runtime-deps tests need it", + ), + pytest.mark.skipif( + sys.platform == "win32", reason="subprocess wire is POSIX-only for now" + ), + pytest.mark.slow, +] + + +import numpy as np + + +@task +def numpy_arange_sum(n: int) -> int: + return int(np.arange(n).sum()) + + +@task(deps=["numpy"]) +def per_task_numpy(n: int) -> int: + return int(np.arange(n).sum()) + + +@task +def imports_unavailable() -> str: + import numpy # noqa: F401 + + return "ok" + + +def test_runtime_deps_default_path(tmp_path, monkeypatch): + """Default ``isolate=False`` — in-process splice.""" + monkeypatch.setenv("PYMONIK_ENVS_ROOT", str(tmp_path / "envs")) + monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path / "uv-cache")) + with LocalCluster() as client: + with client.session(deps=["numpy"]) as s: + assert numpy_arange_sum.spawn(100).result(timeout=600) == sum(range(100)) + + +def test_runtime_deps_env_reuse(tmp_path, monkeypatch): + """Two tasks in the same session — only one install.""" + monkeypatch.setenv("PYMONIK_ENVS_ROOT", str(tmp_path / "envs")) + monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path / "uv-cache")) + with LocalCluster() as client: + with client.session(deps=["numpy"]) as s: + a = numpy_arange_sum.spawn(10).result(timeout=600) + b = numpy_arange_sum.spawn(20).result(timeout=120) + assert a == sum(range(10)) + assert b == sum(range(20)) + + +def test_runtime_deps_isolate_true_subprocess(tmp_path, monkeypatch): + """Explicit opt-in ``isolate=True`` — subprocess per task.""" + monkeypatch.setenv("PYMONIK_ENVS_ROOT", str(tmp_path / "envs")) + monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path / "uv-cache")) + with LocalCluster() as client: + with client.session(deps=["numpy"], isolate=True) as s: + assert numpy_arange_sum.spawn(50).result(timeout=600) == sum(range(50)) + + +def test_no_deps_no_install(tmp_path, monkeypatch): + """Sessions without deps must NOT touch the envs root. + + Catches a regression where an empty/None deps list accidentally + triggers a venv build. + """ + envs_root = tmp_path / "envs" + monkeypatch.setenv("PYMONIK_ENVS_ROOT", str(envs_root)) + + @task + def trivial(x: int) -> int: + return x * 2 + + with LocalCluster() as client: + with client.session() as s: + assert trivial.spawn(21).result(timeout=30) == 42 + + assert not envs_root.exists() or not any(envs_root.iterdir()) + + +def test_per_task_deps_override(tmp_path, monkeypatch): + monkeypatch.setenv("PYMONIK_ENVS_ROOT", str(tmp_path / "envs")) + monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path / "uv-cache")) + with LocalCluster() as client: + with client.session() as s: + assert per_task_numpy.spawn(10).result(timeout=600) == sum(range(10)) + + +def test_install_failure_surfaces_typed_error(tmp_path, monkeypatch): + monkeypatch.setenv("PYMONIK_ENVS_ROOT", str(tmp_path / "envs")) + monkeypatch.setenv("UV_CACHE_DIR", str(tmp_path / "uv-cache")) + from pymonik.errors import PymonikError + + with LocalCluster() as client: + with client.session(deps=["this-package-does-not-exist-12345-zzz"]) as s: + fut = imports_unavailable.spawn() + with pytest.raises((PymonikError, Exception)) as exc_info: + fut.result(timeout=120) + # Surface should contain the failing package name somewhere. + assert "this-package-does-not-exist" in str(exc_info.value).lower() diff --git a/tests/test_session_deps_sugar.py b/tests/test_session_deps_sugar.py new file mode 100644 index 0000000..a8bd761 --- /dev/null +++ b/tests/test_session_deps_sugar.py @@ -0,0 +1,45 @@ +"""``client.session(deps=..., isolate=..., index_url=...)`` sugar. + +These flags should land on the session's default TaskOpts so every +submitted envelope picks them up (without each ``@task`` having to +opt in individually). +""" + +from __future__ import annotations + +from pymonik import task +from pymonik.testing import LocalCluster + + +@task +def echo(x: int) -> int: + return x + + +def test_session_deps_propagate_to_default_options(): + with LocalCluster() as client: + with client.session(deps=["numpy"]) as s: + assert s._default_opts.deps == ("numpy",) + + +def test_session_isolate_false_propagates(): + with LocalCluster() as client: + with client.session(deps=["numpy"], isolate=False) as s: + assert s._default_opts.deps == ("numpy",) + assert s._default_opts.isolate is False + + +def test_session_index_url_propagates(): + with LocalCluster() as client: + with client.session( + deps=["numpy"], + index_url="https://idx.example/simple/", + ) as s: + assert s._default_opts.index_url == "https://idx.example/simple/" + + +def test_session_no_deps_keeps_empty_opts(): + with LocalCluster() as client: + with client.session() as s: + assert s._default_opts.deps is None + assert s._default_opts.isolate is None diff --git a/tests/test_tail_and_multiresult.py b/tests/test_tail_and_multiresult.py new file mode 100644 index 0000000..4c94d53 --- /dev/null +++ b/tests/test_tail_and_multiresult.py @@ -0,0 +1,458 @@ +"""Tail-call sub-tasking + multi-output tasks. + +Two related primitives: + +- ``task.tail(*args)`` returns a ``TailPromise``; the framework binds it + to an output id at worker dispatch time. Replaces the old + ``_delegate=True`` flag. +- ``MultiResult(field=...)`` runs a multi-output task; the field set is + extracted from the function body's AST at decoration time. Each field + becomes an independent ArmoniK output id; downstream tasks block only + on the field they consume. + +These tests use ``LocalCluster`` end-to-end (real envelope encoding, +real ref resolution, real dispatcher logic). Cluster behaviour mirrors +the local path bit-for-bit. +""" + +from __future__ import annotations + +import pytest + +from pymonik import ( + MultiResult, + MultiResultHandle, + PymonikConnectionError, + TailPromise, + TaskFailed, + task, +) +from pymonik.errors import PymonikError +from pymonik.multiresult import MultiResult as _MR # alias for AST tests +from pymonik.testing import LocalCluster + + +# ---------- decoration-time AST extraction ---------- + + +def test_extract_simple_multiresult_shape(): + @task + def split(x: int): + return MultiResult(double=x * 2, triple=x * 3) + + assert split.multi_fields == ("double", "triple") + + +def test_extract_consistent_branches(): + @task + def conditional(x: int): + if x > 0: + return MultiResult(a=x, b=-x) + return MultiResult(a=-x, b=x) + + assert conditional.multi_fields == ("a", "b") + + +def test_inconsistent_branches_raises_at_decoration(): + with pytest.raises(PymonikError, match="inconsistent MultiResult shapes"): + + @task + def bad(x: int): + if x > 0: + return MultiResult(a=x, b=x) + return MultiResult(a=x, b=x, c=x) + + +def test_kwargs_expansion_is_rejected(): + with pytest.raises(PymonikError, match="\\*\\*kwargs"): + + @task + def bad(x): + d = {"a": 1, "b": 2} + return MultiResult(**d) + + +def test_no_multiresult_means_single_output_task(): + @task + def regular(x: int) -> int: + return x * 2 + + assert regular.multi_fields is None + + +def test_explicit_outputs_decorator_kwarg(): + @task(outputs=("alpha", "beta")) + def via_helper(x): + # AST can't see the construction; explicit outputs declares it. + return _build_mr(x) + + def _build_mr(x): + return MultiResult(alpha=x, beta=-x) + + assert via_helper.multi_fields == ("alpha", "beta") + + +# ---------- tail-call basics ---------- + + +def test_tail_returns_tailpromise(): + @task + def add(a, b): + return a + b + + promise = add.tail(2, 3) + assert isinstance(promise, TailPromise) + assert promise.task is add + + +def test_tail_promise_cant_be_awaited_directly(): + @task + def add(a, b): + return a + b + + promise = add.tail(2, 3) + + async def _try(): + await promise + + import asyncio + + with pytest.raises(PymonikError, match="cannot be awaited"): + asyncio.run(_try()) + + +def test_old_delegate_kwarg_raises(): + @task + def add(a, b): + return a + b + + with LocalCluster() as client: + with client.session() as s: + with pytest.raises(PymonikError, match="_delegate=True"): + add.spawn(2, 3, _delegate=True) + + +# ---------- whole-task tail-call (single-output) ---------- + + +def test_whole_task_tail_call_end_to_end(): + @task + def grow(n: int) -> int: + return n + 1 + + @task + def adaptive(n: int) -> int: + if n < 5: + return grow.tail(n) + return n + + with LocalCluster() as client: + with client.session() as s: + assert adaptive.spawn(2).result(timeout=15) == 3 + assert adaptive.spawn(7).result(timeout=15) == 7 + + +def test_tail_call_chain(): + """A tail-called task can itself tail-call. Each leaf writes to the + chain's original parent output id.""" + + @task + def increment_chain(n: int, acc: int) -> int: + if n == 0: + return acc + return increment_chain.tail(n - 1, acc + 1) + + with LocalCluster() as client: + with client.session() as s: + assert increment_chain.spawn(5, 0).result(timeout=20) == 5 + + +# ---------- multi-output tasks ---------- + + +def test_spawn_returns_multiresulthandle_for_multi_output_task(): + @task + def split(x: int): + return MultiResult(double=x * 2, triple=x * 3) + + with LocalCluster() as client: + with client.session() as s: + out = split.spawn(7) + assert isinstance(out, MultiResultHandle) + assert set(out.fields) == {"double", "triple"} + + +def test_per_field_access_is_independent_future(): + @task + def split(x: int): + return MultiResult(double=x * 2, triple=x * 3) + + with LocalCluster() as client: + with client.session() as s: + out = split.spawn(5) + assert out.double.result(timeout=10) == 10 + assert out.triple.result(timeout=10) == 15 + + +def test_handle_result_returns_view_supporting_attr_and_dict_access(): + @task + def split(x: int): + return MultiResult(double=x * 2, triple=x * 3) + + with LocalCluster() as client: + with client.session() as s: + out = split.spawn(4) + resolved = out.result(timeout=10) + # Equality with plain dict — backwards-compat with prior return type. + assert resolved == {"double": 8, "triple": 12} + # Attribute access. + assert resolved.double == 8 + assert resolved.triple == 12 + # Dict-style access. + assert resolved["double"] == 8 + assert resolved["triple"] == 12 + # Iter, len, contains, dict()-coercion all work. + assert sorted(resolved) == ["double", "triple"] + assert len(resolved) == 2 + assert "double" in resolved + assert dict(resolved) == {"double": 8, "triple": 12} + # Repr is field=value form, no quoted keys. + assert "double=8" in repr(resolved) + assert "triple=12" in repr(resolved) + + +def test_handle_result_view_unknown_attr_raises_attribute_error(): + @task + def split(x: int): + return MultiResult(a=x, b=x * 2) + + with LocalCluster() as client: + with client.session() as s: + out = split.spawn(3) + view = out.result(timeout=10) + with pytest.raises(AttributeError, match="not a field"): + _ = view.nonexistent + + +def test_field_can_feed_downstream_task(): + """Independent scheduling: a downstream task that consumes one field + runs as soon as that field arrives.""" + + @task + def split(x: int): + return MultiResult(a=x * 2, b=x * 3) + + @task + def add_one(v: int) -> int: + return v + 1 + + with LocalCluster() as client: + with client.session() as s: + out = split.spawn(10) + d = add_one.spawn(out.a) + assert d.result(timeout=10) == 21 + + +def test_returning_wrong_shape_fails_task(): + @task + def lying(x: int): + # AST would catch this if the literal differed; but we make + # the runtime see a different *value* shape via a manual + # MultiResult construction. Decoration extracts {"a", "b"}; + # at runtime we omit "b". + return MultiResult(a=x) + + # AST extracted the lying call's fields literally — {"a"}. So + # technically this wouldn't raise. Use the explicit-outputs path + # to force a mismatch. + + @task(outputs=("a", "b")) + def really_lying(x: int): + return MultiResult(a=x, c=x) # ← AST sees {a, c}, but explicit outputs says {a, b} + + # The explicit outputs win. At runtime the worker sees {a, c} + # vs declared {a, b} — should fail with shape mismatch. + with LocalCluster() as client: + with client.session() as s: + fut = really_lying.spawn(7).a + with pytest.raises(TaskFailed, match="shape mismatch"): + fut.result(timeout=10) + + +def test_returning_plain_value_from_multi_output_task_fails(): + @task(outputs=("a", "b")) + def pretender(x): + return x * 2 # plain int instead of MultiResult + + with LocalCluster() as client: + with client.session() as s: + fut = pretender.spawn(5).a + with pytest.raises(TaskFailed, match="MultiResult"): + fut.result(timeout=10) + + +def test_returning_multiresult_from_single_output_task_fails(): + @task + def pretender(x): + # Build MultiResult dynamically so AST doesn't see it + cls = MultiResult + return cls(a=x, b=x) + + with LocalCluster() as client: + with client.session() as s: + with pytest.raises(TaskFailed, match="MultiResult"): + pretender.spawn(7).result(timeout=10) + + +# ---------- per-field tail-call ---------- + + +def test_per_field_tail(): + @task + def heavy(x: int) -> int: + return x * 100 + + @task + def split(x: int): + return MultiResult( + heavy_double=heavy.tail(x * 2), + quick=x + 1, + ) + + with LocalCluster() as client: + with client.session() as s: + out = split.spawn(5) + assert out.heavy_double.result(timeout=15) == 1000 + assert out.quick.result(timeout=15) == 6 + + +def test_per_field_tail_to_multi_output_task_rejected(): + @task + def inner_split(x: int): + return MultiResult(p=x, q=x) + + @task + def outer(x: int): + return MultiResult( + a=inner_split.tail(x), # multi-output child — not allowed per-field + b=x, + ) + + with LocalCluster() as client: + with client.session() as s: + with pytest.raises(TaskFailed, match="multi-output"): + outer.spawn(5).a.result(timeout=10) + + +def test_per_field_future_from_spawn_is_rejected(): + """Inside a MultiResult, fields should not be Futures from .spawn(). + Use .tail() instead.""" + + @task + def inner(x: int) -> int: + return x * 2 + + @task + def parent(x: int): + # Calling .spawn() inside the worker creates a worker-stub + # Future; placing it as a MultiResult field is rejected. + return MultiResult(a=inner.spawn(x), b=x + 1) + + with LocalCluster() as client: + with client.session() as s: + with pytest.raises(TaskFailed, match="\\.tail\\(\\) for delegation"): + parent.spawn(7).a.result(timeout=10) + + +# ---------- whole-task tail-call across multi-output schemas ---------- + + +def test_whole_task_tail_to_matching_multi_output_child(): + @task + def child(x: int): + return MultiResult(a=x * 2, b=x * 3) + + @task + def parent(x: int): + if x > 100: + return child.tail(x) # same shape as parent + return MultiResult(a=x, b=x * 5) + + with LocalCluster() as client: + with client.session() as s: + small = parent.spawn(7).result(timeout=10) + assert small == {"a": 7, "b": 35} + big = parent.spawn(200).result(timeout=10) + assert big == {"a": 400, "b": 600} + + +def test_whole_task_tail_with_mismatched_child_fails(): + @task + def wrong_shape(x: int): + return MultiResult(x=x, y=x, z=x) # different fields + + @task + def parent(x: int): + return wrong_shape.tail(x) # parent is single-output; child is multi + + with LocalCluster() as client: + with client.session() as s: + with pytest.raises(TaskFailed, match="multi-output"): + parent.spawn(5).result(timeout=10) + + +# ---------- TailPromise repr / API ---------- + + +def test_tail_promise_repr_has_task_name(): + @task + def add(a, b): + return a + b + + p = add.tail(1, 2) + assert "add" in repr(p) + + +def test_multiresult_repr_has_fields(): + mr = MultiResult(a=1, b=2) + assert "a=1" in repr(mr) and "b=2" in repr(mr) + + +def test_multiresult_empty_construction_rejected(): + with pytest.raises(PymonikError, match="at least one field"): + MultiResult() + + +def test_multiresult_reserved_field_name_rejected(): + """Field names that collide with MultiResultHandle attributes raise.""" + with pytest.raises(PymonikError, match="collides with a MultiResultHandle"): + MultiResult(task_id="x", b=1) + with pytest.raises(PymonikError, match="collides with a MultiResultHandle"): + MultiResult(result="x", b=1) + + +def test_multiresult_underscore_field_rejected(): + with pytest.raises(PymonikError, match="underscore-prefixed names are reserved"): + MultiResult(_internal=1, b=2) + + +def test_decoration_rejects_reserved_field_in_outputs(): + with pytest.raises(PymonikError, match="collide with MultiResultHandle"): + + @task(outputs=("task_id", "value")) + def bad(x): + return MultiResult(task_id=x, value=x) + + +def test_decoration_rejects_cache_with_multi_output(): + """cache=True is incompatible with multi-output (no per-field cache).""" + with pytest.raises(PymonikError, match="cache=True is not compatible"): + + @task(cache=True) + def cached_split(x): + return MultiResult(a=x, b=x * 2) + + +__all__ = [ + "PymonikConnectionError", +] # silence unused-import warnings diff --git a/tests/test_taskopts_deps.py b/tests/test_taskopts_deps.py new file mode 100644 index 0000000..c67a24f --- /dev/null +++ b/tests/test_taskopts_deps.py @@ -0,0 +1,38 @@ +"""TaskOpts deps/isolate/index_url merge semantics.""" + +from __future__ import annotations + +from pymonik.options import EMPTY, TaskOpts + + +def test_deps_passthrough_on_merge(): + a = TaskOpts(deps=("numpy",)) + b = TaskOpts(retries=3) + merged = a.merge(b) + assert merged.deps == ("numpy",) + assert merged.retries == 3 + + +def test_deps_override_on_merge(): + a = TaskOpts(deps=("numpy",)) + b = TaskOpts(deps=("polars",)) + # Right-hand wins for non-None deps. Composition is intentional — + # @task(deps=...) overrides session deps for that one task. + assert a.merge(b).deps == ("polars",) + + +def test_isolate_default_inherits(): + a = TaskOpts(deps=("numpy",)) + assert a.isolate is None # inherits → worker reads env_spec.isolate=True default + + +def test_isolate_explicit_false_propagates(): + a = TaskOpts(deps=("numpy",), isolate=False) + b = EMPTY + assert a.merge(b).isolate is False + assert b.merge(a).isolate is False + + +def test_index_url_carries(): + a = TaskOpts(deps=("numpy",), index_url="https://idx.example/simple/") + assert a.merge(EMPTY).index_url == "https://idx.example/simple/" diff --git a/tests/test_wait.py b/tests/test_wait.py new file mode 100644 index 0000000..3594e43 --- /dev/null +++ b/tests/test_wait.py @@ -0,0 +1,141 @@ +"""``.wait()`` — block without retrieving the value or raising on failure. + +`.result()` blocks AND returns the value AND raises on failure; +`.wait()` only blocks. Useful when you want to synchronise without +surfacing the value (e.g. fan out, wait for everything, then decide +what to do based on `.done` / `.task_id`). +""" + +from __future__ import annotations + +import pytest + +from pymonik import MultiResult, TaskFailed, TaskTimeout, task +from pymonik.testing import LocalCluster + + +@task +def add(a: int, b: int) -> int: + return a + b + + +@task +def boom(x: int) -> int: + raise ValueError(f"nope {x}") + + +def test_future_wait_returns_self_for_chaining(): + with LocalCluster() as client: + with client.session() as s: + fut = add.spawn(2, 3) + same = fut.wait() + assert same is fut + assert fut.done + assert fut.result() == 5 # cheap retrieval after wait + + +def test_future_wait_doesnt_raise_on_failure(): + """A failed task is *done*; wait() doesn't propagate the error.""" + with LocalCluster() as client: + with client.session() as s: + fut = boom.spawn(7) + fut.wait(timeout=10) # no raise + assert fut.done + with pytest.raises(TaskFailed): # raise lives on .result() + fut.result() + + +def test_future_wait_timeout(): + with LocalCluster() as client: + with client.session() as s: + fut = add.spawn(2, 3) + fut.wait(timeout=10) + # After session exits and fut is resolved, wait still works (already done). + + +def test_future_wait_timeout_raises_when_unfinished(): + """A future that never resolves should raise TaskTimeout.""" + from pymonik.future import Future + + fut: Future[int] = Future.__new__(Future) + import threading + + fut._session = None # type: ignore[assignment] + fut._task_id = "test" + fut._result_id = "test" + fut._done = threading.Event() + fut._aio_done = None + fut._aio_loop = None + fut._outcome = None + fut._error = None + fut._is_worker_stub = False + fut._retry_state = None + fut._retry_attempt = 0 + fut._cache_key = None + + with pytest.raises(TaskTimeout): + fut.wait(timeout=0.1) + + +def test_futurelist_wait_returns_self(): + with LocalCluster() as client: + with client.session() as s: + futs = add.map(range(4), range(1, 5)) + same = futs.wait() + assert same is futs + assert all(f.done for f in futs) + assert futs.results() == [1, 3, 5, 7] + + +def test_multiresulthandle_wait_returns_self(): + @task + def split(x: int): + return MultiResult(double=x * 2, triple=x * 3) + + with LocalCluster() as client: + with client.session() as s: + out = split.spawn(5) + same = out.wait() + assert same is out + assert out.done + assert out.result() == {"double": 10, "triple": 15} + + +def test_wait_then_result_is_two_step_chain(): + """The composed pattern: wait without retrieving, then retrieve.""" + with LocalCluster() as client: + with client.session() as s: + fut = add.spawn(10, 20) + value = fut.wait(timeout=10).result() + assert value == 30 + + +def test_future_wait_async_returns_self(): + """Async ``.wait_async()`` round-trip. asyncio-only — the rest of + PymoniK's async surface is also asyncio-pinned today.""" + import asyncio + + async def _run(): + async with LocalCluster() as client: + async with client.session_async() as s: + fut = add.spawn(2, 3) + same = await fut.wait_async() + assert same is fut + assert fut.done + assert fut.result() == 5 + + asyncio.run(_run()) + + +def test_futurelist_wait_async_returns_self(): + import asyncio + + async def _run(): + async with LocalCluster() as client: + async with client.session_async() as s: + futs = add.map(range(3), range(1, 4)) + same = await futs.wait_async() + assert same is futs + assert all(f.done for f in futs) + + asyncio.run(_run()) diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..2c1e462 --- /dev/null +++ b/uv.lock @@ -0,0 +1,709 @@ +version = 1 +revision = 1 +requires-python = "==3.11.*" + +[[package]] +name = "anyio" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353 }, +] + +[[package]] +name = "armonik" +version = "3.29.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "deprecation" }, + { name = "grpcio" }, + { name = "grpcio-tools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/58/7eb5a770e623146ee1668cde3f9235722b766edf081f372ef238a6bef9b7/armonik-3.29.0.tar.gz", hash = "sha256:d6af212f93a828c5cad8963a1e5e2fd838f8d9143b0a011d6fb76c4204a942e8", size = 90466 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/46/eb7e77f2c42efda283d5c731cdf4ead1eb96d66cc9e35cb3f54a9233f64f/armonik-3.29.0-py3-none-any.whl", hash = "sha256:1b7f9b68d2aa3dee00f288d9c18b0ccfdd45daabe79062ff55d895437df5c9d3", size = 149171 }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548 }, +] + +[[package]] +name = "basedpyright" +version = "1.39.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodejs-wheel-binaries" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/19/5a5b9b9197973da732638957be3a65cf514d2f5a4964eeedbf33b6c65bbd/basedpyright-1.39.3.tar.gz", hash = "sha256:2f794e6b5f4260fb89f614ca6cd23c6f305373bb6b50c4ed7794ff2ae647fb14", size = 25503187 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/5c/f950c1239ad26f3bb453e665428a2cf1893995de725a5eb0b64a2520b366/basedpyright-1.39.3-py3-none-any.whl", hash = "sha256:aba760dc83307727554f936d6b4381caa14482f30dbc2173167710e217c1f7ab", size = 12419181 }, +] + +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344 }, + { url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560 }, + { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613 }, + { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476 }, + { url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374 }, + { url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597 }, + { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574 }, + { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971 }, + { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972 }, + { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078 }, + { url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076 }, + { url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820 }, + { url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635 }, +] + +[[package]] +name = "click" +version = "8.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502 }, +] + +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228 }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, +] + +[[package]] +name = "cryptography" +version = "46.0.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/47/93/ac8f3d5ff04d54bc814e961a43ae5b0b146154c89c61b47bb07557679b18/cryptography-46.0.7.tar.gz", hash = "sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5", size = 750652 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/5d/4a8f770695d73be252331e60e526291e3df0c9b27556a90a6b47bccca4c2/cryptography-46.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:ea42cbe97209df307fdc3b155f1b6fa2577c0defa8f1f7d3be7d31d189108ad4", size = 7179869 }, + { url = "https://files.pythonhosted.org/packages/5f/45/6d80dc379b0bbc1f9d1e429f42e4cb9e1d319c7a8201beffd967c516ea01/cryptography-46.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325", size = 4275492 }, + { url = "https://files.pythonhosted.org/packages/4a/9a/1765afe9f572e239c3469f2cb429f3ba7b31878c893b246b4b2994ffe2fe/cryptography-46.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308", size = 4426670 }, + { url = "https://files.pythonhosted.org/packages/8f/3e/af9246aaf23cd4ee060699adab1e47ced3f5f7e7a8ffdd339f817b446462/cryptography-46.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:73510b83623e080a2c35c62c15298096e2a5dc8d51c3b4e1740211839d0dea77", size = 4280275 }, + { url = "https://files.pythonhosted.org/packages/0f/54/6bbbfc5efe86f9d71041827b793c24811a017c6ac0fd12883e4caa86b8ed/cryptography-46.0.7-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cbd5fb06b62bd0721e1170273d3f4d5a277044c47ca27ee257025146c34cbdd1", size = 4928402 }, + { url = "https://files.pythonhosted.org/packages/2d/cf/054b9d8220f81509939599c8bdbc0c408dbd2bdd41688616a20731371fe0/cryptography-46.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:420b1e4109cc95f0e5700eed79908cef9268265c773d3a66f7af1eef53d409ef", size = 4459985 }, + { url = "https://files.pythonhosted.org/packages/f9/46/4e4e9c6040fb01c7467d47217d2f882daddeb8828f7df800cb806d8a2288/cryptography-46.0.7-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:24402210aa54baae71d99441d15bb5a1919c195398a87b563df84468160a65de", size = 3990652 }, + { url = "https://files.pythonhosted.org/packages/36/5f/313586c3be5a2fbe87e4c9a254207b860155a8e1f3cca99f9910008e7d08/cryptography-46.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8a469028a86f12eb7d2fe97162d0634026d92a21f3ae0ac87ed1c4a447886c83", size = 4279805 }, + { url = "https://files.pythonhosted.org/packages/69/33/60dfc4595f334a2082749673386a4d05e4f0cf4df8248e63b2c3437585f2/cryptography-46.0.7-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9694078c5d44c157ef3162e3bf3946510b857df5a3955458381d1c7cfc143ddb", size = 4892883 }, + { url = "https://files.pythonhosted.org/packages/c7/0b/333ddab4270c4f5b972f980adef4faa66951a4aaf646ca067af597f15563/cryptography-46.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:42a1e5f98abb6391717978baf9f90dc28a743b7d9be7f0751a6f56a75d14065b", size = 4459756 }, + { url = "https://files.pythonhosted.org/packages/d2/14/633913398b43b75f1234834170947957c6b623d1701ffc7a9600da907e89/cryptography-46.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91bbcb08347344f810cbe49065914fe048949648f6bd5c2519f34619142bbe85", size = 4410244 }, + { url = "https://files.pythonhosted.org/packages/10/f2/19ceb3b3dc14009373432af0c13f46aa08e3ce334ec6eff13492e1812ccd/cryptography-46.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5d1c02a14ceb9148cc7816249f64f623fbfee39e8c03b3650d842ad3f34d637e", size = 4674868 }, + { url = "https://files.pythonhosted.org/packages/1a/bb/a5c213c19ee94b15dfccc48f363738633a493812687f5567addbcbba9f6f/cryptography-46.0.7-cp311-abi3-win32.whl", hash = "sha256:d23c8ca48e44ee015cd0a54aeccdf9f09004eba9fc96f38c911011d9ff1bd457", size = 3026504 }, + { url = "https://files.pythonhosted.org/packages/2b/02/7788f9fefa1d060ca68717c3901ae7fffa21ee087a90b7f23c7a603c32ae/cryptography-46.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:397655da831414d165029da9bc483bed2fe0e75dde6a1523ec2fe63f3c46046b", size = 3488363 }, + { url = "https://files.pythonhosted.org/packages/a7/7f/cd42fc3614386bc0c12f0cb3c4ae1fc2bbca5c9662dfed031514911d513d/cryptography-46.0.7-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:462ad5cb1c148a22b2e3bcc5ad52504dff325d17daf5df8d88c17dda1f75f2a4", size = 7165618 }, + { url = "https://files.pythonhosted.org/packages/a5/d0/36a49f0262d2319139d2829f773f1b97ef8aef7f97e6e5bd21455e5a8fb5/cryptography-46.0.7-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:84d4cced91f0f159a7ddacad249cc077e63195c36aac40b4150e7a57e84fffe7", size = 4270628 }, + { url = "https://files.pythonhosted.org/packages/8a/6c/1a42450f464dda6ffbe578a911f773e54dd48c10f9895a23a7e88b3e7db5/cryptography-46.0.7-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:128c5edfe5e5938b86b03941e94fac9ee793a94452ad1365c9fc3f4f62216832", size = 4415405 }, + { url = "https://files.pythonhosted.org/packages/9a/92/4ed714dbe93a066dc1f4b4581a464d2d7dbec9046f7c8b7016f5286329e2/cryptography-46.0.7-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5e51be372b26ef4ba3de3c167cd3d1022934bc838ae9eaad7e644986d2a3d163", size = 4272715 }, + { url = "https://files.pythonhosted.org/packages/b7/e6/a26b84096eddd51494bba19111f8fffe976f6a09f132706f8f1bf03f51f7/cryptography-46.0.7-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cdf1a610ef82abb396451862739e3fc93b071c844399e15b90726ef7470eeaf2", size = 4918400 }, + { url = "https://files.pythonhosted.org/packages/c7/08/ffd537b605568a148543ac3c2b239708ae0bd635064bab41359252ef88ed/cryptography-46.0.7-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1d25aee46d0c6f1a501adcddb2d2fee4b979381346a78558ed13e50aa8a59067", size = 4450634 }, + { url = "https://files.pythonhosted.org/packages/16/01/0cd51dd86ab5b9befe0d031e276510491976c3a80e9f6e31810cce46c4ad/cryptography-46.0.7-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:cdfbe22376065ffcf8be74dc9a909f032df19bc58a699456a21712d6e5eabfd0", size = 3985233 }, + { url = "https://files.pythonhosted.org/packages/92/49/819d6ed3a7d9349c2939f81b500a738cb733ab62fbecdbc1e38e83d45e12/cryptography-46.0.7-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:abad9dac36cbf55de6eb49badd4016806b3165d396f64925bf2999bcb67837ba", size = 4271955 }, + { url = "https://files.pythonhosted.org/packages/80/07/ad9b3c56ebb95ed2473d46df0847357e01583f4c52a85754d1a55e29e4d0/cryptography-46.0.7-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:935ce7e3cfdb53e3536119a542b839bb94ec1ad081013e9ab9b7cfd478b05006", size = 4879888 }, + { url = "https://files.pythonhosted.org/packages/b8/c7/201d3d58f30c4c2bdbe9b03844c291feb77c20511cc3586daf7edc12a47b/cryptography-46.0.7-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:35719dc79d4730d30f1c2b6474bd6acda36ae2dfae1e3c16f2051f215df33ce0", size = 4449961 }, + { url = "https://files.pythonhosted.org/packages/a5/ef/649750cbf96f3033c3c976e112265c33906f8e462291a33d77f90356548c/cryptography-46.0.7-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7bbc6ccf49d05ac8f7d7b5e2e2c33830d4fe2061def88210a126d130d7f71a85", size = 4401696 }, + { url = "https://files.pythonhosted.org/packages/41/52/a8908dcb1a389a459a29008c29966c1d552588d4ae6d43f3a1a4512e0ebe/cryptography-46.0.7-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a1529d614f44b863a7b480c6d000fe93b59acee9c82ffa027cfadc77521a9f5e", size = 4664256 }, + { url = "https://files.pythonhosted.org/packages/4b/fa/f0ab06238e899cc3fb332623f337a7364f36f4bb3f2534c2bb95a35b132c/cryptography-46.0.7-cp38-abi3-win32.whl", hash = "sha256:f247c8c1a1fb45e12586afbb436ef21ff1e80670b2861a90353d9b025583d246", size = 3013001 }, + { url = "https://files.pythonhosted.org/packages/d2/f1/00ce3bde3ca542d1acd8f8cfa38e446840945aa6363f9b74746394b14127/cryptography-46.0.7-cp38-abi3-win_amd64.whl", hash = "sha256:506c4ff91eff4f82bdac7633318a526b1d1309fc07ca76a3ad182cb5b686d6d3", size = 3472985 }, + { url = "https://files.pythonhosted.org/packages/63/0c/dca8abb64e7ca4f6b2978769f6fea5ad06686a190cec381f0a796fdcaaba/cryptography-46.0.7-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc9ab8856ae6cf7c9358430e49b368f3108f050031442eaeb6b9d87e4dcf4e4f", size = 3476879 }, + { url = "https://files.pythonhosted.org/packages/3a/ea/075aac6a84b7c271578d81a2f9968acb6e273002408729f2ddff517fed4a/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d3b99c535a9de0adced13d159c5a9cf65c325601aa30f4be08afd680643e9c15", size = 4219700 }, + { url = "https://files.pythonhosted.org/packages/6c/7b/1c55db7242b5e5612b29fc7a630e91ee7a6e3c8e7bf5406d22e206875fbd/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d02c738dacda7dc2a74d1b2b3177042009d5cab7c7079db74afc19e56ca1b455", size = 4385982 }, + { url = "https://files.pythonhosted.org/packages/cb/da/9870eec4b69c63ef5925bf7d8342b7e13bc2ee3d47791461c4e49ca212f4/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:04959522f938493042d595a736e7dbdff6eb6cc2339c11465b3ff89343b65f65", size = 4219115 }, + { url = "https://files.pythonhosted.org/packages/f4/72/05aa5832b82dd341969e9a734d1812a6aadb088d9eb6f0430fc337cc5a8f/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:3986ac1dee6def53797289999eabe84798ad7817f3e97779b5061a95b0ee4968", size = 4385479 }, + { url = "https://files.pythonhosted.org/packages/20/2a/1b016902351a523aa2bd446b50a5bc1175d7a7d1cf90fe2ef904f9b84ebc/cryptography-46.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4", size = 3412829 }, +] + +[[package]] +name = "deprecated" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298 }, +] + +[[package]] +name = "deprecation" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178 }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.74.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/20/18/a746c8344152d368a5aac738d4c857012f2c5d1fd2eac7e17b647a7861bd/googleapis_common_protos-1.74.0.tar.gz", hash = "sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1", size = 151254 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/b0/be5d3329badb9230b765de6eea66b73abd5944bdeb5afb3562ddcd80ae84/googleapis_common_protos-1.74.0-py3-none-any.whl", hash = "sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5", size = 300743 }, +] + +[[package]] +name = "grpcio" +version = "1.62.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/31/2e/1e2cd0edeaaeaae0ab9df2615492725d0f2f689d3f9aa3b088356af7a584/grpcio-1.62.3.tar.gz", hash = "sha256:4439bbd759636e37b66841117a66444b454937e27f0125205d2d117d7827c643", size = 26297933 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/3a/c3da1df7d55cfe481b02221d8e22e603f43fdf1646f2c02e7d69370d5e4b/grpcio-1.62.3-cp311-cp311-linux_armv7l.whl", hash = "sha256:c8bb1a7aa82af6c7713cdf9dcb8f4ea1024ac7ce82bb0a0a82a49aea5237da34", size = 4774831 }, + { url = "https://files.pythonhosted.org/packages/c8/12/c769a65437081cce5c7aece3fcc0a0e9e5293d85455cbdc9dd4edc9c56b9/grpcio-1.62.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:57823dc7299c4f258ae9c32fd327d29f729d359c34d7612b36e48ed45b3ab8d0", size = 10019810 }, + { url = "https://files.pythonhosted.org/packages/18/08/b2a2c66f183240e55ca99a0dd85c2c2ad1cb0846e7ad628900843a49a155/grpcio-1.62.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:1de3d04d9a4ec31ebe848ae1fe61e4cbc367fb9495cbf6c54368e60609a998d9", size = 5294405 }, + { url = "https://files.pythonhosted.org/packages/fe/ac/6e523ccaf068dc022de3cc798f539bd070fd45e4db953241cff23fd867a6/grpcio-1.62.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:325c56ce94d738c31059cf91376f625d3effdff8f85c96660a5fd6395d5a707f", size = 5830738 }, + { url = "https://files.pythonhosted.org/packages/78/a0/3a1c81854f76d8c1462533e18cc754b9d3e434234678cb2273b1bff5885c/grpcio-1.62.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c175b252d063af388523a397dbe8edbc4319761f5ee892a8a0f5890acc067362", size = 5547176 }, + { url = "https://files.pythonhosted.org/packages/7b/1a/9462e3c81429c4b299a7222df8cd3c1f84625eecc353967d5ddfec43f8f2/grpcio-1.62.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:25cd75dc73c5269932413e517be778640402f18cf9a81147e68645bd8af18ab0", size = 6116809 }, + { url = "https://files.pythonhosted.org/packages/40/6c/99f922adeb6ca65b6f84f6bf1032f5b94c042eef65ab9b13d438819e9205/grpcio-1.62.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1b85d35a7d9638c03321dfe466645b87e23c30df1266f9e04bbb5f44e7579a9", size = 5779755 }, + { url = "https://files.pythonhosted.org/packages/ae/b7/d48ff1a5f6ad59748df7717a3f101679e0e1b9004f5b6efa96831c2b847c/grpcio-1.62.3-cp311-cp311-win32.whl", hash = "sha256:6be243f3954b0ca709f56f9cae926c84ac96e1cce19844711e647a1f1db88b99", size = 3167821 }, + { url = "https://files.pythonhosted.org/packages/9f/90/32ba95836adb03dd04a393f1b9a8bde7919e9f08ee5fd94dc77351f9305f/grpcio-1.62.3-cp311-cp311-win_amd64.whl", hash = "sha256:e9ffdb7bc9ccd56ec201aec3eab3432e1e820335b5a16ad2b37e094218dcd7a6", size = 3729044 }, +] + +[[package]] +name = "grpcio-tools" +version = "1.62.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/54/fa/b69bd8040eafc09b88bb0ec0fea59e8aacd1a801e688af087cead213b0d0/grpcio-tools-1.62.3.tar.gz", hash = "sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833", size = 4538520 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/52/2dfe0a46b63f5ebcd976570aa5fc62f793d5a8b169e211c6a5aede72b7ae/grpcio_tools-1.62.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:703f46e0012af83a36082b5f30341113474ed0d91e36640da713355cd0ea5d23", size = 5147623 }, + { url = "https://files.pythonhosted.org/packages/f0/2e/29fdc6c034e058482e054b4a3c2432f84ff2e2765c1342d4f0aa8a5c5b9a/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:7cc83023acd8bc72cf74c2edbe85b52098501d5b74d8377bfa06f3e929803492", size = 2719538 }, + { url = "https://files.pythonhosted.org/packages/f9/60/abe5deba32d9ec2c76cdf1a2f34e404c50787074a2fee6169568986273f1/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ff7d58a45b75df67d25f8f144936a3e44aabd91afec833ee06826bd02b7fbe7", size = 3070964 }, + { url = "https://files.pythonhosted.org/packages/bc/ad/e2b066684c75f8d9a48508cde080a3a36618064b9cadac16d019ca511444/grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f2483ea232bd72d98a6dc6d7aefd97e5bc80b15cd909b9e356d6f3e326b6e43", size = 2805003 }, + { url = "https://files.pythonhosted.org/packages/9c/3f/59bf7af786eae3f9d24ee05ce75318b87f541d0950190ecb5ffb776a1a58/grpcio_tools-1.62.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:962c84b4da0f3b14b3cdb10bc3837ebc5f136b67d919aea8d7bb3fd3df39528a", size = 3685154 }, + { url = "https://files.pythonhosted.org/packages/f1/79/4dd62478b91e27084c67b35a2316ce8a967bd8b6cb8d6ed6c86c3a0df7cb/grpcio_tools-1.62.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8ad0473af5544f89fc5a1ece8676dd03bdf160fb3230f967e05d0f4bf89620e3", size = 3297942 }, + { url = "https://files.pythonhosted.org/packages/b8/cb/86449ecc58bea056b52c0b891f26977afc8c4464d88c738f9648da941a75/grpcio_tools-1.62.3-cp311-cp311-win32.whl", hash = "sha256:db3bc9fa39afc5e4e2767da4459df82b095ef0cab2f257707be06c44a1c2c3e5", size = 910231 }, + { url = "https://files.pythonhosted.org/packages/45/a4/9736215e3945c30ab6843280b0c6e1bff502910156ea2414cd77fbf1738c/grpcio_tools-1.62.3-cp311-cp311-win_amd64.whl", hash = "sha256:e0898d412a434e768a0c7e365acabe13ff1558b767e400936e26b5b6ed1ee51f", size = 1052496 }, +] + +[[package]] +name = "idna" +version = "3.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/cc/762dfb036166873f0059f3b7de4565e1b5bc3d6f28a414c13da27e442f99/idna-3.13.tar.gz", hash = "sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242", size = 194210 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/13/ad7d7ca3808a898b4612b6fe93cde56b53f3034dcde235acb1f0e1df24c6/idna-3.13-py3-none-any.whl", hash = "sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3", size = 68629 }, +] + +[[package]] +name = "importlib-metadata" +version = "8.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/bd/fa8ce65b0a7d4b6d143ec23b0f5fd3f7ab80121078c465bc02baeaab22dc/importlib_metadata-8.4.0.tar.gz", hash = "sha256:9a547d3bc3608b025f93d403fdd1aae741c24fbb8314df4b155675742ce303c5", size = 54320 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/14/362d31bf1076b21e1bcdcb0dc61944822ff263937b804a79231df2774d28/importlib_metadata-8.4.0-py3-none-any.whl", hash = "sha256:66f342cc6ac9818fc6ff340576acd24d65ba0b3efabb2b4ac08b598965a4a2f1", size = 26269 }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484 }, +] + +[[package]] +name = "msgspec" +version = "0.21.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/60/f79b9b013a16fa3a58350c9295ddc6789f2e335f36ea61ed10a21b215364/msgspec-0.21.1.tar.gz", hash = "sha256:2313508e394b0d208f8f56892ca9b2799e2561329de9763b19619595a6c0f72c", size = 319193 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/7f/bbc4e74cd33d316b75541149e4d35b163b63bce066530ae185a2ec3b5bfc/msgspec-0.21.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b504b6e7f7a22a24b27232b73034421692147865162daaec9f3bf62439007c87", size = 193131 }, + { url = "https://files.pythonhosted.org/packages/c1/60/504886af1aaf854112663b842d5eea9a15d9588f9bf7d0d2df736424b84d/msgspec-0.21.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4692b7c1609155708c4418f88e92f63c13fdf08aa095c84bae82bad75b53389b", size = 186597 }, + { url = "https://files.pythonhosted.org/packages/fa/54/d24ddeaa65b5278c9e67f48ce3c17a9831e8f3722f3c8322ee120aca22ef/msgspec-0.21.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3124010b3815451494c85ff345e693cb9fe5889cfcbbef39ed8622e0e72319c", size = 215158 }, + { url = "https://files.pythonhosted.org/packages/9f/75/bb79c8b89a93ae23cd33c0d802373f16feaf9633f05d8af77091350dda0a/msgspec-0.21.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6badc03b9725352219cca017bfe71c61f2fbd0fb5982b410ac17c97c213deb30", size = 219856 }, + { url = "https://files.pythonhosted.org/packages/b4/9c/c5ca26b46f0ebbd3a6683695ef89396712cb9e4199fd1f0bc1dd968216b1/msgspec-0.21.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5d2d4116ebe3035a78d9ec76e99a9d64e5fa6d44fe61a9c5de7fd1acf54bcc69", size = 220314 }, + { url = "https://files.pythonhosted.org/packages/c8/31/645a351c4285dce40ed6755c3dcc0aa648e26dacb20a98018fe2cce5e87b/msgspec-0.21.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0d1009f6715f5bff3b54d4ff5c7428ad96197e0534e1645b8e9b955890c84664", size = 223215 }, + { url = "https://files.pythonhosted.org/packages/09/af/8bf15736a6dd3cb4f90c5467f6dc39197d2daaf10754490cdc0aa17b7312/msgspec-0.21.1-cp311-cp311-win_amd64.whl", hash = "sha256:c6faffe5bb644ec884052679af4dfd776d4b5ca90e4a7ec7e7e319e4e6b93a6e", size = 188554 }, + { url = "https://files.pythonhosted.org/packages/ef/29/cc7db3a165b62d16e64a83f82eccb79655055cb5bc1f60459a6f9d7c82f2/msgspec-0.21.1-cp311-cp311-win_arm64.whl", hash = "sha256:ee9e3f11fa94603f7d673bf795cfa31b549c4a2c723bc39b45beb1e7f5a3fb99", size = 174517 }, +] + +[[package]] +name = "nodejs-wheel-binaries" +version = "24.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/70/a1e4f4d5986768ab90cc860b1cc3660fd2ded74ca175a900a5c29f839c7d/nodejs_wheel_binaries-24.15.0.tar.gz", hash = "sha256:b43f5c4f6e5768d8845b2ae4682eb703a19bf7aadc84187e2d903ed3a611c859", size = 8057 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/66/54051d14853d6ab4fb85f8be9b042b530be653357fb9a19557498bc91ab7/nodejs_wheel_binaries-24.15.0-py2.py3-none-macosx_13_0_arm64.whl", hash = "sha256:a6232fa8b754220941f52388c8ead923f7c1c7fdf0ea0d98f657523bd9a81ef4", size = 55173485 }, + { url = "https://files.pythonhosted.org/packages/ad/5f/66acada164da5ca10a0824db021aa7394ae18396c550cd9280e839a43126/nodejs_wheel_binaries-24.15.0-py2.py3-none-macosx_13_0_x86_64.whl", hash = "sha256:001a6b62c69d9109c1738163cca00608dd2722e8663af59300054ea02610972d", size = 55348100 }, + { url = "https://files.pythonhosted.org/packages/0d/2d/0cbd5ff40c9bb030ca1735d8f8793bd74f08a4cbd49100a1d19313ea57ab/nodejs_wheel_binaries-24.15.0-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:0fbc48765e60ed0ff30d43898dbf5cadbadf2e5f1e7f204afc2b01493b7ebce6", size = 59668206 }, + { url = "https://files.pythonhosted.org/packages/da/d5/91ac63951ec75927a486b83b8cafe650e360fa70ac01dc94adfb32b93b97/nodejs_wheel_binaries-24.15.0-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:20ee0536809795da8a4942fc1ab4cbdebbcaaf29383eab67ba8874268fb00008", size = 60206736 }, + { url = "https://files.pythonhosted.org/packages/db/72/dc22776974d928869c0c30d23ee98ed7df254243c2df68f09f5963e8e8b8/nodejs_wheel_binaries-24.15.0-py2.py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1fade6c214285e72472ca40a631e98ff36559671cd5eefc8bf009471d67f04b4", size = 61720456 }, + { url = "https://files.pythonhosted.org/packages/01/0a/34461b9050cb45ee371dccdefc622aef6351506ea2691b08fc761ca67150/nodejs_wheel_binaries-24.15.0-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3984cb8d87766567aee67a49743227ab40ede6f47734ec990ff90e50b74e7740", size = 62326172 }, + { url = "https://files.pythonhosted.org/packages/c9/17/09252bf35672dba926649d59dfe51443a0f6955ad13784e91131d5ec82a2/nodejs_wheel_binaries-24.15.0-py2.py3-none-win_amd64.whl", hash = "sha256:a437601956b532dcb3082046e6978e622733f90edc0932cbb9adb3bb97a16501", size = 41543461 }, + { url = "https://files.pythonhosted.org/packages/0a/7e/b649777d148e1e0c2ce349156603cdb12f7ed99921b95d93717393650193/nodejs_wheel_binaries-24.15.0-py2.py3-none-win_arm64.whl", hash = "sha256:bdf4a431e08321a32efc604111c6f23941f87055d796a537e8c4110daecad23f", size = 39233248 }, +] + +[[package]] +name = "numpy" +version = "2.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/9f/b8cef5bffa569759033adda9481211426f12f53299629b410340795c2514/numpy-2.4.4.tar.gz", hash = "sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0", size = 20731587 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/c6/4218570d8c8ecc9704b5157a3348e486e84ef4be0ed3e38218ab473c83d2/numpy-2.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f983334aea213c99992053ede6168500e5f086ce74fbc4acc3f2b00f5762e9db", size = 16976799 }, + { url = "https://files.pythonhosted.org/packages/dd/92/b4d922c4a5f5dab9ed44e6153908a5c665b71acf183a83b93b690996e39b/numpy-2.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:72944b19f2324114e9dc86a159787333b77874143efcf89a5167ef83cfee8af0", size = 14971552 }, + { url = "https://files.pythonhosted.org/packages/8a/dc/df98c095978fa6ee7b9a9387d1d58cbb3d232d0e69ad169a4ce784bde4fd/numpy-2.4.4-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:86b6f55f5a352b48d7fbfd2dbc3d5b780b2d79f4d3c121f33eb6efb22e9a2015", size = 5476566 }, + { url = "https://files.pythonhosted.org/packages/28/34/b3fdcec6e725409223dd27356bdf5a3c2cc2282e428218ecc9cb7acc9763/numpy-2.4.4-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:ba1f4fc670ed79f876f70082eff4f9583c15fb9a4b89d6188412de4d18ae2f40", size = 6806482 }, + { url = "https://files.pythonhosted.org/packages/68/62/63417c13aa35d57bee1337c67446761dc25ea6543130cf868eace6e8157b/numpy-2.4.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a87ec22c87be071b6bdbd27920b129b94f2fc964358ce38f3822635a3e2e03d", size = 15973376 }, + { url = "https://files.pythonhosted.org/packages/cf/c5/9fcb7e0e69cef59cf10c746b84f7d58b08bc66a6b7d459783c5a4f6101a6/numpy-2.4.4-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df3775294accfdd75f32c74ae39fcba920c9a378a2fc18a12b6820aa8c1fb502", size = 16925137 }, + { url = "https://files.pythonhosted.org/packages/7e/43/80020edacb3f84b9efdd1591120a4296462c23fd8db0dde1666f6ef66f13/numpy-2.4.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0d4e437e295f18ec29bc79daf55e8a47a9113df44d66f702f02a293d93a2d6dd", size = 17329414 }, + { url = "https://files.pythonhosted.org/packages/fd/06/af0658593b18a5f73532d377188b964f239eb0894e664a6c12f484472f97/numpy-2.4.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6aa3236c78803afbcb255045fbef97a9e25a1f6c9888357d205ddc42f4d6eba5", size = 18658397 }, + { url = "https://files.pythonhosted.org/packages/e6/ce/13a09ed65f5d0ce5c7dd0669250374c6e379910f97af2c08c57b0608eee4/numpy-2.4.4-cp311-cp311-win32.whl", hash = "sha256:30caa73029a225b2d40d9fae193e008e24b2026b7ee1a867b7ee8d96ca1a448e", size = 6239499 }, + { url = "https://files.pythonhosted.org/packages/bd/63/05d193dbb4b5eec1eca73822d80da98b511f8328ad4ae3ca4caf0f4db91d/numpy-2.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:6bbe4eb67390b0a0265a2c25458f6b90a409d5d069f1041e6aff1e27e3d9a79e", size = 12614257 }, + { url = "https://files.pythonhosted.org/packages/87/c5/8168052f080c26fa984c413305012be54741c9d0d74abd7fbeeccae3889f/numpy-2.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:fcfe2045fd2e8f3cb0ce9d4ba6dba6333b8fa05bb8a4939c908cd43322d14c7e", size = 10486775 }, + { url = "https://files.pythonhosted.org/packages/6b/33/8fae8f964a4f63ed528264ddf25d2b683d0b663e3cba26961eb838a7c1bd/numpy-2.4.4-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:58c8b5929fcb8287cbd6f0a3fae19c6e03a5c48402ae792962ac465224a629a4", size = 16854491 }, + { url = "https://files.pythonhosted.org/packages/bc/d0/1aabee441380b981cf8cdda3ae7a46aa827d1b5a8cce84d14598bc94d6d9/numpy-2.4.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:eea7ac5d2dce4189771cedb559c738a71512768210dc4e4753b107a2048b3d0e", size = 14895830 }, + { url = "https://files.pythonhosted.org/packages/a5/b8/aafb0d1065416894fccf4df6b49ef22b8db045187949545bced89c034b8e/numpy-2.4.4-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:51fc224f7ca4d92656d5a5eb315f12eb5fe2c97a66249aa7b5f562528a3be38c", size = 5400927 }, + { url = "https://files.pythonhosted.org/packages/d6/77/063baa20b08b431038c7f9ff5435540c7b7265c78cf56012a483019ca72d/numpy-2.4.4-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:28a650663f7314afc3e6ec620f44f333c386aad9f6fc472030865dc0ebb26ee3", size = 6715557 }, + { url = "https://files.pythonhosted.org/packages/c7/a8/379542d45a14f149444c5c4c4e7714707239ce9cc1de8c2803958889da14/numpy-2.4.4-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19710a9ca9992d7174e9c52f643d4272dcd1558c5f7af7f6f8190f633bd651a7", size = 15804253 }, + { url = "https://files.pythonhosted.org/packages/a2/c8/f0a45426d6d21e7ea3310a15cf90c43a14d9232c31a837702dba437f3373/numpy-2.4.4-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b2aec6af35c113b05695ebb5749a787acd63cafc83086a05771d1e1cd1e555f", size = 16753552 }, + { url = "https://files.pythonhosted.org/packages/04/74/f4c001f4714c3ad9ce037e18cf2b9c64871a84951eaa0baf683a9ca9301c/numpy-2.4.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:f2cf083b324a467e1ab358c105f6cad5ea950f50524668a80c486ff1db24e119", size = 12509075 }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "importlib-metadata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/83/93114b6de85a98963aec218a51509a52ed3f8de918fe91eb0f7299805c3f/opentelemetry_api-1.27.0.tar.gz", hash = "sha256:ed673583eaa5f81b5ce5e86ef7cdaf622f88ef65f0b9aab40b843dcae5bef342", size = 62693 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/1f/737dcdbc9fea2fa96c1b392ae47275165a7c641663fbb08a8d252968eed2/opentelemetry_api-1.27.0-py3-none-any.whl", hash = "sha256:953d5871815e7c30c81b56d910c707588000fff7a3ca1c73e6531911d53065e7", size = 63970 }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/2e/7eaf4ba595fb5213cf639c9158dfb64aacb2e4c7d74bfa664af89fa111f4/opentelemetry_exporter_otlp_proto_common-1.27.0.tar.gz", hash = "sha256:159d27cf49f359e3798c4c3eb8da6ef4020e292571bd8c5604a2a573231dd5c8", size = 17860 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/27/4610ab3d9bb3cde4309b6505f98b3aabca04a26aa480aa18cede23149837/opentelemetry_exporter_otlp_proto_common-1.27.0-py3-none-any.whl", hash = "sha256:675db7fffcb60946f3a5c43e17d1168a3307a94a930ecf8d2ea1f286f3d4f79a", size = 17848 }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/d0/c1e375b292df26e0ffebf194e82cd197e4c26cc298582bda626ce3ce74c5/opentelemetry_exporter_otlp_proto_grpc-1.27.0.tar.gz", hash = "sha256:af6f72f76bcf425dfb5ad11c1a6d6eca2863b91e63575f89bb7b4b55099d968f", size = 26244 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/80/32217460c2c64c0568cea38410124ff680a9b65f6732867bbf857c4d8626/opentelemetry_exporter_otlp_proto_grpc-1.27.0-py3-none-any.whl", hash = "sha256:56b5bbd5d61aab05e300d9d62a6b3c134827bbd28d0b12f2649c2da368006c9e", size = 18541 }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/59/959f0beea798ae0ee9c979b90f220736fbec924eedbefc60ca581232e659/opentelemetry_proto-1.27.0.tar.gz", hash = "sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6", size = 34749 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/56/3d2d826834209b19a5141eed717f7922150224d1a982385d19a9444cbf8d/opentelemetry_proto-1.27.0-py3-none-any.whl", hash = "sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace", size = 52464 }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/9a/82a6ac0f06590f3d72241a587cb8b0b751bd98728e896cc4cbd4847248e6/opentelemetry_sdk-1.27.0.tar.gz", hash = "sha256:d525017dea0ccce9ba4e0245100ec46ecdc043f2d7b8315d56b19aff0904fa6f", size = 145019 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/bd/a6602e71e315055d63b2ff07172bd2d012b4cba2d4e00735d74ba42fc4d6/opentelemetry_sdk-1.27.0-py3-none-any.whl", hash = "sha256:365f5e32f920faf0fd9e14fdfd92c086e317eaa5f860edba9cdc17a380d9197d", size = 110505 }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.48b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "opentelemetry-api" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/89/1724ad69f7411772446067cdfa73b598694c8c91f7f8c922e344d96d81f9/opentelemetry_semantic_conventions-0.48b0.tar.gz", hash = "sha256:12d74983783b6878162208be57c9effcb89dc88691c64992d70bb89dc00daa1a", size = 89445 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/7a/4f0063dbb0b6c971568291a8bc19a4ca70d3c185db2d956230dd67429dfc/opentelemetry_semantic_conventions-0.48b0-py3-none-any.whl", hash = "sha256:a0de9f45c413a8669788a38569c7e0a11ce6ce97861a628cca785deecdc32a1f", size = 149685 }, +] + +[[package]] +name = "outcome" +version = "1.3.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/df/77698abfac98571e65ffeb0c1fba8ffd692ab8458d617a0eed7d9a8d38f2/outcome-1.3.0.post0.tar.gz", hash = "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8", size = 21060 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/8b/5ab7257531a5d830fc8000c476e63c935488d74609b50f9384a643ec0a62/outcome-1.3.0.post0-py2.py3-none-any.whl", hash = "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b", size = 10692 }, +] + +[[package]] +name = "packaging" +version = "26.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/de/0d2b39fb4af88a0258f3bac87dfcbb48e73fbdea4a2ed0e2213f9a4c2f9a/packaging-26.1.tar.gz", hash = "sha256:f042152b681c4bfac5cae2742a55e103d27ab2ec0f3d88037136b6bfe7c9c5de", size = 215519 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/c2/920ef838e2f0028c8262f16101ec09ebd5969864e5a64c4c05fad0617c56/packaging-26.1-py3-none-any.whl", hash = "sha256:5d9c0669c6285e491e0ced2eee587eaf67b670d94a19e94e3984a481aba6802f", size = 95831 }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538 }, +] + +[[package]] +name = "polars" +version = "1.40.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "polars-runtime-32" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/8c/bc9bc948058348ed43117cecc3007cd608f395915dae8a00974579a5dab1/polars-1.40.1.tar.gz", hash = "sha256:ab2694134b137596b5a59bfd7b4c54ebbc9b59f9403127f18e32d363777552e8", size = 733574 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/91/74fc60d94488685a92ac9d49d7ec55f3e91fe9b77942a6235a5fa7f249c3/polars-1.40.1-py3-none-any.whl", hash = "sha256:c0f861219d1319cdea45c4ce4d30355a47176b8f98dcedf95ea8269f131b8abd", size = 828723 }, +] + +[[package]] +name = "polars-runtime-32" +version = "1.40.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ba/26d40f039be9f552b5fd7365a621bdfc0f8e912ef77094ae4693491b0bae/polars_runtime_32-1.40.1.tar.gz", hash = "sha256:37f3065615d1bf90d03b5326222df4c5c1f8a5d33e50470aa588e3465e6eb814", size = 2935843 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/46/22c8af5eed68ac2eeb556e0fa3ca8a7b798e984ceff4450888f3b5ac61fd/polars_runtime_32-1.40.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b748ef652270cc49e9e69f99a035e0eb4d5f856d42bcd6ac4d9d80a40142aa1e", size = 52098755 }, + { url = "https://files.pythonhosted.org/packages/c6/3e/48599a38009ca60ff82a6f38c8a621ce3c0286aa7397c7d79e741bd9060e/polars_runtime_32-1.40.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:d249b3743e05986060cec0a7aaa542d020df6c6b876e556023a310efd581f9be", size = 46367542 }, + { url = "https://files.pythonhosted.org/packages/43/e9/384bc069367a1a36ee31c13782c178dbd039b2b873b772d4a0fc23a2373d/polars_runtime_32-1.40.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5987b30e7aa1059d069498496e8dda35afd592b0ac3d46ed87e3ff8df1ad652c", size = 50252104 }, + { url = "https://files.pythonhosted.org/packages/15/ef/7d57ceb0651af74194e97ed6583e148d352f03d696090221b8059cdfc90b/polars_runtime_32-1.40.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d7f42a8b3f16fc66002cc0f6516f7dd7653396886ae0ed362ab95c0b3408b59", size = 56250788 }, + { url = "https://files.pythonhosted.org/packages/10/0f/e4b3ffc748827a14a474ec9c42e45c066050e440fec57e914091d9adda75/polars_runtime_32-1.40.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e5f7becc237a7ec9d9a10878dc8e54b73bbf4e2d94a2991c37d7a0b38590d8f9", size = 50432590 }, + { url = "https://files.pythonhosted.org/packages/d9/0b/b8d95fbed869fa4caabe9c400e4210374913b376e925e96fdcfa9be6416b/polars_runtime_32-1.40.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:992d14cf191dde043d36fbdbc98a65e43fbc7e9a5024cecd45f838ac4988c1ee", size = 54155564 }, + { url = "https://files.pythonhosted.org/packages/06/d9/d091d8fb5cbed5e9536adfed955c4c89987a4cc3b8e73ae4532402b91c74/polars_runtime_32-1.40.1-cp310-abi3-win_amd64.whl", hash = "sha256:f78bb2abd00101cbb23cc0cb068f7e36e081057a15d2ec2dde3dda280709f030", size = 51829755 }, + { url = "https://files.pythonhosted.org/packages/65/ad/b33c3022a394f3eb55c3310597cec615412a8a33880055eee191d154a628/polars_runtime_32-1.40.1-cp310-abi3-win_arm64.whl", hash = "sha256:b5cbfaf6b085b420b4bfcbe24e8f665076d1cccfdb80c0484c02a023ce205537", size = 45822104 }, +] + +[[package]] +name = "protobuf" +version = "4.25.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/8e/d08c41a8c004e1d437ef467e7c4f9c3295cd784eba48ed5d1d01f94b1dad/protobuf-4.25.9.tar.gz", hash = "sha256:b0dc7e7c68de8b1ce831dacb12fb407e838edbb8b6cc0dc3a2a6b4cbf6de9cff", size = 381040 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/e9/59435bd04bdd46cb38c42a336b22f9843e8e586ff83c35a5423f8b14704e/protobuf-4.25.9-cp310-abi3-win32.whl", hash = "sha256:bde396f568b0b46fc8fbfe9f02facf25b6755b2578a3b8ac61e74b9d69499e03", size = 392879 }, + { url = "https://files.pythonhosted.org/packages/f3/16/42a5c7f1001783d2b5bfcecde10127f09010f78982c86ae409122ce3ece6/protobuf-4.25.9-cp310-abi3-win_amd64.whl", hash = "sha256:3683c05154252206f7cb2d371626514b3708199d9bcf683b503dabf3a2e38e06", size = 413900 }, + { url = "https://files.pythonhosted.org/packages/56/5b/0074a0a9eb01f3d1c4648ca5e81b22090c811b210b61df9018ac6d6c5cda/protobuf-4.25.9-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:9560813560e6ee72c11ca8873878bdb7ee003c96a57ebb013245fe84e2540904", size = 394826 }, + { url = "https://files.pythonhosted.org/packages/54/aa/b2dba856f64c36b2a06c67be1472de98cca07a2322d0f0cbf03279a40e5b/protobuf-4.25.9-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:999146ef02e7fa6a692477badd1528bcd7268df211852a3df2d834ba2b480791", size = 294191 }, + { url = "https://files.pythonhosted.org/packages/a8/5c/53f18822017b8bda6bd8bb4e02048e911fdc79a3dafdc83ab994fe922a84/protobuf-4.25.9-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:438c636de8fb706a0de94a12a268ef1ae8f5ba5ae655a7671fcda5968ba3c9be", size = 295178 }, + { url = "https://files.pythonhosted.org/packages/16/28/d5065b212685875d3924bcdb3201cbf467cb4d58a18aa19a8dfd99ea80a9/protobuf-4.25.9-py3-none-any.whl", hash = "sha256:d49b615e7c935194ac161f0965699ac84df6112c378e05ec53da65d2e4cbb6d4", size = 156822 }, +] + +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172 }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151 }, +] + +[[package]] +name = "pymonik" +version = "2.0.0a0" +source = { editable = "." } +dependencies = [ + { name = "anyio" }, + { name = "armonik" }, + { name = "click" }, + { name = "cloudpickle" }, + { name = "grpcio" }, + { name = "msgspec" }, + { name = "pyyaml" }, + { name = "structlog" }, +] + +[package.optional-dependencies] +otel = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-sdk" }, +] + +[package.dev-dependencies] +dev = [ + { name = "basedpyright" }, + { name = "numpy" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-sdk" }, + { name = "polars" }, + { name = "pytest" }, + { name = "ruff" }, + { name = "trio" }, + { name = "types-pyyaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "anyio", specifier = ">=4.3" }, + { name = "armonik", specifier = ">=3.25.0" }, + { name = "click", specifier = ">=8.1" }, + { name = "cloudpickle", specifier = ">=3.1" }, + { name = "grpcio", specifier = ">=1.60" }, + { name = "msgspec", specifier = ">=0.19" }, + { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.27" }, + { name = "opentelemetry-exporter-otlp-proto-grpc", marker = "extra == 'otel'", specifier = ">=1.27" }, + { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.27" }, + { name = "pyyaml", specifier = ">=6.0.3" }, + { name = "structlog", specifier = ">=24.1" }, +] +provides-extras = ["otel"] + +[package.metadata.requires-dev] +dev = [ + { name = "basedpyright", specifier = ">=1.18" }, + { name = "numpy", specifier = ">=2" }, + { name = "opentelemetry-api", specifier = ">=1.27.0" }, + { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.27.0" }, + { name = "opentelemetry-sdk", specifier = ">=1.27.0" }, + { name = "polars", specifier = ">=1" }, + { name = "pytest", specifier = ">=8" }, + { name = "ruff", specifier = ">=0.11" }, + { name = "trio", specifier = ">=0.27" }, + { name = "types-pyyaml", specifier = ">=6.0.12.20260408" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249 }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826 }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577 }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556 }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114 }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638 }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463 }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986 }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543 }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763 }, +] + +[[package]] +name = "ruff" +version = "0.15.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/8d/192f3d7103816158dfd5ea50d098ef2aec19194e6cbccd4b3485bdb2eb2d/ruff-0.15.11.tar.gz", hash = "sha256:f092b21708bf0e7437ce9ada249dfe688ff9a0954fc94abab05dcea7dcd29c33", size = 4637264 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/1e/6aca3427f751295ab011828e15e9bf452200ac74484f1db4be0197b8170b/ruff-0.15.11-py3-none-linux_armv6l.whl", hash = "sha256:e927cfff503135c558eb581a0c9792264aae9507904eb27809cdcff2f2c847b7", size = 10607943 }, + { url = "https://files.pythonhosted.org/packages/e7/26/1341c262e74f36d4e84f3d6f4df0ac68cd53331a66bfc5080daa17c84c0b/ruff-0.15.11-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7a1b5b2938d8f890b76084d4fa843604d787a912541eae85fd7e233398bbb73e", size = 10988592 }, + { url = "https://files.pythonhosted.org/packages/03/71/850b1d6ffa9564fbb6740429bad53df1094082fe515c8c1e74b6d8d05f18/ruff-0.15.11-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d4176f3d194afbdaee6e41b9ccb1a2c287dba8700047df474abfbe773825d1cb", size = 10338501 }, + { url = "https://files.pythonhosted.org/packages/f2/11/cc1284d3e298c45a817a6aadb6c3e1d70b45c9b36d8d9cce3387b495a03a/ruff-0.15.11-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b17c886fb88203ced3afe7f14e8d5ae96e9d2f4ccc0ee66aa19f2c2675a27e4", size = 10670693 }, + { url = "https://files.pythonhosted.org/packages/ce/9e/f8288b034ab72b371513c13f9a41d9ba3effac54e24bfb467b007daee2ca/ruff-0.15.11-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:49fafa220220afe7758a487b048de4c8f9f767f37dfefad46b9dd06759d003eb", size = 10416177 }, + { url = "https://files.pythonhosted.org/packages/85/71/504d79abfd3d92532ba6bbe3d1c19fada03e494332a59e37c7c2dabae427/ruff-0.15.11-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2ab8427e74a00d93b8bda1307b1e60970d40f304af38bccb218e056c220120d", size = 11221886 }, + { url = "https://files.pythonhosted.org/packages/43/5a/947e6ab7a5ad603d65b474be15a4cbc6d29832db5d762cd142e4e3a74164/ruff-0.15.11-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:195072c0c8e1fc8f940652073df082e37a5d9cb43b4ab1e4d0566ab8977a13b7", size = 12075183 }, + { url = "https://files.pythonhosted.org/packages/9f/a1/0b7bb6268775fdd3a0818aee8efd8f5b4e231d24dd4d528ced2534023182/ruff-0.15.11-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a3a0996d486af3920dec930a2e7daed4847dfc12649b537a9335585ada163e9e", size = 11516575 }, + { url = "https://files.pythonhosted.org/packages/30/c3/bb5168fc4d233cc06e95f482770d0f3c87945a0cd9f614b90ea8dc2f2833/ruff-0.15.11-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bef2cb556d509259f1fe440bb9cd33c756222cf0a7afe90d15edf0866702431", size = 11306537 }, + { url = "https://files.pythonhosted.org/packages/e4/92/4cfae6441f3967317946f3b788136eecf093729b94d6561f963ed810c82e/ruff-0.15.11-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:030d921a836d7d4a12cf6e8d984a88b66094ccb0e0f17ddd55067c331191bf19", size = 11296813 }, + { url = "https://files.pythonhosted.org/packages/43/26/972784c5dde8313acde8ac71ba8ac65475b85db4a2352a76c9934361f9bc/ruff-0.15.11-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0e783b599b4577788dbbb66b9addcef87e9a8832f4ce0c19e34bf55543a2f890", size = 10633136 }, + { url = "https://files.pythonhosted.org/packages/5b/53/3985a4f185020c2f367f2e08a103032e12564829742a1b417980ce1514a0/ruff-0.15.11-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ae90592246625ba4a34349d68ec28d4400d75182b71baa196ddb9f82db025ef5", size = 10424701 }, + { url = "https://files.pythonhosted.org/packages/d3/57/bf0dfb32241b56c83bb663a826133da4bf17f682ba8c096973065f6e6a68/ruff-0.15.11-py3-none-musllinux_1_2_i686.whl", hash = "sha256:1f111d62e3c983ed20e0ca2e800f8d77433a5b1161947df99a5c2a3fb60514f0", size = 10873887 }, + { url = "https://files.pythonhosted.org/packages/02/05/e48076b2a57dc33ee8c7a957296f97c744ca891a8ffb4ffb1aaa3b3f517d/ruff-0.15.11-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:06f483d6646f59eaffba9ae30956370d3a886625f511a3108994000480621d1c", size = 11404316 }, + { url = "https://files.pythonhosted.org/packages/88/27/0195d15fe7a897cbcba0904792c4b7c9fdd958456c3a17d2ea6093716a9a/ruff-0.15.11-py3-none-win32.whl", hash = "sha256:476a2aa56b7da0b73a3ee80b6b2f0e19cce544245479adde7baa65466664d5f3", size = 10655535 }, + { url = "https://files.pythonhosted.org/packages/3a/5e/c927b325bd4c1d3620211a4b96f47864633199feed60fa936025ab27e090/ruff-0.15.11-py3-none-win_amd64.whl", hash = "sha256:8b6756d88d7e234fb0c98c91511aae3cd519d5e3ed271cae31b20f39cb2a12a3", size = 11779692 }, + { url = "https://files.pythonhosted.org/packages/63/b6/aeadee5443e49baa2facd51131159fd6301cc4ccfc1541e4df7b021c37dd/ruff-0.15.11-py3-none-win_arm64.whl", hash = "sha256:063fed18cc1bbe0ee7393957284a6fe8b588c6a406a285af3ee3f46da2391ee4", size = 11032614 }, +] + +[[package]] +name = "setuptools" +version = "82.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223 }, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575 }, +] + +[[package]] +name = "structlog" +version = "25.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ef/52/9ba0f43b686e7f3ddfeaa78ac3af750292662284b3661e91ad5494f21dbc/structlog-25.5.0.tar.gz", hash = "sha256:098522a3bebed9153d4570c6d0288abf80a031dfdb2048d59a49e9dc2190fc98", size = 1460830 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/45/a132b9074aa18e799b891b91ad72133c98d8042c70f6240e4c5f9dabee2f/structlog-25.5.0-py3-none-any.whl", hash = "sha256:a8453e9b9e636ec59bd9e79bbd4a72f025981b3ba0f5837aebf48f02f37a7f9f", size = 72510 }, +] + +[[package]] +name = "trio" +version = "0.33.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "cffi", marker = "implementation_name != 'pypy' and os_name == 'nt'" }, + { name = "idna" }, + { name = "outcome" }, + { name = "sniffio" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/b6/c744031c6f89b18b3f5f4f7338603ab381d740a7f45938c4607b2302481f/trio-0.33.0.tar.gz", hash = "sha256:a29b92b73f09d4b48ed249acd91073281a7f1063f09caba5dc70465b5c7aa970", size = 605109 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/93/dab25dc87ac48da0fe0f6419e07d0bfd98799bed4e05e7b9e0f85a1a4b4b/trio-0.33.0-py3-none-any.whl", hash = "sha256:3bd5d87f781d9b0192d592aef28691f8951d6c2e41b7e1da4c25cde6c180ae9b", size = 510294 }, +] + +[[package]] +name = "types-pyyaml" +version = "6.0.12.20260408" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/74/73/b759b1e413c31034cc01ecdfb96b38115d0ab4db55a752a3929f0cd449fd/types_pyyaml-6.0.12.20260408.tar.gz", hash = "sha256:92a73f2b8d7f39ef392a38131f76b970f8c66e4c42b3125ae872b7c93b556307", size = 17735 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/f0/c391068b86abb708882c6d75a08cd7d25b2c7227dab527b3a3685a3c635b/types_pyyaml-6.0.12.20260408-py3-none-any.whl", hash = "sha256:fbc42037d12159d9c801ebfcc79ebd28335a7c13b08a4cfbc6916df78fee9384", size = 20339 }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614 }, +] + +[[package]] +name = "wrapt" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/64/925f213fdcbb9baeb1530449ac71a4d57fc361c053d06bf78d0c5c7cd80c/wrapt-2.1.2.tar.gz", hash = "sha256:3996a67eecc2c68fd47b4e3c564405a5777367adfd9b8abb58387b63ee83b21e", size = 81678 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/81/60c4471fce95afa5922ca09b88a25f03c93343f759aae0f31fb4412a85c7/wrapt-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:96159a0ee2b0277d44201c3b5be479a9979cf154e8c82fa5df49586a8e7679bb", size = 60666 }, + { url = "https://files.pythonhosted.org/packages/6b/be/80e80e39e7cb90b006a0eaf11c73ac3a62bbfb3068469aec15cc0bc795de/wrapt-2.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:98ba61833a77b747901e9012072f038795de7fc77849f1faa965464f3f87ff2d", size = 61601 }, + { url = "https://files.pythonhosted.org/packages/b0/be/d7c88cd9293c859fc74b232abdc65a229bb953997995d6912fc85af18323/wrapt-2.1.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:767c0dbbe76cae2a60dd2b235ac0c87c9cccf4898aef8062e57bead46b5f6894", size = 114057 }, + { url = "https://files.pythonhosted.org/packages/ea/25/36c04602831a4d685d45a93b3abea61eca7fe35dab6c842d6f5d570ef94a/wrapt-2.1.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c691a6bc752c0cc4711cc0c00896fcd0f116abc253609ef64ef930032821842", size = 116099 }, + { url = "https://files.pythonhosted.org/packages/5c/4e/98a6eb417ef551dc277bec1253d5246b25003cf36fdf3913b65cb7657a56/wrapt-2.1.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f3b7d73012ea75aee5844de58c88f44cf62d0d62711e39da5a82824a7c4626a8", size = 112457 }, + { url = "https://files.pythonhosted.org/packages/cb/a6/a6f7186a5297cad8ec53fd7578533b28f795fdf5372368c74bd7e6e9841c/wrapt-2.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:577dff354e7acd9d411eaf4bfe76b724c89c89c8fc9b7e127ee28c5f7bcb25b6", size = 115351 }, + { url = "https://files.pythonhosted.org/packages/97/6f/06e66189e721dbebd5cf20e138acc4d1150288ce118462f2fcbff92d38db/wrapt-2.1.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3d7b6fd105f8b24e5bd23ccf41cb1d1099796524bcc6f7fbb8fe576c44befbc9", size = 111748 }, + { url = "https://files.pythonhosted.org/packages/ef/43/4808b86f499a51370fbdbdfa6cb91e9b9169e762716456471b619fca7a70/wrapt-2.1.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:866abdbf4612e0b34764922ef8b1c5668867610a718d3053d59e24a5e5fcfc15", size = 113783 }, + { url = "https://files.pythonhosted.org/packages/91/2c/a3f28b8fa7ac2cefa01cfcaca3471f9b0460608d012b693998cd61ef43df/wrapt-2.1.2-cp311-cp311-win32.whl", hash = "sha256:5a0a0a3a882393095573344075189eb2d566e0fd205a2b6414e9997b1b800a8b", size = 57977 }, + { url = "https://files.pythonhosted.org/packages/3f/c3/2b1c7bd07a27b1db885a2fab469b707bdd35bddf30a113b4917a7e2139d2/wrapt-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:64a07a71d2730ba56f11d1a4b91f7817dc79bc134c11516b75d1921a7c6fcda1", size = 60336 }, + { url = "https://files.pythonhosted.org/packages/ec/5c/76ece7b401b088daa6503d6264dd80f9a727df3e6042802de9a223084ea2/wrapt-2.1.2-cp311-cp311-win_arm64.whl", hash = "sha256:b89f095fe98bc12107f82a9f7d570dc83a0870291aeb6b1d7a7d35575f55d98a", size = 58756 }, + { url = "https://files.pythonhosted.org/packages/1a/c7/8528ac2dfa2c1e6708f647df7ae144ead13f0a31146f43c7264b4942bf12/wrapt-2.1.2-py3-none-any.whl", hash = "sha256:b8fd6fa2b2c4e7621808f8c62e8317f4aae56e59721ad933bac5239d913cf0e8", size = 43993 }, +] + +[[package]] +name = "zipp" +version = "3.23.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/21/093488dfc7cc8964ded15ab726fad40f25fd3d788fd741cc1c5a17d78ee8/zipp-3.23.1.tar.gz", hash = "sha256:32120e378d32cd9714ad503c1d024619063ec28aad2248dc6672ad13edfa5110", size = 25965 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/8a/0861bec20485572fbddf3dfba2910e38fe249796cb73ecdeb74e07eeb8d3/zipp-3.23.1-py3-none-any.whl", hash = "sha256:0b3596c50a5c700c9cb40ba8d86d9f2cc4807e9bedb06bcdf7fac85633e444dc", size = 10378 }, +] diff --git a/worker-image/Dockerfile b/worker-image/Dockerfile new file mode 100644 index 0000000..5009d41 --- /dev/null +++ b/worker-image/Dockerfile @@ -0,0 +1,41 @@ +# syntax=docker/dockerfile:1.7 +# +# PymoniK v2 worker image. Baked-cloudpickle mode: the image ships with the +# pymonik package installed; user tasks arrive as cloudpickle blobs inside a +# msgspec envelope and are dispatched by pymonik.worker.run(). +# +# Build (from the repo root): +# +# docker build -t pymonik-worker: -f worker-image/Dockerfile . +# +# CI publishes this image automatically on release; see +# .github/workflows/publish-images.yml. + +ARG PYTHON_VERSION=3.11 +FROM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-bookworm-slim + +# ArmoniK convention: workers run as `armonikuser` (uid/gid 5000) with +# /cache owned by that user. The agent <-> worker socket lives under /cache. +RUN groupadd --gid 5000 armonikuser \ + && useradd --home-dir /home/armonikuser --create-home --uid 5000 --gid 5000 \ + --shell /bin/sh --skel /dev/null armonikuser \ + && mkdir /cache \ + && chown armonikuser: /cache + +WORKDIR /app +RUN chown -R armonikuser: /app +USER armonikuser + +# Install the pymonik package into /app/.venv. +COPY --chown=armonikuser:armonikuser pyproject.toml README.md ./ +COPY --chown=armonikuser:armonikuser src/ ./src/ + +RUN uv venv /app/.venv \ + && uv pip install --no-cache . + +ENV PATH="/app/.venv/bin:${PATH}" +ENV PYTHONUNBUFFERED=1 + +# The console script pymonik-worker wraps pymonik.worker.run(), which uses +# armonik.worker.armonik_worker() to bind to the agent socket under /cache. +ENTRYPOINT ["pymonik-worker"]