From f1d8248adf25fa40a25c1cc8ed630d6a95f0c06e Mon Sep 17 00:00:00 2001 From: Vibe Nuage Agent Date: Tue, 19 May 2026 14:36:29 +0000 Subject: [PATCH 01/16] Add workflow payload compression encoder --- pyproject.toml | 4 + src/mistralai/extra/exceptions.py | 4 + .../extra/tests/fixtures/__init__.py | 0 .../extra/tests/fixtures/workflow_encoding.py | 26 + .../extra/tests/test_workflow_encoding.py | 455 +++++++++++++++++- src/mistralai/extra/workflows/__init__.py | 14 +- .../extra/workflows/encoding/__init__.py | 12 +- .../extra/workflows/encoding/config.py | 21 +- .../extra/workflows/encoding/models.py | 36 +- .../workflows/encoding/payload_compressor.py | 108 +++++ .../workflows/encoding/payload_encoder.py | 143 ++++-- uv.lock | 98 +++- 12 files changed, 871 insertions(+), 50 deletions(-) create mode 100644 src/mistralai/extra/tests/fixtures/__init__.py create mode 100644 src/mistralai/extra/tests/fixtures/workflow_encoding.py create mode 100644 src/mistralai/extra/workflows/encoding/payload_compressor.py diff --git a/pyproject.toml b/pyproject.toml index f79b279f..3c246a62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,9 @@ workflow_payload_offloading = [ workflow_payload_encryption = [ "cryptography>=41.0.0,<47.0.0", ] +workflow_payload_compression = [ + "zstandard>=0.25.0,<0.26", +] [project.urls] @@ -69,6 +72,7 @@ dev = [ "griffe>=1.7.3,<2", "authlib>=1.5.2,<2", "websockets >=13.0", + "zstandard>=0.25.0,<0.26", ] lint = [ "ruff>=0.11.10,<0.12", diff --git a/src/mistralai/extra/exceptions.py b/src/mistralai/extra/exceptions.py index a6b9ff11..a51d5dc3 100644 --- a/src/mistralai/extra/exceptions.py +++ b/src/mistralai/extra/exceptions.py @@ -28,6 +28,10 @@ class WorkflowPayloadEncryptionException(MistralClientException): """Workflow payload encryption exception""" +class WorkflowPayloadCompressionException(MistralClientException): + """Workflow payload compression exception""" + + class RunException(MistralClientException): """Conversation run errors.""" diff --git a/src/mistralai/extra/tests/fixtures/__init__.py b/src/mistralai/extra/tests/fixtures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/mistralai/extra/tests/fixtures/workflow_encoding.py b/src/mistralai/extra/tests/fixtures/workflow_encoding.py new file mode 100644 index 00000000..2542ad12 --- /dev/null +++ b/src/mistralai/extra/tests/fixtures/workflow_encoding.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from typing import Any + + +class InMemoryBlobStorage: + def __init__(self) -> None: + self.blobs: dict[str, bytes] = {} + + async def __aenter__(self) -> "InMemoryBlobStorage": + return self + + async def __aexit__(self, *_args: Any) -> None: + pass + + async def upload_blob(self, key: str, content: bytes) -> str: + self.blobs[key] = content + return key + + async def get_blob(self, key: str) -> bytes: + return self.blobs[key] + + async def get_blob_properties(self, key: str) -> dict[str, Any] | None: + if key not in self.blobs: + return None + return {"size": len(self.blobs[key]), "last_modified": "test"} diff --git a/src/mistralai/extra/tests/test_workflow_encoding.py b/src/mistralai/extra/tests/test_workflow_encoding.py index 3703012b..3107043c 100644 --- a/src/mistralai/extra/tests/test_workflow_encoding.py +++ b/src/mistralai/extra/tests/test_workflow_encoding.py @@ -1,8 +1,11 @@ """Tests for workflow encoding configuration lifecycle.""" +import base64 import gc +import json import pytest +import zstandard from pydantic import SecretStr from mistralai.client import Mistral @@ -12,10 +15,24 @@ configure_workflow_encoding, ) from mistralai.extra.workflows import ( - WorkflowEncodingConfig, + BlobStorageConfig, + EncryptedStrField, + PayloadCompressionConfig, PayloadEncryptionConfig, PayloadEncryptionMode, + PayloadOffloadingConfig, + StorageProvider, + WorkflowEncodingConfig, + ZstdCompressionConfig, ) +from mistralai.extra.exceptions import WorkflowPayloadCompressionException +from mistralai.extra.workflows.encoding.models import ( + EncodedPayloadOptions, + NetworkEncodedInput, + WorkflowContext, +) +from mistralai.extra.workflows.encoding.payload_encoder import PayloadEncoder +from mistralai.extra.tests.fixtures.workflow_encoding import InMemoryBlobStorage @pytest.fixture @@ -29,7 +46,9 @@ def encryption_config() -> WorkflowEncodingConfig: ) -def test_payload_encoder_cleanup_on_client_gc(encryption_config: WorkflowEncodingConfig): +def test_payload_encoder_cleanup_on_client_gc( + encryption_config: WorkflowEncodingConfig, +): """Test that PayloadEncoder is cleaned up when client is garbage collected.""" initial_config_count = len(_workflow_configs) @@ -56,7 +75,9 @@ def test_payload_encoder_cleanup_on_client_gc(encryption_config: WorkflowEncodin assert len(_workflow_configs) == initial_config_count -def test_multiple_clients_independent_configs(encryption_config: WorkflowEncodingConfig): +def test_multiple_clients_independent_configs( + encryption_config: WorkflowEncodingConfig, +): """Test that multiple clients have independent configs.""" initial_config_count = len(_workflow_configs) @@ -132,3 +153,431 @@ def test_reconfigure_same_client(encryption_config: WorkflowEncodingConfig): del client gc.collect() assert config_id not in _workflow_configs + + +@pytest.mark.asyncio +async def test_payload_encoder_compresses_network_inputs(): + config = WorkflowEncodingConfig( + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=3) + ) + ) + encoder = PayloadEncoder(encoding_config=config) + payload = {"data": "x" * 20_000} + + encoded = await encoder.encode_network_input( + payload, WorkflowContext(namespace="test", execution_id="exec") + ) + + assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] + assert encoded.encoding_metadata == {} + + compressed_payload = json.loads(encoded.get_payload()) + assert compressed_payload["algorithm_config"] == {"algorithm": "zstd", "level": 3} + + decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) + assert decoded == payload + + +@pytest.mark.asyncio +async def test_payload_encoder_skips_compression_below_min_size(): + config = WorkflowEncodingConfig( + payload_compression=PayloadCompressionConfig(min_size_bytes=1_000_000) + ) + encoder = PayloadEncoder(encoding_config=config) + payload = {"data": "x" * 20_000} + + encoded = await encoder.encode_network_input( + payload, WorkflowContext(namespace="test", execution_id="exec") + ) + + assert encoded.encoding_options == [] + assert encoded.encoding_metadata == {} + decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) + assert decoded == payload + + +@pytest.mark.asyncio +async def test_payload_encoder_skips_compression_when_not_smaller(): + config = WorkflowEncodingConfig( + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=3) + ) + ) + encoder = PayloadEncoder(encoding_config=config) + payload = {"d": "x"} + + encoded = await encoder.encode_network_input( + payload, WorkflowContext(namespace="test", execution_id="exec") + ) + + assert encoded.encoding_options == [] + assert encoded.encoding_metadata == {} + decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) + assert decoded == payload + + +@pytest.mark.asyncio +async def test_payload_encoder_skips_compression_without_config(): + encoder = PayloadEncoder(encoding_config=WorkflowEncodingConfig()) + payload = {"data": "x" * 20_000} + + encoded = await encoder.encode_network_input( + payload, WorkflowContext(namespace="test", execution_id="exec") + ) + + assert encoded.encoding_options == [] + assert encoded.encoding_metadata == {} + decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) + assert decoded == payload + + +@pytest.mark.asyncio +async def test_payload_encoder_compression_can_prevent_offloading(monkeypatch): + storage = InMemoryBlobStorage() + monkeypatch.setattr( + "mistralai.extra.workflows.encoding.payload_encoder.get_blob_storage", + lambda _: storage, + ) + config = WorkflowEncodingConfig( + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=3) + ), + payload_offloading=PayloadOffloadingConfig( + min_size_bytes=1_000, + storage_config=BlobStorageConfig( + storage_provider=StorageProvider.S3, + bucket_name="test-bucket", + ), + ), + ) + encoder = PayloadEncoder(encoding_config=config) + payload = {"data": "x" * 20_000} + + encoded = await encoder.encode_network_input( + payload, WorkflowContext(namespace="test", execution_id="exec") + ) + + assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] + assert encoded.encoding_metadata == {} + assert storage.blobs == {} + decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) + assert decoded == payload + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "decoder_config", + [ + WorkflowEncodingConfig(), + WorkflowEncodingConfig( + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=1) + ) + ), + ], +) +async def test_payload_encoder_decodes_compressed_payload_with_decoder_config( + decoder_config: WorkflowEncodingConfig, +): + encoder = PayloadEncoder( + encoding_config=WorkflowEncodingConfig( + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=22) + ) + ) + ) + payload = {"data": "x" * 20_000} + + encoded = await encoder.encode_network_input( + payload, WorkflowContext(namespace="test", execution_id="exec") + ) + compressed_payload = json.loads(encoded.get_payload()) + decoded = await PayloadEncoder(decoder_config).decode_network_result( + encoded.model_dump(mode="json") + ) + + assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] + assert compressed_payload["algorithm_config"] == {"algorithm": "zstd", "level": 22} + assert decoded == payload + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("encryption_mode", "expected_options"), + [ + ( + PayloadEncryptionMode.PARTIAL, + [EncodedPayloadOptions.PARTIALLY_ENCRYPTED, EncodedPayloadOptions.COMPRESSED], + ), + ( + PayloadEncryptionMode.FULL, + [EncodedPayloadOptions.COMPRESSED, EncodedPayloadOptions.ENCRYPTED], + ), + ], +) +async def test_payload_encoder_decodes_encrypted_compressed_payload_with_different_level( + encryption_mode: PayloadEncryptionMode, + expected_options: list[EncodedPayloadOptions], +): + encryption_config = PayloadEncryptionConfig( + mode=encryption_mode, + main_key=SecretStr("0" * 64), + ) + encoder = PayloadEncoder( + encoding_config=WorkflowEncodingConfig( + payload_encryption=encryption_config, + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=10) + ), + ) + ) + decoder = PayloadEncoder( + encoding_config=WorkflowEncodingConfig( + payload_encryption=encryption_config, + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=1) + ), + ) + ) + payload = { + "data": "x" * 20_000, + "secret": EncryptedStrField(data="secret value").model_dump(), + } + + encoded = await encoder.encode_network_input( + payload, WorkflowContext(namespace="test", execution_id="exec") + ) + decoded = await decoder.decode_network_result(encoded.model_dump(mode="json")) + + assert encoded.encoding_options == expected_options + assert decoded == payload + + +@pytest.mark.asyncio +async def test_payload_encoder_decodes_with_tampered_compression_level(): + encoder = PayloadEncoder( + encoding_config=WorkflowEncodingConfig( + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=22) + ) + ) + ) + payload = {"data": "x" * 20_000} + + encoded = await encoder.encode_network_input( + payload, WorkflowContext(namespace="test", execution_id="exec") + ) + compressed_payload = json.loads(encoded.get_payload()) + compressed_payload["algorithm_config"]["level"] = 1 + tampered = NetworkEncodedInput.from_data( + json.dumps(compressed_payload).encode(), encoded.encoding_options + ) + + decoded = await PayloadEncoder(WorkflowEncodingConfig()).decode_network_result( + tampered.model_dump(mode="json") + ) + + assert decoded == payload + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "compressed_payload", + [ + b"not-json", + b'{"algorithm_config":{"algorithm":"lz4","level":1},"b64data":"AAAA"}', + b'{"algorithm_config":{"level":3},"b64data":"AAAA"}', + ], +) +async def test_payload_encoder_invalid_compressed_payload_is_error( + compressed_payload: bytes, +): + encoded = NetworkEncodedInput.from_data( + compressed_payload, [EncodedPayloadOptions.COMPRESSED] + ) + + with pytest.raises(WorkflowPayloadCompressionException, match="Invalid compressed payload"): + await PayloadEncoder(WorkflowEncodingConfig()).decode_network_result( + encoded.model_dump(mode="json") + ) + + +@pytest.mark.asyncio +async def test_payload_encoder_corrupted_compressed_data_is_error(): + encoded = NetworkEncodedInput.from_data( + json.dumps( + { + "algorithm_config": {"algorithm": "zstd", "level": 3}, + "b64data": base64.b64encode(b"corrupted-data").decode(), + } + ).encode(), + [EncodedPayloadOptions.COMPRESSED], + ) + + with pytest.raises(zstandard.ZstdError): + await PayloadEncoder(WorkflowEncodingConfig()).decode_network_result( + encoded.model_dump(mode="json") + ) + + +@pytest.mark.asyncio +async def test_payload_encoder_partially_encrypts_before_offloading(monkeypatch): + storage = InMemoryBlobStorage() + monkeypatch.setattr( + "mistralai.extra.workflows.encoding.payload_encoder.get_blob_storage", + lambda _: storage, + ) + config = WorkflowEncodingConfig( + payload_encryption=PayloadEncryptionConfig( + mode=PayloadEncryptionMode.PARTIAL, + main_key=SecretStr("0" * 64), + ), + payload_offloading=PayloadOffloadingConfig( + min_size_bytes=1, + storage_config=BlobStorageConfig( + storage_provider=StorageProvider.S3, + bucket_name="test-bucket", + ), + ), + ) + encoder = PayloadEncoder(encoding_config=config) + payload = { + "data": "plain value", + "secret": EncryptedStrField(data="secret value").model_dump(), + } + + encoded = await encoder.encode_network_input( + payload, WorkflowContext(namespace="test", execution_id="exec") + ) + offloaded_payload = json.loads(encoded.get_payload()) + offloaded_bytes = storage.blobs[offloaded_payload["key"]] + + assert encoded.encoding_options == [ + EncodedPayloadOptions.PARTIALLY_ENCRYPTED, + EncodedPayloadOptions.OFFLOADED, + ] + assert b"plain value" in offloaded_bytes + assert b"secret value" not in offloaded_bytes + + decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) + assert decoded == payload + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("encryption_mode", "expected_options"), + [ + ( + PayloadEncryptionMode.PARTIAL, + [ + EncodedPayloadOptions.PARTIALLY_ENCRYPTED, + EncodedPayloadOptions.COMPRESSED, + EncodedPayloadOptions.OFFLOADED, + ], + ), + ( + PayloadEncryptionMode.FULL, + [ + EncodedPayloadOptions.COMPRESSED, + EncodedPayloadOptions.OFFLOADED, + EncodedPayloadOptions.ENCRYPTED, + ], + ), + ], +) +async def test_payload_encoder_compression_offloading_encryption_roundtrip( + monkeypatch, + encryption_mode: PayloadEncryptionMode, + expected_options: list[EncodedPayloadOptions], +): + storage = InMemoryBlobStorage() + monkeypatch.setattr( + "mistralai.extra.workflows.encoding.payload_encoder.get_blob_storage", + lambda _: storage, + ) + config = WorkflowEncodingConfig( + payload_encryption=PayloadEncryptionConfig( + mode=encryption_mode, + main_key=SecretStr("0" * 64), + ), + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=3) + ), + payload_offloading=PayloadOffloadingConfig( + min_size_bytes=1, + storage_config=BlobStorageConfig( + storage_provider=StorageProvider.S3, + bucket_name="test-bucket", + ), + ), + ) + encoder = PayloadEncoder(encoding_config=config) + payload = { + "data": "x" * 20_000, + "secret": EncryptedStrField(data="secret value").model_dump(), + } + + encoded = await encoder.encode_network_input( + payload, WorkflowContext(namespace="test", execution_id="exec") + ) + + assert encoded.encoding_options == expected_options + assert encoded.encoding_metadata == {} + assert len(storage.blobs) == 1 + decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) + assert decoded == payload + + +@pytest.mark.asyncio +async def test_payload_encoder_does_not_partially_encrypt_when_no_marked_fields(): + config = WorkflowEncodingConfig( + payload_encryption=PayloadEncryptionConfig( + mode=PayloadEncryptionMode.PARTIAL, + main_key=SecretStr("0" * 64), + ), + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=3) + ), + ) + encoder = PayloadEncoder(encoding_config=config) + payload = {"data": "x" * 20_000} + + encoded = await encoder.encode_network_input( + payload, WorkflowContext(namespace="test", execution_id="exec") + ) + + assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] + assert encoded.encoding_metadata == {} + decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) + assert decoded == payload + + +@pytest.mark.asyncio +async def test_payload_encoder_event_payload_orders_compression_before_full_encryption(): + config = WorkflowEncodingConfig( + payload_encryption=PayloadEncryptionConfig( + mode=PayloadEncryptionMode.FULL, + main_key=SecretStr("0" * 64), + ), + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=3) + ), + ) + encoder = PayloadEncoder(encoding_config=config) + payload = json.dumps({"data": "x" * 20_000}).encode() + + ( + encoded, + encoding_options, + encoding_metadata, + ) = await encoder.encode_event_payload_content(payload) + decoded = await encoder.decode_payload_content( + encoded, encoding_options, encoding_metadata + ) + + assert encoding_options == [ + EncodedPayloadOptions.COMPRESSED, + EncodedPayloadOptions.ENCRYPTED, + ] + assert decoded == payload diff --git a/src/mistralai/extra/workflows/__init__.py b/src/mistralai/extra/workflows/__init__.py index f26edf4e..cc806268 100644 --- a/src/mistralai/extra/workflows/__init__.py +++ b/src/mistralai/extra/workflows/__init__.py @@ -9,13 +9,16 @@ WorkflowExtensions, ) from .encoding import ( - WorkflowEncodingConfig, - PayloadOffloadingConfig, + AlgorithmConfig, + BlobStorageConfig, + EncryptedStrField, + PayloadCompressionConfig, PayloadEncryptionConfig, PayloadEncryptionMode, - BlobStorageConfig, + PayloadOffloadingConfig, StorageProvider, - EncryptedStrField, + WorkflowEncodingConfig, + ZstdCompressionConfig, configure_workflow_encoding, generate_two_part_id, ) @@ -27,10 +30,13 @@ "ConnectorSlot", "WorkflowExtensions", "execute_with_connector_auth_async", + "AlgorithmConfig", "WorkflowEncodingConfig", "PayloadOffloadingConfig", "PayloadEncryptionConfig", "PayloadEncryptionMode", + "PayloadCompressionConfig", + "ZstdCompressionConfig", "BlobStorageConfig", "StorageProvider", "EncryptedStrField", diff --git a/src/mistralai/extra/workflows/encoding/__init__.py b/src/mistralai/extra/workflows/encoding/__init__.py index c1881636..263b629b 100644 --- a/src/mistralai/extra/workflows/encoding/__init__.py +++ b/src/mistralai/extra/workflows/encoding/__init__.py @@ -1,20 +1,26 @@ from .config import ( - WorkflowEncodingConfig, - PayloadOffloadingConfig, + AlgorithmConfig, + BlobStorageConfig, + PayloadCompressionConfig, PayloadEncryptionConfig, PayloadEncryptionMode, - BlobStorageConfig, + PayloadOffloadingConfig, StorageProvider, + WorkflowEncodingConfig, + ZstdCompressionConfig, ) from .models import EncryptedStrField from .payload_encoder import PayloadEncoder from .helpers import configure_workflow_encoding, generate_two_part_id __all__ = [ + "AlgorithmConfig", "WorkflowEncodingConfig", "PayloadOffloadingConfig", "PayloadEncryptionConfig", "PayloadEncryptionMode", + "PayloadCompressionConfig", + "ZstdCompressionConfig", "BlobStorageConfig", "StorageProvider", "EncryptedStrField", diff --git a/src/mistralai/extra/workflows/encoding/config.py b/src/mistralai/extra/workflows/encoding/config.py index ff849fd7..ccb73d28 100644 --- a/src/mistralai/extra/workflows/encoding/config.py +++ b/src/mistralai/extra/workflows/encoding/config.py @@ -1,6 +1,7 @@ from enum import Enum -from pydantic import SecretStr, BaseModel -from typing import Optional +from typing import Annotated, Literal, Optional, Union + +from pydantic import BaseModel, Field, SecretStr class StorageProvider(str, Enum): @@ -47,6 +48,22 @@ class PayloadEncryptionConfig(BaseModel): secondary_key: Optional[SecretStr] = None +class ZstdCompressionConfig(BaseModel): + algorithm: Literal["zstd"] = "zstd" + level: int = Field(default=3, ge=1, le=22) + + +AlgorithmConfig = Annotated[ + Union[ZstdCompressionConfig], Field(discriminator="algorithm") +] + + +class PayloadCompressionConfig(BaseModel): + min_size_bytes: int = 1024 * 1024 # 1MB + algorithm_config: AlgorithmConfig = Field(default_factory=ZstdCompressionConfig) + + class WorkflowEncodingConfig(BaseModel): payload_offloading: PayloadOffloadingConfig | None = None payload_encryption: PayloadEncryptionConfig | None = None + payload_compression: PayloadCompressionConfig | None = None diff --git a/src/mistralai/extra/workflows/encoding/models.py b/src/mistralai/extra/workflows/encoding/models.py index 496be8b1..0cf1a648 100644 --- a/src/mistralai/extra/workflows/encoding/models.py +++ b/src/mistralai/extra/workflows/encoding/models.py @@ -4,11 +4,14 @@ import base64 from pydantic import BaseModel, Field +from mistralai.extra.workflows.encoding.config import AlgorithmConfig + class EncodedPayloadOptions(str, Enum): OFFLOADED = "offloaded" ENCRYPTED = "encrypted" PARTIALLY_ENCRYPTED = "encrypted-partial" + COMPRESSED = "compressed" class EncryptableFieldTypes(str, Enum): @@ -22,6 +25,23 @@ class EncryptedStrField(BaseModel): data: str +class CompressedPayloadData(BaseModel): + algorithm_config: AlgorithmConfig + b64data: str + + @staticmethod + def from_data( + data: bytes, algorithm_config: AlgorithmConfig + ) -> "CompressedPayloadData": + return CompressedPayloadData( + algorithm_config=algorithm_config, + b64data=base64.b64encode(data).decode("utf-8"), + ) + + def get_data(self) -> bytes: + return base64.b64decode(self.b64data) + + class WorkflowContext(BaseModel): namespace: str execution_id: str @@ -35,6 +55,10 @@ class EncodedPayload(BaseModel): encoding_options: list[EncodedPayloadOptions] = Field( description="The encoding of the payload", default=[] ) + encoding_metadata: dict[str, str] = Field( + description="Additional metadata required to decode the payload", + default_factory=dict, + ) payload: bytes = Field(description="The encoded payload") @@ -43,6 +67,10 @@ class NetworkEncodedBase(BaseModel): encoding_options: list[EncodedPayloadOptions] = Field( description="The encoding of the payload", default=[] ) + encoding_metadata: dict[str, str] = Field( + description="Additional metadata required to decode the payload", + default_factory=dict, + ) def get_payload(self) -> bytes: return base64.b64decode(self.b64payload) @@ -57,6 +85,7 @@ def to_encoded_payload( return EncodedPayload( payload=base64.b64decode(self.b64payload), encoding_options=self.encoding_options, + encoding_metadata=self.encoding_metadata, context=WorkflowContext( namespace=namespace, execution_id=execution_id, @@ -69,15 +98,19 @@ def from_encoded_payload(encoded_payload: EncodedPayload) -> "NetworkEncodedInpu return NetworkEncodedInput( b64payload=base64.b64encode(encoded_payload.payload).decode("utf-8"), encoding_options=encoded_payload.encoding_options, + encoding_metadata=encoded_payload.encoding_metadata, ) @staticmethod def from_data( - data: bytes, encoding_options: list[EncodedPayloadOptions] + data: bytes, + encoding_options: list[EncodedPayloadOptions], + encoding_metadata: dict[str, str] | None = None, ) -> "NetworkEncodedInput": return NetworkEncodedInput( b64payload=base64.b64encode(data).decode("utf-8"), encoding_options=encoding_options, + encoding_metadata=encoding_metadata or {}, ) @@ -87,4 +120,5 @@ def from_encoded_payload(encoded_payload: EncodedPayload) -> "NetworkEncodedResu return NetworkEncodedResult( b64payload=base64.b64encode(encoded_payload.payload).decode("utf-8"), encoding_options=encoded_payload.encoding_options, + encoding_metadata=encoded_payload.encoding_metadata, ) diff --git a/src/mistralai/extra/workflows/encoding/payload_compressor.py b/src/mistralai/extra/workflows/encoding/payload_compressor.py new file mode 100644 index 00000000..ff8b02d4 --- /dev/null +++ b/src/mistralai/extra/workflows/encoding/payload_compressor.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from functools import lru_cache +from importlib import import_module +from types import ModuleType + +from pydantic import TypeAdapter, ValidationError + +from mistralai.extra.exceptions import WorkflowPayloadCompressionException +from mistralai.extra.workflows.encoding.config import ( + AlgorithmConfig, + PayloadCompressionConfig, + ZstdCompressionConfig, +) +from mistralai.extra.workflows.encoding.models import CompressedPayloadData + +_ALGORITHM_CONFIG_ADAPTER: TypeAdapter[AlgorithmConfig] = TypeAdapter(AlgorithmConfig) + + +class Compressor(ABC): + @property + @abstractmethod + def tag(self) -> str: + """Wire tag stored in encoding options to identify the algorithm.""" + + @property + @abstractmethod + def algorithm_config(self) -> AlgorithmConfig: + """Algorithm config stored with compressed bytes for config-independent decoding.""" + + @abstractmethod + def compress(self, data: bytes) -> bytes: ... + + @abstractmethod + def decompress(self, data: bytes) -> bytes: ... + + +def _require_zstandard() -> ModuleType: + try: + return import_module("zstandard") + except ImportError: + raise WorkflowPayloadCompressionException( + "Payload compression requires installing mistralai[workflow-payload-compression]" + ) from None + + +class ZstdCompressor(Compressor): + @property + def tag(self) -> str: + return "zstd" + + @property + def algorithm_config(self) -> AlgorithmConfig: + return self._config + + def __init__(self, cfg: ZstdCompressionConfig) -> None: + zstd = _require_zstandard() + self._config = cfg + self._compressor = zstd.ZstdCompressor(level=cfg.level) + self._decompressor = zstd.ZstdDecompressor() + + def compress(self, data: bytes) -> bytes: + result: bytes = self._compressor.compress(data) + return result + + def decompress(self, data: bytes) -> bytes: + result: bytes = self._decompressor.decompress(data) + return result + + +@lru_cache(maxsize=8) +def _build_compressor_for_config(config_json: str) -> Compressor: + try: + algo_config = _ALGORITHM_CONFIG_ADAPTER.validate_json(config_json) + except ValidationError as exc: + raise WorkflowPayloadCompressionException( + f"Invalid compression config in payload: {exc}" + ) from exc + + if isinstance(algo_config, ZstdCompressionConfig): + return ZstdCompressor(algo_config) + raise WorkflowPayloadCompressionException( + f"Unsupported compression algorithm: {algo_config.algorithm!r}" + ) + + +def build_compressor( + compression_config: PayloadCompressionConfig | None, +) -> Compressor | None: + if compression_config is None: + return None + return _build_compressor_for_config( + compression_config.algorithm_config.model_dump_json() + ) + + +def decompress_payload(data: bytes) -> bytes: + try: + compressed_payload = CompressedPayloadData.model_validate_json(data) + except ValidationError as exc: + raise WorkflowPayloadCompressionException( + f"Invalid compressed payload: {exc}" + ) from exc + compressor = _build_compressor_for_config( + compressed_payload.algorithm_config.model_dump_json() + ) + return compressor.decompress(compressed_payload.get_data()) diff --git a/src/mistralai/extra/workflows/encoding/payload_encoder.py b/src/mistralai/extra/workflows/encoding/payload_encoder.py index 611f33fa..d9d3e711 100644 --- a/src/mistralai/extra/workflows/encoding/payload_encoder.py +++ b/src/mistralai/extra/workflows/encoding/payload_encoder.py @@ -30,8 +30,13 @@ PayloadOffloadingConfig, WorkflowEncodingConfig, ) +from mistralai.extra.workflows.encoding.payload_compressor import ( + build_compressor, + decompress_payload, +) from .storage.blob_storage import get_blob_storage, BlobNotFoundError from mistralai.extra.workflows.encoding.models import ( + CompressedPayloadData, EncodedPayloadOptions, EncryptableFieldTypes, NetworkEncodedInput, @@ -52,8 +57,9 @@ class OffloadedPayloadData(BaseModel): class PayloadEncoder: """This class is in charge of payload encoding/decoding operations such as: + - Field-level or full-payload encryption + - Compression - Blob storage offloading - - Encryption """ BLOB_STORAGE_KEY_PREFIX = "temporal-payload" @@ -78,6 +84,8 @@ def __init__( "Blob storage config is not set for workflow payload encoding" ) + self.compression_config = encoding_config.payload_compression + self.compressor = build_compressor(self.compression_config) self.encryption_config = encoding_config.payload_encryption if self.encryption_config is not None: if not _HAS_CRYPTO: @@ -151,7 +159,10 @@ def _decrypt(self, data: bytes) -> bytes: async def _handle_offloading( self, data: bytes, context: Optional[WorkflowContext] ) -> tuple[bytes, bool]: - if self.offloading_config is None or self.offloading_config.storage_config is None: + if ( + self.offloading_config is None + or self.offloading_config.storage_config is None + ): raise WorkflowPayloadOffloadingException( "You must configure payload offloading storage" ) @@ -228,47 +239,81 @@ async def _partially_decrypt_fields(self, data: bytes) -> tuple[bytes, bool]: return json.dumps(obj).encode(), len(encrypted_fields) > 0 + def _compress(self, data: bytes) -> tuple[bytes, bool]: + if ( + self.compression_config is None + or len(data) < self.compression_config.min_size_bytes + ): + return data, False + assert self.compressor is not None, ( + "This should never be reached: compression is enabled in config " + "but PayloadEncoder.__init__ did not build a compressor" + ) + compressed = self.compressor.compress(data) + if len(compressed) >= len(data): + return data, False + compressed_payload = CompressedPayloadData.from_data( + compressed, self.compressor.algorithm_config + ) + return compressed_payload.model_dump_json().encode(), True + + def _decompress(self, data: bytes) -> bytes: + return decompress_payload(data) + async def encode_payload_content( self, data: Union[bytes, str], context: Optional[WorkflowContext] - ) -> tuple[bytes, list[EncodedPayloadOptions]]: - """Handle payload encoding: - - Payload offloading (if context provided) - - Encryption + ) -> tuple[bytes, list[EncodedPayloadOptions], dict[str, str]]: + """Handle payload encoding. + + Encoding options are appended in the exact order in which transforms are + applied; decode_payload_content reverses that list to restore the data. """ if isinstance(data, str): data = data.encode() - encoding_options = [] + encoding_options: list[EncodedPayloadOptions] = [] + encoding_metadata: dict[str, str] = {} + + # Partial encryption needs the original JSON fields. It must run before + # compression or offloading, which make field-level markers unavailable. + if ( + self.encryption_config is not None + and self.encryption_config.mode == PayloadEncryptionMode.PARTIAL + ): + data, partially_encrypted = await self._partially_encrypt_fields(data) + if partially_encrypted: + encoding_options.append(EncodedPayloadOptions.PARTIALLY_ENCRYPTED) + + # Compress before offloading so the offloading threshold applies to the + # bytes that would otherwise cross the network, not to the raw JSON size. + data, compressed = self._compress(data) + if compressed: + encoding_options.append(EncodedPayloadOptions.COMPRESSED) if self.offloading_config is not None: data, offloaded = await self._handle_offloading(data, context) if offloaded: encoding_options.append(EncodedPayloadOptions.OFFLOADED) + # Full encryption intentionally remains the final transform. If the + # payload was offloaded, this encrypts the small blob-reference envelope + # rather than the blob bytes. if ( self.encryption_config is not None and self.encryption_config.mode == PayloadEncryptionMode.FULL ): data = self._encrypt(data) encoding_options.append(EncodedPayloadOptions.ENCRYPTED) - elif ( - self.encryption_config is not None - and self.encryption_config.mode == PayloadEncryptionMode.PARTIAL - and EncodedPayloadOptions.OFFLOADED not in encoding_options - ): - # Do not partially encrypt offloaded payload (fields not in the payload anymore) - data, partially_encrypted = await self._partially_encrypt_fields(data) - if partially_encrypted: - encoding_options.append(EncodedPayloadOptions.PARTIALLY_ENCRYPTED) - return data, encoding_options + return data, encoding_options, encoding_metadata async def encode_event_payload_content( self, data: Union[bytes, str], force_full_encryption: bool = False - ) -> tuple[bytes, list[EncodedPayloadOptions]]: - """Encrypt event payload content. + ) -> tuple[bytes, list[EncodedPayloadOptions], dict[str, str]]: + """Encode event payload content. - Unlike encode_payload_content, this only handles encryption (no offloading). + Unlike encode_payload_content, this only handles in-payload transforms + (partial/full encryption and compression), never blob offloading. Args: data: The payload data to encrypt. @@ -278,29 +323,47 @@ async def encode_event_payload_content( if isinstance(data, str): data = data.encode() - if self.encryption_config is None: - return data, [] + encoding_options: list[EncodedPayloadOptions] = [] + encoding_metadata: dict[str, str] = {} - if force_full_encryption or self.encryption_config.mode == PayloadEncryptionMode.FULL: - encrypted_data = self._encrypt(data) - return encrypted_data, [EncodedPayloadOptions.ENCRYPTED] + # Partial encryption needs JSON bytes; compression should happen before + # full encryption because encrypted bytes are intentionally high entropy. + if ( + not force_full_encryption + and self.encryption_config is not None + and self.encryption_config.mode == PayloadEncryptionMode.PARTIAL + ): + data, partially_encrypted = await self._partially_encrypt_fields(data) + if partially_encrypted: + encoding_options.append(EncodedPayloadOptions.PARTIALLY_ENCRYPTED) - # Partial encryption mode - data, partially_encrypted = await self._partially_encrypt_fields(data) - if partially_encrypted: - return data, [EncodedPayloadOptions.PARTIALLY_ENCRYPTED] + data, compressed = self._compress(data) + if compressed: + encoding_options.append(EncodedPayloadOptions.COMPRESSED) - return data, [] + if self.encryption_config is not None and ( + force_full_encryption + or self.encryption_config.mode == PayloadEncryptionMode.FULL + ): + data = self._encrypt(data) + encoding_options.append(EncodedPayloadOptions.ENCRYPTED) + + return data, encoding_options, encoding_metadata async def decode_payload_content( - self, data: bytes, encoding_options: List[EncodedPayloadOptions] + self, + data: bytes, + encoding_options: List[EncodedPayloadOptions], + encoding_metadata: dict[str, str] | None = None, ) -> bytes: - # Decode in the reverse order of encoding + # Decode in the exact reverse order of the encoding_options wire list. for option in reversed(encoding_options): if option == EncodedPayloadOptions.ENCRYPTED: data = self._decrypt(data) elif option == EncodedPayloadOptions.PARTIALLY_ENCRYPTED: data, _ = await self._partially_decrypt_fields(data) + elif option == EncodedPayloadOptions.COMPRESSED: + data = self._decompress(data) elif option == EncodedPayloadOptions.OFFLOADED: if ( self.offloading_config is None @@ -340,7 +403,10 @@ async def decode_event_payload( encoding_options = [EncodedPayloadOptions(opt) for opt in encoding_options_strs] encrypted_bytes = base64.b64decode(payload_data["value"]) - decrypted_bytes = await self.decode_payload_content(encrypted_bytes, encoding_options) + encoding_metadata = payload_data.get("encoding_metadata", {}) + decrypted_bytes = await self.decode_payload_content( + encrypted_bytes, encoding_options, encoding_metadata + ) decrypted_value = json.loads(decrypted_bytes) return { @@ -355,10 +421,14 @@ async def encode_network_input( """This method MUST be called to format every payload send to Mistral Workflows control plane to ensure a proper encoding of the payload. """ - encoded_data, encoding_options = await self.encode_payload_content( - to_json(data), context + ( + encoded_data, + encoding_options, + encoding_metadata, + ) = await self.encode_payload_content(to_json(data), context) + network_input = NetworkEncodedInput.from_data( + encoded_data, encoding_options, encoding_metadata ) - network_input = NetworkEncodedInput.from_data(encoded_data, encoding_options) return network_input async def decode_network_result(self, data: Any) -> Any: @@ -374,6 +444,7 @@ async def decode_network_result(self, data: Any) -> Any: byte_results = await self.decode_payload_content( network_encoded_payload.get_payload(), network_encoded_payload.encoding_options, + network_encoded_payload.encoding_metadata, ) try: return from_json(byte_results) diff --git a/uv.lock b/uv.lock index 2f23fc8b..9974adc7 100644 --- a/uv.lock +++ b/uv.lock @@ -1041,6 +1041,9 @@ gcp = [ realtime = [ { name = "websockets" }, ] +workflow-payload-compression = [ + { name = "zstandard" }, +] workflow-payload-encryption = [ { name = "cryptography" }, ] @@ -1075,6 +1078,7 @@ dev = [ { name = "types-python-dateutil" }, { name = "types-pyyaml" }, { name = "websockets" }, + { name = "zstandard" }, ] lint = [ { name = "mypy" }, @@ -1105,8 +1109,9 @@ requires-dist = [ { name = "requests", marker = "extra == 'gcp'", specifier = ">=2.32.3" }, { name = "typing-inspection", specifier = ">=0.4.0" }, { name = "websockets", marker = "extra == 'realtime'", specifier = ">=13.0" }, + { name = "zstandard", marker = "extra == 'workflow-payload-compression'", specifier = ">=0.25.0,<0.26" }, ] -provides-extras = ["gcp", "agents", "realtime", "workflow-payload-offloading-azure", "workflow-payload-offloading-gcs", "workflow-payload-offloading-s3", "workflow-payload-offloading", "workflow-payload-encryption"] +provides-extras = ["gcp", "agents", "realtime", "workflow-payload-offloading-azure", "workflow-payload-offloading-gcs", "workflow-payload-offloading-s3", "workflow-payload-offloading", "workflow-payload-encryption", "workflow-payload-compression"] [package.metadata.requires-dev] dev = [ @@ -1124,6 +1129,7 @@ dev = [ { name = "types-python-dateutil", specifier = ">=2.9.0.20240316,<3" }, { name = "types-pyyaml", specifier = ">=6.0.12.20250516,<7" }, { name = "websockets", specifier = ">=13.0" }, + { name = "zstandard", specifier = ">=0.25.0,<0.26" }, ] lint = [ { name = "mypy", specifier = "==1.15.0" }, @@ -2536,3 +2542,93 @@ sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50e wheels = [ { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, ] + +[[package]] +name = "zstandard" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/7a/28efd1d371f1acd037ac64ed1c5e2b41514a6cc937dd6ab6a13ab9f0702f/zstandard-0.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e59fdc271772f6686e01e1b3b74537259800f57e24280be3f29c8a0deb1904dd", size = 795256, upload-time = "2025-09-14T22:15:56.415Z" }, + { url = "https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4d441506e9b372386a5271c64125f72d5df6d2a8e8a2a45a0ae09b03cb781ef7", size = 640565, upload-time = "2025-09-14T22:15:58.177Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1b/4fdb2c12eb58f31f28c4d28e8dc36611dd7205df8452e63f52fb6261d13e/zstandard-0.25.0-cp310-cp310-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:ab85470ab54c2cb96e176f40342d9ed41e58ca5733be6a893b730e7af9c40550", size = 5345306, upload-time = "2025-09-14T22:16:00.165Z" }, + { url = "https://files.pythonhosted.org/packages/73/28/a44bdece01bca027b079f0e00be3b6bd89a4df180071da59a3dd7381665b/zstandard-0.25.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e05ab82ea7753354bb054b92e2f288afb750e6b439ff6ca78af52939ebbc476d", size = 5055561, upload-time = "2025-09-14T22:16:02.22Z" }, + { url = "https://files.pythonhosted.org/packages/e9/74/68341185a4f32b274e0fc3410d5ad0750497e1acc20bd0f5b5f64ce17785/zstandard-0.25.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:78228d8a6a1c177a96b94f7e2e8d012c55f9c760761980da16ae7546a15a8e9b", size = 5402214, upload-time = "2025-09-14T22:16:04.109Z" }, + { url = "https://files.pythonhosted.org/packages/8b/67/f92e64e748fd6aaffe01e2b75a083c0c4fd27abe1c8747fee4555fcee7dd/zstandard-0.25.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b6bd67528ee8b5c5f10255735abc21aa106931f0dbaf297c7be0c886353c3d0", size = 5449703, upload-time = "2025-09-14T22:16:06.312Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4b6d83057e713ff235a12e73916b6d356e3084fd3d14ced499d84240f3eecee0", size = 5556583, upload-time = "2025-09-14T22:16:08.457Z" }, + { url = "https://files.pythonhosted.org/packages/d7/83/41939e60d8d7ebfe2b747be022d0806953799140a702b90ffe214d557638/zstandard-0.25.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9174f4ed06f790a6869b41cba05b43eeb9a35f8993c4422ab853b705e8112bbd", size = 5045332, upload-time = "2025-09-14T22:16:10.444Z" }, + { url = "https://files.pythonhosted.org/packages/b3/87/d3ee185e3d1aa0133399893697ae91f221fda79deb61adbe998a7235c43f/zstandard-0.25.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:25f8f3cd45087d089aef5ba3848cd9efe3ad41163d3400862fb42f81a3a46701", size = 5572283, upload-time = "2025-09-14T22:16:12.128Z" }, + { url = "https://files.pythonhosted.org/packages/0a/1d/58635ae6104df96671076ac7d4ae7816838ce7debd94aecf83e30b7121b0/zstandard-0.25.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3756b3e9da9b83da1796f8809dd57cb024f838b9eeafde28f3cb472012797ac1", size = 4959754, upload-time = "2025-09-14T22:16:14.225Z" }, + { url = "https://files.pythonhosted.org/packages/75/d6/57e9cb0a9983e9a229dd8fd2e6e96593ef2aa82a3907188436f22b111ccd/zstandard-0.25.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81dad8d145d8fd981b2962b686b2241d3a1ea07733e76a2f15435dfb7fb60150", size = 5266477, upload-time = "2025-09-14T22:16:16.343Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a9/ee891e5edf33a6ebce0a028726f0bbd8567effe20fe3d5808c42323e8542/zstandard-0.25.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a5a419712cf88862a45a23def0ae063686db3d324cec7edbe40509d1a79a0aab", size = 5440914, upload-time = "2025-09-14T22:16:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/58/08/a8522c28c08031a9521f27abc6f78dbdee7312a7463dd2cfc658b813323b/zstandard-0.25.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e7360eae90809efd19b886e59a09dad07da4ca9ba096752e61a2e03c8aca188e", size = 5819847, upload-time = "2025-09-14T22:16:20.559Z" }, + { url = "https://files.pythonhosted.org/packages/6f/11/4c91411805c3f7b6f31c60e78ce347ca48f6f16d552fc659af6ec3b73202/zstandard-0.25.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:75ffc32a569fb049499e63ce68c743155477610532da1eb38e7f24bf7cd29e74", size = 5363131, upload-time = "2025-09-14T22:16:22.206Z" }, + { url = "https://files.pythonhosted.org/packages/ef/d6/8c4bd38a3b24c4c7676a7a3d8de85d6ee7a983602a734b9f9cdefb04a5d6/zstandard-0.25.0-cp310-cp310-win32.whl", hash = "sha256:106281ae350e494f4ac8a80470e66d1fe27e497052c8d9c3b95dc4cf1ade81aa", size = 436469, upload-time = "2025-09-14T22:16:25.002Z" }, + { url = "https://files.pythonhosted.org/packages/93/90/96d50ad417a8ace5f841b3228e93d1bb13e6ad356737f42e2dde30d8bd68/zstandard-0.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea9d54cc3d8064260114a0bbf3479fc4a98b21dffc89b3459edd506b69262f6e", size = 506100, upload-time = "2025-09-14T22:16:23.569Z" }, + { url = "https://files.pythonhosted.org/packages/2a/83/c3ca27c363d104980f1c9cee1101cc8ba724ac8c28a033ede6aab89585b1/zstandard-0.25.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c", size = 795254, upload-time = "2025-09-14T22:16:26.137Z" }, + { url = "https://files.pythonhosted.org/packages/ac/4d/e66465c5411a7cf4866aeadc7d108081d8ceba9bc7abe6b14aa21c671ec3/zstandard-0.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f", size = 640559, upload-time = "2025-09-14T22:16:27.973Z" }, + { url = "https://files.pythonhosted.org/packages/12/56/354fe655905f290d3b147b33fe946b0f27e791e4b50a5f004c802cb3eb7b/zstandard-0.25.0-cp311-cp311-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431", size = 5348020, upload-time = "2025-09-14T22:16:29.523Z" }, + { url = "https://files.pythonhosted.org/packages/3b/13/2b7ed68bd85e69a2069bcc72141d378f22cae5a0f3b353a2c8f50ef30c1b/zstandard-0.25.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a", size = 5058126, upload-time = "2025-09-14T22:16:31.811Z" }, + { url = "https://files.pythonhosted.org/packages/c9/dd/fdaf0674f4b10d92cb120ccff58bbb6626bf8368f00ebfd2a41ba4a0dc99/zstandard-0.25.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc", size = 5405390, upload-time = "2025-09-14T22:16:33.486Z" }, + { url = "https://files.pythonhosted.org/packages/0f/67/354d1555575bc2490435f90d67ca4dd65238ff2f119f30f72d5cde09c2ad/zstandard-0.25.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6", size = 5452914, upload-time = "2025-09-14T22:16:35.277Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1f/e9cfd801a3f9190bf3e759c422bbfd2247db9d7f3d54a56ecde70137791a/zstandard-0.25.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072", size = 5559635, upload-time = "2025-09-14T22:16:37.141Z" }, + { url = "https://files.pythonhosted.org/packages/21/88/5ba550f797ca953a52d708c8e4f380959e7e3280af029e38fbf47b55916e/zstandard-0.25.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277", size = 5048277, upload-time = "2025-09-14T22:16:38.807Z" }, + { url = "https://files.pythonhosted.org/packages/46/c0/ca3e533b4fa03112facbe7fbe7779cb1ebec215688e5df576fe5429172e0/zstandard-0.25.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313", size = 5574377, upload-time = "2025-09-14T22:16:40.523Z" }, + { url = "https://files.pythonhosted.org/packages/12/9b/3fb626390113f272abd0799fd677ea33d5fc3ec185e62e6be534493c4b60/zstandard-0.25.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097", size = 4961493, upload-time = "2025-09-14T22:16:43.3Z" }, + { url = "https://files.pythonhosted.org/packages/cb/d3/23094a6b6a4b1343b27ae68249daa17ae0651fcfec9ed4de09d14b940285/zstandard-0.25.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778", size = 5269018, upload-time = "2025-09-14T22:16:45.292Z" }, + { url = "https://files.pythonhosted.org/packages/8c/a7/bb5a0c1c0f3f4b5e9d5b55198e39de91e04ba7c205cc46fcb0f95f0383c1/zstandard-0.25.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065", size = 5443672, upload-time = "2025-09-14T22:16:47.076Z" }, + { url = "https://files.pythonhosted.org/packages/27/22/503347aa08d073993f25109c36c8d9f029c7d5949198050962cb568dfa5e/zstandard-0.25.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa", size = 5822753, upload-time = "2025-09-14T22:16:49.316Z" }, + { url = "https://files.pythonhosted.org/packages/e2/be/94267dc6ee64f0f8ba2b2ae7c7a2df934a816baaa7291db9e1aa77394c3c/zstandard-0.25.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7", size = 5366047, upload-time = "2025-09-14T22:16:51.328Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a3/732893eab0a3a7aecff8b99052fecf9f605cf0fb5fb6d0290e36beee47a4/zstandard-0.25.0-cp311-cp311-win32.whl", hash = "sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4", size = 436484, upload-time = "2025-09-14T22:16:55.005Z" }, + { url = "https://files.pythonhosted.org/packages/43/a3/c6155f5c1cce691cb80dfd38627046e50af3ee9ddc5d0b45b9b063bfb8c9/zstandard-0.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2", size = 506183, upload-time = "2025-09-14T22:16:52.753Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3e/8945ab86a0820cc0e0cdbf38086a92868a9172020fdab8a03ac19662b0e5/zstandard-0.25.0-cp311-cp311-win_arm64.whl", hash = "sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137", size = 462533, upload-time = "2025-09-14T22:16:53.878Z" }, + { url = "https://files.pythonhosted.org/packages/82/fc/f26eb6ef91ae723a03e16eddb198abcfce2bc5a42e224d44cc8b6765e57e/zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b", size = 795738, upload-time = "2025-09-14T22:16:56.237Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1c/d920d64b22f8dd028a8b90e2d756e431a5d86194caa78e3819c7bf53b4b3/zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00", size = 640436, upload-time = "2025-09-14T22:16:57.774Z" }, + { url = "https://files.pythonhosted.org/packages/53/6c/288c3f0bd9fcfe9ca41e2c2fbfd17b2097f6af57b62a81161941f09afa76/zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64", size = 5343019, upload-time = "2025-09-14T22:16:59.302Z" }, + { url = "https://files.pythonhosted.org/packages/1e/15/efef5a2f204a64bdb5571e6161d49f7ef0fffdbca953a615efbec045f60f/zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea", size = 5063012, upload-time = "2025-09-14T22:17:01.156Z" }, + { url = "https://files.pythonhosted.org/packages/b7/37/a6ce629ffdb43959e92e87ebdaeebb5ac81c944b6a75c9c47e300f85abdf/zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb", size = 5394148, upload-time = "2025-09-14T22:17:03.091Z" }, + { url = "https://files.pythonhosted.org/packages/e3/79/2bf870b3abeb5c070fe2d670a5a8d1057a8270f125ef7676d29ea900f496/zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a", size = 5451652, upload-time = "2025-09-14T22:17:04.979Z" }, + { url = "https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902", size = 5546993, upload-time = "2025-09-14T22:17:06.781Z" }, + { url = "https://files.pythonhosted.org/packages/85/c7/3483ad9ff0662623f3648479b0380d2de5510abf00990468c286c6b04017/zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f", size = 5046806, upload-time = "2025-09-14T22:17:08.415Z" }, + { url = "https://files.pythonhosted.org/packages/08/b3/206883dd25b8d1591a1caa44b54c2aad84badccf2f1de9e2d60a446f9a25/zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b", size = 5576659, upload-time = "2025-09-14T22:17:10.164Z" }, + { url = "https://files.pythonhosted.org/packages/9d/31/76c0779101453e6c117b0ff22565865c54f48f8bd807df2b00c2c404b8e0/zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6", size = 4953933, upload-time = "2025-09-14T22:17:11.857Z" }, + { url = "https://files.pythonhosted.org/packages/18/e1/97680c664a1bf9a247a280a053d98e251424af51f1b196c6d52f117c9720/zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91", size = 5268008, upload-time = "2025-09-14T22:17:13.627Z" }, + { url = "https://files.pythonhosted.org/packages/1e/73/316e4010de585ac798e154e88fd81bb16afc5c5cb1a72eeb16dd37e8024a/zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708", size = 5433517, upload-time = "2025-09-14T22:17:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/5b/60/dd0f8cfa8129c5a0ce3ea6b7f70be5b33d2618013a161e1ff26c2b39787c/zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512", size = 5814292, upload-time = "2025-09-14T22:17:17.827Z" }, + { url = "https://files.pythonhosted.org/packages/fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa", size = 5360237, upload-time = "2025-09-14T22:17:19.954Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8d/0309daffea4fcac7981021dbf21cdb2e3427a9e76bafbcdbdf5392ff99a4/zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd", size = 436922, upload-time = "2025-09-14T22:17:24.398Z" }, + { url = "https://files.pythonhosted.org/packages/79/3b/fa54d9015f945330510cb5d0b0501e8253c127cca7ebe8ba46a965df18c5/zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01", size = 506276, upload-time = "2025-09-14T22:17:21.429Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6b/8b51697e5319b1f9ac71087b0af9a40d8a6288ff8025c36486e0c12abcc4/zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9", size = 462679, upload-time = "2025-09-14T22:17:23.147Z" }, + { url = "https://files.pythonhosted.org/packages/35/0b/8df9c4ad06af91d39e94fa96cc010a24ac4ef1378d3efab9223cc8593d40/zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94", size = 795735, upload-time = "2025-09-14T22:17:26.042Z" }, + { url = "https://files.pythonhosted.org/packages/3f/06/9ae96a3e5dcfd119377ba33d4c42a7d89da1efabd5cb3e366b156c45ff4d/zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1", size = 640440, upload-time = "2025-09-14T22:17:27.366Z" }, + { url = "https://files.pythonhosted.org/packages/d9/14/933d27204c2bd404229c69f445862454dcc101cd69ef8c6068f15aaec12c/zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f", size = 5343070, upload-time = "2025-09-14T22:17:28.896Z" }, + { url = "https://files.pythonhosted.org/packages/6d/db/ddb11011826ed7db9d0e485d13df79b58586bfdec56e5c84a928a9a78c1c/zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea", size = 5063001, upload-time = "2025-09-14T22:17:31.044Z" }, + { url = "https://files.pythonhosted.org/packages/db/00/87466ea3f99599d02a5238498b87bf84a6348290c19571051839ca943777/zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e", size = 5394120, upload-time = "2025-09-14T22:17:32.711Z" }, + { url = "https://files.pythonhosted.org/packages/2b/95/fc5531d9c618a679a20ff6c29e2b3ef1d1f4ad66c5e161ae6ff847d102a9/zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551", size = 5451230, upload-time = "2025-09-14T22:17:34.41Z" }, + { url = "https://files.pythonhosted.org/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a", size = 5547173, upload-time = "2025-09-14T22:17:36.084Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d5/ba05ed95c6b8ec30bd468dfeab20589f2cf709b5c940483e31d991f2ca58/zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611", size = 5046736, upload-time = "2025-09-14T22:17:37.891Z" }, + { url = "https://files.pythonhosted.org/packages/50/d5/870aa06b3a76c73eced65c044b92286a3c4e00554005ff51962deef28e28/zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3", size = 5576368, upload-time = "2025-09-14T22:17:40.206Z" }, + { url = "https://files.pythonhosted.org/packages/5d/35/398dc2ffc89d304d59bc12f0fdd931b4ce455bddf7038a0a67733a25f550/zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b", size = 4954022, upload-time = "2025-09-14T22:17:41.879Z" }, + { url = "https://files.pythonhosted.org/packages/9a/5c/36ba1e5507d56d2213202ec2b05e8541734af5f2ce378c5d1ceaf4d88dc4/zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851", size = 5267889, upload-time = "2025-09-14T22:17:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/70/e8/2ec6b6fb7358b2ec0113ae202647ca7c0e9d15b61c005ae5225ad0995df5/zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250", size = 5433952, upload-time = "2025-09-14T22:17:45.271Z" }, + { url = "https://files.pythonhosted.org/packages/7b/01/b5f4d4dbc59ef193e870495c6f1275f5b2928e01ff5a81fecb22a06e22fb/zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98", size = 5814054, upload-time = "2025-09-14T22:17:47.08Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf", size = 5360113, upload-time = "2025-09-14T22:17:48.893Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e0/69a553d2047f9a2c7347caa225bb3a63b6d7704ad74610cb7823baa08ed7/zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09", size = 436936, upload-time = "2025-09-14T22:17:52.658Z" }, + { url = "https://files.pythonhosted.org/packages/d9/82/b9c06c870f3bd8767c201f1edbdf9e8dc34be5b0fbc5682c4f80fe948475/zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5", size = 506232, upload-time = "2025-09-14T22:17:50.402Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/60c3c01243bb81d381c9916e2a6d9e149ab8627c0c7d7abb2d73384b3c0c/zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049", size = 462671, upload-time = "2025-09-14T22:17:51.533Z" }, + { url = "https://files.pythonhosted.org/packages/3d/5c/f8923b595b55fe49e30612987ad8bf053aef555c14f05bb659dd5dbe3e8a/zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3", size = 795887, upload-time = "2025-09-14T22:17:54.198Z" }, + { url = "https://files.pythonhosted.org/packages/8d/09/d0a2a14fc3439c5f874042dca72a79c70a532090b7ba0003be73fee37ae2/zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f", size = 640658, upload-time = "2025-09-14T22:17:55.423Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7c/8b6b71b1ddd517f68ffb55e10834388d4f793c49c6b83effaaa05785b0b4/zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c", size = 5379849, upload-time = "2025-09-14T22:17:57.372Z" }, + { url = "https://files.pythonhosted.org/packages/a4/86/a48e56320d0a17189ab7a42645387334fba2200e904ee47fc5a26c1fd8ca/zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439", size = 5058095, upload-time = "2025-09-14T22:17:59.498Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ad/eb659984ee2c0a779f9d06dbfe45e2dc39d99ff40a319895df2d3d9a48e5/zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043", size = 5551751, upload-time = "2025-09-14T22:18:01.618Z" }, + { url = "https://files.pythonhosted.org/packages/61/b3/b637faea43677eb7bd42ab204dfb7053bd5c4582bfe6b1baefa80ac0c47b/zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859", size = 6364818, upload-time = "2025-09-14T22:18:03.769Z" }, + { url = "https://files.pythonhosted.org/packages/31/dc/cc50210e11e465c975462439a492516a73300ab8caa8f5e0902544fd748b/zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0", size = 5560402, upload-time = "2025-09-14T22:18:05.954Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ae/56523ae9c142f0c08efd5e868a6da613ae76614eca1305259c3bf6a0ed43/zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7", size = 4955108, upload-time = "2025-09-14T22:18:07.68Z" }, + { url = "https://files.pythonhosted.org/packages/98/cf/c899f2d6df0840d5e384cf4c4121458c72802e8bda19691f3b16619f51e9/zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2", size = 5269248, upload-time = "2025-09-14T22:18:09.753Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c0/59e912a531d91e1c192d3085fc0f6fb2852753c301a812d856d857ea03c6/zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344", size = 5430330, upload-time = "2025-09-14T22:18:11.966Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/7e31db1240de2df22a58e2ea9a93fc6e38cc29353e660c0272b6735d6669/zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c", size = 5811123, upload-time = "2025-09-14T22:18:13.907Z" }, + { url = "https://files.pythonhosted.org/packages/f6/49/fac46df5ad353d50535e118d6983069df68ca5908d4d65b8c466150a4ff1/zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088", size = 5359591, upload-time = "2025-09-14T22:18:16.465Z" }, + { url = "https://files.pythonhosted.org/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" }, + { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, +] From a466079c2bbfd0837315f29c2c540fbba5714f21 Mon Sep 17 00:00:00 2001 From: Vibe Nuage Agent Date: Tue, 19 May 2026 16:42:03 +0000 Subject: [PATCH 02/16] Remove unused workflow encoding metadata --- .../extra/tests/test_workflow_encoding.py | 17 ++----------- .../extra/workflows/encoding/models.py | 13 ---------- .../workflows/encoding/payload_encoder.py | 25 ++++++------------- 3 files changed, 10 insertions(+), 45 deletions(-) diff --git a/src/mistralai/extra/tests/test_workflow_encoding.py b/src/mistralai/extra/tests/test_workflow_encoding.py index 3107043c..6756c81c 100644 --- a/src/mistralai/extra/tests/test_workflow_encoding.py +++ b/src/mistralai/extra/tests/test_workflow_encoding.py @@ -170,7 +170,6 @@ async def test_payload_encoder_compresses_network_inputs(): ) assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] - assert encoded.encoding_metadata == {} compressed_payload = json.loads(encoded.get_payload()) assert compressed_payload["algorithm_config"] == {"algorithm": "zstd", "level": 3} @@ -192,7 +191,6 @@ async def test_payload_encoder_skips_compression_below_min_size(): ) assert encoded.encoding_options == [] - assert encoded.encoding_metadata == {} decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) assert decoded == payload @@ -212,7 +210,6 @@ async def test_payload_encoder_skips_compression_when_not_smaller(): ) assert encoded.encoding_options == [] - assert encoded.encoding_metadata == {} decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) assert decoded == payload @@ -227,7 +224,6 @@ async def test_payload_encoder_skips_compression_without_config(): ) assert encoded.encoding_options == [] - assert encoded.encoding_metadata == {} decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) assert decoded == payload @@ -259,7 +255,6 @@ async def test_payload_encoder_compression_can_prevent_offloading(monkeypatch): ) assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] - assert encoded.encoding_metadata == {} assert storage.blobs == {} decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) assert decoded == payload @@ -523,7 +518,6 @@ async def test_payload_encoder_compression_offloading_encryption_roundtrip( ) assert encoded.encoding_options == expected_options - assert encoded.encoding_metadata == {} assert len(storage.blobs) == 1 decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) assert decoded == payload @@ -548,7 +542,6 @@ async def test_payload_encoder_does_not_partially_encrypt_when_no_marked_fields( ) assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] - assert encoded.encoding_metadata == {} decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) assert decoded == payload @@ -567,14 +560,8 @@ async def test_payload_encoder_event_payload_orders_compression_before_full_encr encoder = PayloadEncoder(encoding_config=config) payload = json.dumps({"data": "x" * 20_000}).encode() - ( - encoded, - encoding_options, - encoding_metadata, - ) = await encoder.encode_event_payload_content(payload) - decoded = await encoder.decode_payload_content( - encoded, encoding_options, encoding_metadata - ) + encoded, encoding_options = await encoder.encode_event_payload_content(payload) + decoded = await encoder.decode_payload_content(encoded, encoding_options) assert encoding_options == [ EncodedPayloadOptions.COMPRESSED, diff --git a/src/mistralai/extra/workflows/encoding/models.py b/src/mistralai/extra/workflows/encoding/models.py index 0cf1a648..1a26d21d 100644 --- a/src/mistralai/extra/workflows/encoding/models.py +++ b/src/mistralai/extra/workflows/encoding/models.py @@ -55,10 +55,6 @@ class EncodedPayload(BaseModel): encoding_options: list[EncodedPayloadOptions] = Field( description="The encoding of the payload", default=[] ) - encoding_metadata: dict[str, str] = Field( - description="Additional metadata required to decode the payload", - default_factory=dict, - ) payload: bytes = Field(description="The encoded payload") @@ -67,10 +63,6 @@ class NetworkEncodedBase(BaseModel): encoding_options: list[EncodedPayloadOptions] = Field( description="The encoding of the payload", default=[] ) - encoding_metadata: dict[str, str] = Field( - description="Additional metadata required to decode the payload", - default_factory=dict, - ) def get_payload(self) -> bytes: return base64.b64decode(self.b64payload) @@ -85,7 +77,6 @@ def to_encoded_payload( return EncodedPayload( payload=base64.b64decode(self.b64payload), encoding_options=self.encoding_options, - encoding_metadata=self.encoding_metadata, context=WorkflowContext( namespace=namespace, execution_id=execution_id, @@ -98,19 +89,16 @@ def from_encoded_payload(encoded_payload: EncodedPayload) -> "NetworkEncodedInpu return NetworkEncodedInput( b64payload=base64.b64encode(encoded_payload.payload).decode("utf-8"), encoding_options=encoded_payload.encoding_options, - encoding_metadata=encoded_payload.encoding_metadata, ) @staticmethod def from_data( data: bytes, encoding_options: list[EncodedPayloadOptions], - encoding_metadata: dict[str, str] | None = None, ) -> "NetworkEncodedInput": return NetworkEncodedInput( b64payload=base64.b64encode(data).decode("utf-8"), encoding_options=encoding_options, - encoding_metadata=encoding_metadata or {}, ) @@ -120,5 +108,4 @@ def from_encoded_payload(encoded_payload: EncodedPayload) -> "NetworkEncodedResu return NetworkEncodedResult( b64payload=base64.b64encode(encoded_payload.payload).decode("utf-8"), encoding_options=encoded_payload.encoding_options, - encoding_metadata=encoded_payload.encoding_metadata, ) diff --git a/src/mistralai/extra/workflows/encoding/payload_encoder.py b/src/mistralai/extra/workflows/encoding/payload_encoder.py index d9d3e711..cccf48e0 100644 --- a/src/mistralai/extra/workflows/encoding/payload_encoder.py +++ b/src/mistralai/extra/workflows/encoding/payload_encoder.py @@ -262,7 +262,7 @@ def _decompress(self, data: bytes) -> bytes: async def encode_payload_content( self, data: Union[bytes, str], context: Optional[WorkflowContext] - ) -> tuple[bytes, list[EncodedPayloadOptions], dict[str, str]]: + ) -> tuple[bytes, list[EncodedPayloadOptions]]: """Handle payload encoding. Encoding options are appended in the exact order in which transforms are @@ -272,7 +272,6 @@ async def encode_payload_content( data = data.encode() encoding_options: list[EncodedPayloadOptions] = [] - encoding_metadata: dict[str, str] = {} # Partial encryption needs the original JSON fields. It must run before # compression or offloading, which make field-level markers unavailable. @@ -305,11 +304,11 @@ async def encode_payload_content( data = self._encrypt(data) encoding_options.append(EncodedPayloadOptions.ENCRYPTED) - return data, encoding_options, encoding_metadata + return data, encoding_options async def encode_event_payload_content( self, data: Union[bytes, str], force_full_encryption: bool = False - ) -> tuple[bytes, list[EncodedPayloadOptions], dict[str, str]]: + ) -> tuple[bytes, list[EncodedPayloadOptions]]: """Encode event payload content. Unlike encode_payload_content, this only handles in-payload transforms @@ -324,7 +323,6 @@ async def encode_event_payload_content( data = data.encode() encoding_options: list[EncodedPayloadOptions] = [] - encoding_metadata: dict[str, str] = {} # Partial encryption needs JSON bytes; compression should happen before # full encryption because encrypted bytes are intentionally high entropy. @@ -348,13 +346,12 @@ async def encode_event_payload_content( data = self._encrypt(data) encoding_options.append(EncodedPayloadOptions.ENCRYPTED) - return data, encoding_options, encoding_metadata + return data, encoding_options async def decode_payload_content( self, data: bytes, encoding_options: List[EncodedPayloadOptions], - encoding_metadata: dict[str, str] | None = None, ) -> bytes: # Decode in the exact reverse order of the encoding_options wire list. for option in reversed(encoding_options): @@ -403,9 +400,8 @@ async def decode_event_payload( encoding_options = [EncodedPayloadOptions(opt) for opt in encoding_options_strs] encrypted_bytes = base64.b64decode(payload_data["value"]) - encoding_metadata = payload_data.get("encoding_metadata", {}) decrypted_bytes = await self.decode_payload_content( - encrypted_bytes, encoding_options, encoding_metadata + encrypted_bytes, encoding_options ) decrypted_value = json.loads(decrypted_bytes) @@ -421,14 +417,10 @@ async def encode_network_input( """This method MUST be called to format every payload send to Mistral Workflows control plane to ensure a proper encoding of the payload. """ - ( - encoded_data, - encoding_options, - encoding_metadata, - ) = await self.encode_payload_content(to_json(data), context) - network_input = NetworkEncodedInput.from_data( - encoded_data, encoding_options, encoding_metadata + encoded_data, encoding_options = await self.encode_payload_content( + to_json(data), context ) + network_input = NetworkEncodedInput.from_data(encoded_data, encoding_options) return network_input async def decode_network_result(self, data: Any) -> Any: @@ -444,7 +436,6 @@ async def decode_network_result(self, data: Any) -> Any: byte_results = await self.decode_payload_content( network_encoded_payload.get_payload(), network_encoded_payload.encoding_options, - network_encoded_payload.encoding_metadata, ) try: return from_json(byte_results) From 18ff480bf69381f6638c6e600b67e10ab73d55e1 Mon Sep 17 00:00:00 2001 From: Vibe Nuage Agent Date: Tue, 19 May 2026 22:23:22 +0000 Subject: [PATCH 03/16] Clean up workflow compression encoder --- docs/models/encodedpayloadoptions.md | 1 + .../client/models/encodedpayloadoptions.py | 1 + .../extra/tests/test_workflow_encoding.py | 24 ----------- .../workflows/encoding/payload_compressor.py | 11 +---- .../workflows/encoding/payload_encoder.py | 42 ------------------- 5 files changed, 3 insertions(+), 76 deletions(-) diff --git a/docs/models/encodedpayloadoptions.md b/docs/models/encodedpayloadoptions.md index 51fcfdc4..39051266 100644 --- a/docs/models/encodedpayloadoptions.md +++ b/docs/models/encodedpayloadoptions.md @@ -17,3 +17,4 @@ This is an open enum. Unrecognized values will not fail type checks. - `"offloaded"` - `"encrypted"` - `"encrypted-partial"` +- `"compressed"` diff --git a/src/mistralai/client/models/encodedpayloadoptions.py b/src/mistralai/client/models/encodedpayloadoptions.py index cc19b543..54bcaf13 100644 --- a/src/mistralai/client/models/encodedpayloadoptions.py +++ b/src/mistralai/client/models/encodedpayloadoptions.py @@ -11,6 +11,7 @@ "offloaded", "encrypted", "encrypted-partial", + "compressed", ], UnrecognizedStr, ] diff --git a/src/mistralai/extra/tests/test_workflow_encoding.py b/src/mistralai/extra/tests/test_workflow_encoding.py index 6756c81c..99827011 100644 --- a/src/mistralai/extra/tests/test_workflow_encoding.py +++ b/src/mistralai/extra/tests/test_workflow_encoding.py @@ -544,27 +544,3 @@ async def test_payload_encoder_does_not_partially_encrypt_when_no_marked_fields( assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) assert decoded == payload - - -@pytest.mark.asyncio -async def test_payload_encoder_event_payload_orders_compression_before_full_encryption(): - config = WorkflowEncodingConfig( - payload_encryption=PayloadEncryptionConfig( - mode=PayloadEncryptionMode.FULL, - main_key=SecretStr("0" * 64), - ), - payload_compression=PayloadCompressionConfig( - min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=3) - ), - ) - encoder = PayloadEncoder(encoding_config=config) - payload = json.dumps({"data": "x" * 20_000}).encode() - - encoded, encoding_options = await encoder.encode_event_payload_content(payload) - decoded = await encoder.decode_payload_content(encoded, encoding_options) - - assert encoding_options == [ - EncodedPayloadOptions.COMPRESSED, - EncodedPayloadOptions.ENCRYPTED, - ] - assert decoded == payload diff --git a/src/mistralai/extra/workflows/encoding/payload_compressor.py b/src/mistralai/extra/workflows/encoding/payload_compressor.py index ff8b02d4..843069ec 100644 --- a/src/mistralai/extra/workflows/encoding/payload_compressor.py +++ b/src/mistralai/extra/workflows/encoding/payload_compressor.py @@ -19,11 +19,6 @@ class Compressor(ABC): - @property - @abstractmethod - def tag(self) -> str: - """Wire tag stored in encoding options to identify the algorithm.""" - @property @abstractmethod def algorithm_config(self) -> AlgorithmConfig: @@ -41,15 +36,11 @@ def _require_zstandard() -> ModuleType: return import_module("zstandard") except ImportError: raise WorkflowPayloadCompressionException( - "Payload compression requires installing mistralai[workflow-payload-compression]" + "Payload compression requires installing mistralai[workflow_payload_compression]" ) from None class ZstdCompressor(Compressor): - @property - def tag(self) -> str: - return "zstd" - @property def algorithm_config(self) -> AlgorithmConfig: return self._config diff --git a/src/mistralai/extra/workflows/encoding/payload_encoder.py b/src/mistralai/extra/workflows/encoding/payload_encoder.py index cccf48e0..d8209ce8 100644 --- a/src/mistralai/extra/workflows/encoding/payload_encoder.py +++ b/src/mistralai/extra/workflows/encoding/payload_encoder.py @@ -306,48 +306,6 @@ async def encode_payload_content( return data, encoding_options - async def encode_event_payload_content( - self, data: Union[bytes, str], force_full_encryption: bool = False - ) -> tuple[bytes, list[EncodedPayloadOptions]]: - """Encode event payload content. - - Unlike encode_payload_content, this only handles in-payload transforms - (partial/full encryption and compression), never blob offloading. - - Args: - data: The payload data to encrypt. - force_full_encryption: Force full encryption regardless of configured mode. - Use for payloads like json_patch that don't support partial encryption. - """ - if isinstance(data, str): - data = data.encode() - - encoding_options: list[EncodedPayloadOptions] = [] - - # Partial encryption needs JSON bytes; compression should happen before - # full encryption because encrypted bytes are intentionally high entropy. - if ( - not force_full_encryption - and self.encryption_config is not None - and self.encryption_config.mode == PayloadEncryptionMode.PARTIAL - ): - data, partially_encrypted = await self._partially_encrypt_fields(data) - if partially_encrypted: - encoding_options.append(EncodedPayloadOptions.PARTIALLY_ENCRYPTED) - - data, compressed = self._compress(data) - if compressed: - encoding_options.append(EncodedPayloadOptions.COMPRESSED) - - if self.encryption_config is not None and ( - force_full_encryption - or self.encryption_config.mode == PayloadEncryptionMode.FULL - ): - data = self._encrypt(data) - encoding_options.append(EncodedPayloadOptions.ENCRYPTED) - - return data, encoding_options - async def decode_payload_content( self, data: bytes, From a647ba6ae677e4540fc114f45195412d857484cd Mon Sep 17 00:00:00 2001 From: Vibe Nuage Agent Date: Tue, 19 May 2026 23:19:34 +0000 Subject: [PATCH 04/16] Store workflow compression config in metadata --- .../extra/tests/test_workflow_encoding.py | 43 +++++++++---------- .../extra/workflows/encoding/models.py | 34 ++++++--------- .../workflows/encoding/payload_compressor.py | 26 +++++++---- .../workflows/encoding/payload_encoder.py | 43 +++++++++++++------ 4 files changed, 80 insertions(+), 66 deletions(-) diff --git a/src/mistralai/extra/tests/test_workflow_encoding.py b/src/mistralai/extra/tests/test_workflow_encoding.py index 99827011..36ea2ce9 100644 --- a/src/mistralai/extra/tests/test_workflow_encoding.py +++ b/src/mistralai/extra/tests/test_workflow_encoding.py @@ -1,6 +1,5 @@ """Tests for workflow encoding configuration lifecycle.""" -import base64 import gc import json @@ -170,9 +169,10 @@ async def test_payload_encoder_compresses_network_inputs(): ) assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] - - compressed_payload = json.loads(encoded.get_payload()) - assert compressed_payload["algorithm_config"] == {"algorithm": "zstd", "level": 3} + assert encoded.encoding_metadata == { + "compression": {"algorithm": "zstd", "level": 3} + } + assert not encoded.get_payload().startswith(b"{") decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) assert decoded == payload @@ -287,13 +287,14 @@ async def test_payload_encoder_decodes_compressed_payload_with_decoder_config( encoded = await encoder.encode_network_input( payload, WorkflowContext(namespace="test", execution_id="exec") ) - compressed_payload = json.loads(encoded.get_payload()) decoded = await PayloadEncoder(decoder_config).decode_network_result( encoded.model_dump(mode="json") ) assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] - assert compressed_payload["algorithm_config"] == {"algorithm": "zstd", "level": 22} + assert encoded.encoding_metadata == { + "compression": {"algorithm": "zstd", "level": 22} + } assert decoded == payload @@ -363,10 +364,10 @@ async def test_payload_encoder_decodes_with_tampered_compression_level(): encoded = await encoder.encode_network_input( payload, WorkflowContext(namespace="test", execution_id="exec") ) - compressed_payload = json.loads(encoded.get_payload()) - compressed_payload["algorithm_config"]["level"] = 1 tampered = NetworkEncodedInput.from_data( - json.dumps(compressed_payload).encode(), encoded.encoding_options + encoded.get_payload(), + encoded.encoding_options, + {"compression": {"algorithm": "zstd", "level": 1}}, ) decoded = await PayloadEncoder(WorkflowEncodingConfig()).decode_network_result( @@ -378,21 +379,21 @@ async def test_payload_encoder_decodes_with_tampered_compression_level(): @pytest.mark.asyncio @pytest.mark.parametrize( - "compressed_payload", + "encoding_metadata", [ - b"not-json", - b'{"algorithm_config":{"algorithm":"lz4","level":1},"b64data":"AAAA"}', - b'{"algorithm_config":{"level":3},"b64data":"AAAA"}', + {}, + {"compression": {"algorithm": "lz4", "level": 1}}, + {"compression": {"level": 3}}, ], ) -async def test_payload_encoder_invalid_compressed_payload_is_error( - compressed_payload: bytes, +async def test_payload_encoder_invalid_compression_metadata_is_error( + encoding_metadata: dict[str, object], ): encoded = NetworkEncodedInput.from_data( - compressed_payload, [EncodedPayloadOptions.COMPRESSED] + b"compressed-data", [EncodedPayloadOptions.COMPRESSED], encoding_metadata ) - with pytest.raises(WorkflowPayloadCompressionException, match="Invalid compressed payload"): + with pytest.raises(WorkflowPayloadCompressionException): await PayloadEncoder(WorkflowEncodingConfig()).decode_network_result( encoded.model_dump(mode="json") ) @@ -401,13 +402,9 @@ async def test_payload_encoder_invalid_compressed_payload_is_error( @pytest.mark.asyncio async def test_payload_encoder_corrupted_compressed_data_is_error(): encoded = NetworkEncodedInput.from_data( - json.dumps( - { - "algorithm_config": {"algorithm": "zstd", "level": 3}, - "b64data": base64.b64encode(b"corrupted-data").decode(), - } - ).encode(), + b"corrupted-data", [EncodedPayloadOptions.COMPRESSED], + {"compression": {"algorithm": "zstd", "level": 3}}, ) with pytest.raises(zstandard.ZstdError): diff --git a/src/mistralai/extra/workflows/encoding/models.py b/src/mistralai/extra/workflows/encoding/models.py index 1a26d21d..984401c3 100644 --- a/src/mistralai/extra/workflows/encoding/models.py +++ b/src/mistralai/extra/workflows/encoding/models.py @@ -1,11 +1,9 @@ from enum import Enum -from typing import Literal, Optional +from typing import Any, Literal, Optional import base64 from pydantic import BaseModel, Field -from mistralai.extra.workflows.encoding.config import AlgorithmConfig - class EncodedPayloadOptions(str, Enum): OFFLOADED = "offloaded" @@ -25,23 +23,6 @@ class EncryptedStrField(BaseModel): data: str -class CompressedPayloadData(BaseModel): - algorithm_config: AlgorithmConfig - b64data: str - - @staticmethod - def from_data( - data: bytes, algorithm_config: AlgorithmConfig - ) -> "CompressedPayloadData": - return CompressedPayloadData( - algorithm_config=algorithm_config, - b64data=base64.b64encode(data).decode("utf-8"), - ) - - def get_data(self) -> bytes: - return base64.b64decode(self.b64data) - - class WorkflowContext(BaseModel): namespace: str execution_id: str @@ -55,6 +36,10 @@ class EncodedPayload(BaseModel): encoding_options: list[EncodedPayloadOptions] = Field( description="The encoding of the payload", default=[] ) + encoding_metadata: dict[str, Any] = Field( + description="Additional metadata required to decode the payload", + default_factory=dict, + ) payload: bytes = Field(description="The encoded payload") @@ -63,6 +48,10 @@ class NetworkEncodedBase(BaseModel): encoding_options: list[EncodedPayloadOptions] = Field( description="The encoding of the payload", default=[] ) + encoding_metadata: dict[str, Any] = Field( + description="Additional metadata required to decode the payload", + default_factory=dict, + ) def get_payload(self) -> bytes: return base64.b64decode(self.b64payload) @@ -77,6 +66,7 @@ def to_encoded_payload( return EncodedPayload( payload=base64.b64decode(self.b64payload), encoding_options=self.encoding_options, + encoding_metadata=self.encoding_metadata, context=WorkflowContext( namespace=namespace, execution_id=execution_id, @@ -89,16 +79,19 @@ def from_encoded_payload(encoded_payload: EncodedPayload) -> "NetworkEncodedInpu return NetworkEncodedInput( b64payload=base64.b64encode(encoded_payload.payload).decode("utf-8"), encoding_options=encoded_payload.encoding_options, + encoding_metadata=encoded_payload.encoding_metadata, ) @staticmethod def from_data( data: bytes, encoding_options: list[EncodedPayloadOptions], + encoding_metadata: dict[str, Any] | None = None, ) -> "NetworkEncodedInput": return NetworkEncodedInput( b64payload=base64.b64encode(data).decode("utf-8"), encoding_options=encoding_options, + encoding_metadata=encoding_metadata or {}, ) @@ -108,4 +101,5 @@ def from_encoded_payload(encoded_payload: EncodedPayload) -> "NetworkEncodedResu return NetworkEncodedResult( b64payload=base64.b64encode(encoded_payload.payload).decode("utf-8"), encoding_options=encoded_payload.encoding_options, + encoding_metadata=encoded_payload.encoding_metadata, ) diff --git a/src/mistralai/extra/workflows/encoding/payload_compressor.py b/src/mistralai/extra/workflows/encoding/payload_compressor.py index 843069ec..2f372ac4 100644 --- a/src/mistralai/extra/workflows/encoding/payload_compressor.py +++ b/src/mistralai/extra/workflows/encoding/payload_compressor.py @@ -13,16 +13,16 @@ PayloadCompressionConfig, ZstdCompressionConfig, ) -from mistralai.extra.workflows.encoding.models import CompressedPayloadData _ALGORITHM_CONFIG_ADAPTER: TypeAdapter[AlgorithmConfig] = TypeAdapter(AlgorithmConfig) +_COMPRESSION_METADATA_KEY = "compression" class Compressor(ABC): @property @abstractmethod def algorithm_config(self) -> AlgorithmConfig: - """Algorithm config stored with compressed bytes for config-independent decoding.""" + """Algorithm config stored with metadata for config-independent decoding.""" @abstractmethod def compress(self, data: bytes) -> bytes: ... @@ -86,14 +86,22 @@ def build_compressor( ) -def decompress_payload(data: bytes) -> bytes: +def compression_metadata(compressor: Compressor) -> dict[str, object]: + return { + _COMPRESSION_METADATA_KEY: compressor.algorithm_config.model_dump(mode="json") + } + + +def compressor_from_metadata(metadata: dict[str, object]) -> Compressor: + config = metadata.get(_COMPRESSION_METADATA_KEY) + if not isinstance(config, dict): + raise WorkflowPayloadCompressionException( + "Missing compression config in payload metadata" + ) try: - compressed_payload = CompressedPayloadData.model_validate_json(data) + algo_config = _ALGORITHM_CONFIG_ADAPTER.validate_python(config) except ValidationError as exc: raise WorkflowPayloadCompressionException( - f"Invalid compressed payload: {exc}" + f"Invalid compression config in payload metadata: {exc}" ) from exc - compressor = _build_compressor_for_config( - compressed_payload.algorithm_config.model_dump_json() - ) - return compressor.decompress(compressed_payload.get_data()) + return _build_compressor_for_config(algo_config.model_dump_json()) diff --git a/src/mistralai/extra/workflows/encoding/payload_encoder.py b/src/mistralai/extra/workflows/encoding/payload_encoder.py index d8209ce8..6813ffd3 100644 --- a/src/mistralai/extra/workflows/encoding/payload_encoder.py +++ b/src/mistralai/extra/workflows/encoding/payload_encoder.py @@ -32,11 +32,11 @@ ) from mistralai.extra.workflows.encoding.payload_compressor import ( build_compressor, - decompress_payload, + compression_metadata, + compressor_from_metadata, ) from .storage.blob_storage import get_blob_storage, BlobNotFoundError from mistralai.extra.workflows.encoding.models import ( - CompressedPayloadData, EncodedPayloadOptions, EncryptableFieldTypes, NetworkEncodedInput, @@ -252,17 +252,21 @@ def _compress(self, data: bytes) -> tuple[bytes, bool]: compressed = self.compressor.compress(data) if len(compressed) >= len(data): return data, False - compressed_payload = CompressedPayloadData.from_data( - compressed, self.compressor.algorithm_config + return compressed, True + + def _compression_metadata(self) -> dict[str, object]: + assert self.compressor is not None, ( + "This should never be reached: compression metadata was requested " + "but PayloadEncoder.__init__ did not build a compressor" ) - return compressed_payload.model_dump_json().encode(), True + return compression_metadata(self.compressor) - def _decompress(self, data: bytes) -> bytes: - return decompress_payload(data) + def _decompress(self, data: bytes, encoding_metadata: dict[str, object]) -> bytes: + return compressor_from_metadata(encoding_metadata).decompress(data) async def encode_payload_content( self, data: Union[bytes, str], context: Optional[WorkflowContext] - ) -> tuple[bytes, list[EncodedPayloadOptions]]: + ) -> tuple[bytes, list[EncodedPayloadOptions], dict[str, object]]: """Handle payload encoding. Encoding options are appended in the exact order in which transforms are @@ -272,6 +276,7 @@ async def encode_payload_content( data = data.encode() encoding_options: list[EncodedPayloadOptions] = [] + encoding_metadata: dict[str, object] = {} # Partial encryption needs the original JSON fields. It must run before # compression or offloading, which make field-level markers unavailable. @@ -288,6 +293,7 @@ async def encode_payload_content( data, compressed = self._compress(data) if compressed: encoding_options.append(EncodedPayloadOptions.COMPRESSED) + encoding_metadata.update(self._compression_metadata()) if self.offloading_config is not None: data, offloaded = await self._handle_offloading(data, context) @@ -304,13 +310,15 @@ async def encode_payload_content( data = self._encrypt(data) encoding_options.append(EncodedPayloadOptions.ENCRYPTED) - return data, encoding_options + return data, encoding_options, encoding_metadata async def decode_payload_content( self, data: bytes, encoding_options: List[EncodedPayloadOptions], + encoding_metadata: dict[str, object] | None = None, ) -> bytes: + encoding_metadata = encoding_metadata or {} # Decode in the exact reverse order of the encoding_options wire list. for option in reversed(encoding_options): if option == EncodedPayloadOptions.ENCRYPTED: @@ -318,7 +326,7 @@ async def decode_payload_content( elif option == EncodedPayloadOptions.PARTIALLY_ENCRYPTED: data, _ = await self._partially_decrypt_fields(data) elif option == EncodedPayloadOptions.COMPRESSED: - data = self._decompress(data) + data = self._decompress(data, encoding_metadata) elif option == EncodedPayloadOptions.OFFLOADED: if ( self.offloading_config is None @@ -359,7 +367,9 @@ async def decode_event_payload( encoding_options = [EncodedPayloadOptions(opt) for opt in encoding_options_strs] encrypted_bytes = base64.b64decode(payload_data["value"]) decrypted_bytes = await self.decode_payload_content( - encrypted_bytes, encoding_options + encrypted_bytes, + encoding_options, + payload_data.get("encoding_metadata", {}), ) decrypted_value = json.loads(decrypted_bytes) @@ -375,10 +385,14 @@ async def encode_network_input( """This method MUST be called to format every payload send to Mistral Workflows control plane to ensure a proper encoding of the payload. """ - encoded_data, encoding_options = await self.encode_payload_content( - to_json(data), context + ( + encoded_data, + encoding_options, + encoding_metadata, + ) = await self.encode_payload_content(to_json(data), context) + network_input = NetworkEncodedInput.from_data( + encoded_data, encoding_options, encoding_metadata ) - network_input = NetworkEncodedInput.from_data(encoded_data, encoding_options) return network_input async def decode_network_result(self, data: Any) -> Any: @@ -394,6 +408,7 @@ async def decode_network_result(self, data: Any) -> Any: byte_results = await self.decode_payload_content( network_encoded_payload.get_payload(), network_encoded_payload.encoding_options, + network_encoded_payload.encoding_metadata, ) try: return from_json(byte_results) From 50ec6c70b19eb3af76b06819a5266154fe638e22 Mon Sep 17 00:00:00 2001 From: Vibe Nuage Agent Date: Wed, 20 May 2026 07:59:14 +0000 Subject: [PATCH 05/16] Restore workflow event payload metadata encoding --- .../extra/tests/test_workflow_encoding.py | 78 ++++++++++++++++++- .../workflows/encoding/payload_encoder.py | 33 ++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/src/mistralai/extra/tests/test_workflow_encoding.py b/src/mistralai/extra/tests/test_workflow_encoding.py index 36ea2ce9..3bdecbde 100644 --- a/src/mistralai/extra/tests/test_workflow_encoding.py +++ b/src/mistralai/extra/tests/test_workflow_encoding.py @@ -1,5 +1,6 @@ """Tests for workflow encoding configuration lifecycle.""" +import base64 import gc import json @@ -304,7 +305,10 @@ async def test_payload_encoder_decodes_compressed_payload_with_decoder_config( [ ( PayloadEncryptionMode.PARTIAL, - [EncodedPayloadOptions.PARTIALLY_ENCRYPTED, EncodedPayloadOptions.COMPRESSED], + [ + EncodedPayloadOptions.PARTIALLY_ENCRYPTED, + EncodedPayloadOptions.COMPRESSED, + ], ), ( PayloadEncryptionMode.FULL, @@ -541,3 +545,75 @@ async def test_payload_encoder_does_not_partially_encrypt_when_no_marked_fields( assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) assert decoded == payload + + +@pytest.mark.asyncio +async def test_payload_encoder_event_payload_sets_compression_metadata(): + config = WorkflowEncodingConfig( + payload_encryption=PayloadEncryptionConfig( + mode=PayloadEncryptionMode.FULL, + main_key=SecretStr("0" * 64), + ), + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=3) + ), + ) + encoder = PayloadEncoder(encoding_config=config) + decoder = PayloadEncoder( + encoding_config=WorkflowEncodingConfig( + payload_encryption=config.payload_encryption, + ) + ) + payload = json.dumps({"data": "x" * 20_000}).encode() + + ( + encoded, + encoding_options, + encoding_metadata, + ) = await encoder.encode_event_payload_content(payload) + decoded = await decoder.decode_payload_content( + encoded, encoding_options, encoding_metadata + ) + + assert encoding_options == [ + EncodedPayloadOptions.COMPRESSED, + EncodedPayloadOptions.ENCRYPTED, + ] + assert encoding_metadata == {"compression": {"algorithm": "zstd", "level": 3}} + assert decoded == payload + + +@pytest.mark.asyncio +async def test_payload_encoder_decodes_event_payload_with_compression_metadata(): + config = WorkflowEncodingConfig( + payload_encryption=PayloadEncryptionConfig( + mode=PayloadEncryptionMode.FULL, + main_key=SecretStr("0" * 64), + ), + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=3) + ), + ) + encoder = PayloadEncoder(encoding_config=config) + decoder = PayloadEncoder( + encoding_config=WorkflowEncodingConfig( + payload_encryption=config.payload_encryption, + ) + ) + payload = {"data": "x" * 20_000} + ( + encoded, + encoding_options, + encoding_metadata, + ) = await encoder.encode_event_payload_content(json.dumps(payload).encode()) + + decoded = await decoder.decode_event_payload( + { + "type": "json", + "value": base64.b64encode(encoded).decode(), + "encoding_options": [option.value for option in encoding_options], + "encoding_metadata": encoding_metadata, + } + ) + + assert decoded == {"type": "json", "value": payload, "encoding_options": []} diff --git a/src/mistralai/extra/workflows/encoding/payload_encoder.py b/src/mistralai/extra/workflows/encoding/payload_encoder.py index 6813ffd3..bbe8b576 100644 --- a/src/mistralai/extra/workflows/encoding/payload_encoder.py +++ b/src/mistralai/extra/workflows/encoding/payload_encoder.py @@ -312,6 +312,39 @@ async def encode_payload_content( return data, encoding_options, encoding_metadata + async def encode_event_payload_content( + self, data: Union[bytes, str], force_full_encryption: bool = False + ) -> tuple[bytes, list[EncodedPayloadOptions], dict[str, object]]: + """Encode event payload content without blob offloading.""" + if isinstance(data, str): + data = data.encode() + + encoding_options: list[EncodedPayloadOptions] = [] + encoding_metadata: dict[str, object] = {} + + if ( + not force_full_encryption + and self.encryption_config is not None + and self.encryption_config.mode == PayloadEncryptionMode.PARTIAL + ): + data, partially_encrypted = await self._partially_encrypt_fields(data) + if partially_encrypted: + encoding_options.append(EncodedPayloadOptions.PARTIALLY_ENCRYPTED) + + data, compressed = self._compress(data) + if compressed: + encoding_options.append(EncodedPayloadOptions.COMPRESSED) + encoding_metadata.update(self._compression_metadata()) + + if self.encryption_config is not None and ( + force_full_encryption + or self.encryption_config.mode == PayloadEncryptionMode.FULL + ): + data = self._encrypt(data) + encoding_options.append(EncodedPayloadOptions.ENCRYPTED) + + return data, encoding_options, encoding_metadata + async def decode_payload_content( self, data: bytes, From 7cb81743431d7133ef3fa461349bb07b6fd47997 Mon Sep 17 00:00:00 2001 From: Vibe Nuage Agent Date: Wed, 20 May 2026 08:16:21 +0000 Subject: [PATCH 06/16] Consolidate workflow payload encoding paths --- .../extra/tests/test_workflow_encoding.py | 69 +++++-------------- .../workflows/encoding/payload_encoder.py | 45 +++--------- 2 files changed, 28 insertions(+), 86 deletions(-) diff --git a/src/mistralai/extra/tests/test_workflow_encoding.py b/src/mistralai/extra/tests/test_workflow_encoding.py index 3bdecbde..8bab8ef1 100644 --- a/src/mistralai/extra/tests/test_workflow_encoding.py +++ b/src/mistralai/extra/tests/test_workflow_encoding.py @@ -1,6 +1,5 @@ """Tests for workflow encoding configuration lifecycle.""" -import base64 import gc import json @@ -548,29 +547,35 @@ async def test_payload_encoder_does_not_partially_encrypt_when_no_marked_fields( @pytest.mark.asyncio -async def test_payload_encoder_event_payload_sets_compression_metadata(): +async def test_payload_encoder_encodes_event_content_without_offloading(): + encryption_config = PayloadEncryptionConfig( + mode=PayloadEncryptionMode.PARTIAL, + main_key=SecretStr("0" * 64), + ) config = WorkflowEncodingConfig( - payload_encryption=PayloadEncryptionConfig( - mode=PayloadEncryptionMode.FULL, - main_key=SecretStr("0" * 64), - ), + payload_encryption=encryption_config, payload_compression=PayloadCompressionConfig( min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=3) ), + payload_offloading=PayloadOffloadingConfig( + min_size_bytes=1, + storage_config=BlobStorageConfig( + storage_provider=StorageProvider.S3, + bucket_name="test-bucket", + ), + ), ) encoder = PayloadEncoder(encoding_config=config) decoder = PayloadEncoder( - encoding_config=WorkflowEncodingConfig( - payload_encryption=config.payload_encryption, - ) + encoding_config=WorkflowEncodingConfig(payload_encryption=encryption_config) ) payload = json.dumps({"data": "x" * 20_000}).encode() - ( - encoded, - encoding_options, - encoding_metadata, - ) = await encoder.encode_event_payload_content(payload) + encoded, encoding_options, encoding_metadata = await encoder.encode_payload_content( + payload, + allow_offloading=False, + force_full_encryption=True, + ) decoded = await decoder.decode_payload_content( encoded, encoding_options, encoding_metadata ) @@ -581,39 +586,3 @@ async def test_payload_encoder_event_payload_sets_compression_metadata(): ] assert encoding_metadata == {"compression": {"algorithm": "zstd", "level": 3}} assert decoded == payload - - -@pytest.mark.asyncio -async def test_payload_encoder_decodes_event_payload_with_compression_metadata(): - config = WorkflowEncodingConfig( - payload_encryption=PayloadEncryptionConfig( - mode=PayloadEncryptionMode.FULL, - main_key=SecretStr("0" * 64), - ), - payload_compression=PayloadCompressionConfig( - min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=3) - ), - ) - encoder = PayloadEncoder(encoding_config=config) - decoder = PayloadEncoder( - encoding_config=WorkflowEncodingConfig( - payload_encryption=config.payload_encryption, - ) - ) - payload = {"data": "x" * 20_000} - ( - encoded, - encoding_options, - encoding_metadata, - ) = await encoder.encode_event_payload_content(json.dumps(payload).encode()) - - decoded = await decoder.decode_event_payload( - { - "type": "json", - "value": base64.b64encode(encoded).decode(), - "encoding_options": [option.value for option in encoding_options], - "encoding_metadata": encoding_metadata, - } - ) - - assert decoded == {"type": "json", "value": payload, "encoding_options": []} diff --git a/src/mistralai/extra/workflows/encoding/payload_encoder.py b/src/mistralai/extra/workflows/encoding/payload_encoder.py index bbe8b576..0ef49de2 100644 --- a/src/mistralai/extra/workflows/encoding/payload_encoder.py +++ b/src/mistralai/extra/workflows/encoding/payload_encoder.py @@ -265,7 +265,12 @@ def _decompress(self, data: bytes, encoding_metadata: dict[str, object]) -> byte return compressor_from_metadata(encoding_metadata).decompress(data) async def encode_payload_content( - self, data: Union[bytes, str], context: Optional[WorkflowContext] + self, + data: Union[bytes, str], + context: Optional[WorkflowContext] = None, + *, + allow_offloading: bool = True, + force_full_encryption: bool = False, ) -> tuple[bytes, list[EncodedPayloadOptions], dict[str, object]]: """Handle payload encoding. @@ -281,7 +286,8 @@ async def encode_payload_content( # Partial encryption needs the original JSON fields. It must run before # compression or offloading, which make field-level markers unavailable. if ( - self.encryption_config is not None + not force_full_encryption + and self.encryption_config is not None and self.encryption_config.mode == PayloadEncryptionMode.PARTIAL ): data, partially_encrypted = await self._partially_encrypt_fields(data) @@ -295,7 +301,7 @@ async def encode_payload_content( encoding_options.append(EncodedPayloadOptions.COMPRESSED) encoding_metadata.update(self._compression_metadata()) - if self.offloading_config is not None: + if allow_offloading and self.offloading_config is not None: data, offloaded = await self._handle_offloading(data, context) if offloaded: encoding_options.append(EncodedPayloadOptions.OFFLOADED) @@ -303,39 +309,6 @@ async def encode_payload_content( # Full encryption intentionally remains the final transform. If the # payload was offloaded, this encrypts the small blob-reference envelope # rather than the blob bytes. - if ( - self.encryption_config is not None - and self.encryption_config.mode == PayloadEncryptionMode.FULL - ): - data = self._encrypt(data) - encoding_options.append(EncodedPayloadOptions.ENCRYPTED) - - return data, encoding_options, encoding_metadata - - async def encode_event_payload_content( - self, data: Union[bytes, str], force_full_encryption: bool = False - ) -> tuple[bytes, list[EncodedPayloadOptions], dict[str, object]]: - """Encode event payload content without blob offloading.""" - if isinstance(data, str): - data = data.encode() - - encoding_options: list[EncodedPayloadOptions] = [] - encoding_metadata: dict[str, object] = {} - - if ( - not force_full_encryption - and self.encryption_config is not None - and self.encryption_config.mode == PayloadEncryptionMode.PARTIAL - ): - data, partially_encrypted = await self._partially_encrypt_fields(data) - if partially_encrypted: - encoding_options.append(EncodedPayloadOptions.PARTIALLY_ENCRYPTED) - - data, compressed = self._compress(data) - if compressed: - encoding_options.append(EncodedPayloadOptions.COMPRESSED) - encoding_metadata.update(self._compression_metadata()) - if self.encryption_config is not None and ( force_full_encryption or self.encryption_config.mode == PayloadEncryptionMode.FULL From c78d437284b737dc5576b57194589299ae70328c Mon Sep 17 00:00:00 2001 From: Lucas Marandat <31749711+lucasmrdt@users.noreply.github.com> Date: Wed, 20 May 2026 18:51:18 +0200 Subject: [PATCH 07/16] Keep workflow compression payloads self-describing (#534) * Wrap compressed workflow payloads * Clean up workflow compression tests * Use compressed payload helper in tests --------- Co-authored-by: Vibe Nuage Agent --- .../extra/tests/test_workflow_encoding.py | 143 +++++++++++++++--- .../workflows/encoding/payload_compressor.py | 36 ++--- .../workflows/encoding/payload_encoder.py | 71 ++++++--- 3 files changed, 173 insertions(+), 77 deletions(-) diff --git a/src/mistralai/extra/tests/test_workflow_encoding.py b/src/mistralai/extra/tests/test_workflow_encoding.py index 8bab8ef1..453a4952 100644 --- a/src/mistralai/extra/tests/test_workflow_encoding.py +++ b/src/mistralai/extra/tests/test_workflow_encoding.py @@ -30,10 +30,32 @@ NetworkEncodedInput, WorkflowContext, ) -from mistralai.extra.workflows.encoding.payload_encoder import PayloadEncoder +from mistralai.extra.workflows.encoding.payload_encoder import ( + CompressedPayloadData, + PayloadEncoder, +) from mistralai.extra.tests.fixtures.workflow_encoding import InMemoryBlobStorage +_COMPRESSED_TEST_PAYLOAD = CompressedPayloadData.from_payload( + b"compressed-data", ZstdCompressionConfig(level=3) +) + + +def _compressed_payload_json( + compressed_payload: CompressedPayloadData, + *, + invalid_compression: dict[str, object] | None = None, + invalid_base64: bool = False, +) -> bytes: + payload_data = compressed_payload.model_dump(mode="json") + if invalid_compression is not None: + payload_data["compression"] = invalid_compression + if invalid_base64: + payload_data["b64payload"] = f"{payload_data['b64payload']}!" + return json.dumps(payload_data).encode() + + @pytest.fixture def encryption_config() -> WorkflowEncodingConfig: """Create a test encryption config.""" @@ -169,15 +191,77 @@ async def test_payload_encoder_compresses_network_inputs(): ) assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] - assert encoded.encoding_metadata == { - "compression": {"algorithm": "zstd", "level": 3} - } - assert not encoded.get_payload().startswith(b"{") + compressed_payload = CompressedPayloadData.model_validate_json(encoded.get_payload()) + assert compressed_payload.compression == ZstdCompressionConfig(level=3) decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) assert decoded == payload +@pytest.mark.asyncio +async def test_payload_encoder_content_keeps_two_value_contract_for_compression(): + # Workflow workers use this low-level API directly from their Temporal codec. + # Keep compression self-describing without changing the two-value contract. + config = WorkflowEncodingConfig( + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=3) + ) + ) + encoder = PayloadEncoder(encoding_config=config) + raw = json.dumps({"data": "x" * 20_000}).encode() + + encoded_data, encoding_options = await encoder.encode_payload_content( + raw, WorkflowContext(namespace="test", execution_id="exec") + ) + + assert isinstance(encoded_data, bytes) + assert encoding_options == [EncodedPayloadOptions.COMPRESSED] + + +@pytest.mark.asyncio +async def test_payload_encoder_wraps_compression_config_in_payload_content(): + # Temporal metadata only carries encoding_options, so compressed bytes must + # include the algorithm config needed to decode them independently. + config = WorkflowEncodingConfig( + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=3) + ) + ) + encoder = PayloadEncoder(encoding_config=config) + raw = json.dumps({"data": "x" * 20_000}).encode() + + encoded_data, encoding_options = await encoder.encode_payload_content( + raw, WorkflowContext(namespace="test", execution_id="exec") + ) + compressed_payload = CompressedPayloadData.model_validate_json(encoded_data) + + assert encoding_options == [EncodedPayloadOptions.COMPRESSED] + assert compressed_payload.compression == ZstdCompressionConfig(level=3) + assert compressed_payload.get_payload() != raw + + +@pytest.mark.asyncio +async def test_payload_encoder_decodes_compressed_payload_content_without_metadata(): + # This mirrors Temporal payload decoding, where the codec passes only bytes + # plus encoding_options back into PayloadEncoder. + config = WorkflowEncodingConfig( + payload_compression=PayloadCompressionConfig( + min_size_bytes=1, algorithm_config=ZstdCompressionConfig(level=3) + ) + ) + encoder = PayloadEncoder(encoding_config=config) + raw = json.dumps({"data": "x" * 20_000}).encode() + + encoded_data, encoding_options = await encoder.encode_payload_content( + raw, WorkflowContext(namespace="test", execution_id="exec") + ) + decoded = await PayloadEncoder(WorkflowEncodingConfig()).decode_payload_content( + encoded_data, encoding_options + ) + + assert decoded == raw + + @pytest.mark.asyncio async def test_payload_encoder_skips_compression_below_min_size(): config = WorkflowEncodingConfig( @@ -292,9 +376,8 @@ async def test_payload_encoder_decodes_compressed_payload_with_decoder_config( ) assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] - assert encoded.encoding_metadata == { - "compression": {"algorithm": "zstd", "level": 22} - } + compressed_payload = CompressedPayloadData.model_validate_json(encoded.get_payload()) + assert compressed_payload.compression == ZstdCompressionConfig(level=22) assert decoded == payload @@ -355,6 +438,8 @@ async def test_payload_encoder_decodes_encrypted_compressed_payload_with_differe @pytest.mark.asyncio async def test_payload_encoder_decodes_with_tampered_compression_level(): + # Zstd decompression must depend on the frame data, not on the compression + # level that was used when the payload was encoded. encoder = PayloadEncoder( encoding_config=WorkflowEncodingConfig( payload_compression=PayloadCompressionConfig( @@ -367,10 +452,12 @@ async def test_payload_encoder_decodes_with_tampered_compression_level(): encoded = await encoder.encode_network_input( payload, WorkflowContext(namespace="test", execution_id="exec") ) + compressed_payload = CompressedPayloadData.model_validate_json(encoded.get_payload()) + tampered_payload = compressed_payload.model_copy( + update={"compression": ZstdCompressionConfig(level=1)} + ) tampered = NetworkEncodedInput.from_data( - encoded.get_payload(), - encoded.encoding_options, - {"compression": {"algorithm": "zstd", "level": 1}}, + tampered_payload.model_dump_json().encode(), encoded.encoding_options ) decoded = await PayloadEncoder(WorkflowEncodingConfig()).decode_network_result( @@ -382,18 +469,25 @@ async def test_payload_encoder_decodes_with_tampered_compression_level(): @pytest.mark.asyncio @pytest.mark.parametrize( - "encoding_metadata", + "compressed_payload", [ - {}, - {"compression": {"algorithm": "lz4", "level": 1}}, - {"compression": {"level": 3}}, + b"compressed-data", + _compressed_payload_json( + _COMPRESSED_TEST_PAYLOAD, + invalid_compression={"algorithm": "lz4", "level": 1}, + ), + _compressed_payload_json( + _COMPRESSED_TEST_PAYLOAD, + invalid_compression={"level": 3}, + ), + _compressed_payload_json(_COMPRESSED_TEST_PAYLOAD, invalid_base64=True), ], ) -async def test_payload_encoder_invalid_compression_metadata_is_error( - encoding_metadata: dict[str, object], +async def test_payload_encoder_invalid_compressed_payload_is_error( + compressed_payload: bytes, ): encoded = NetworkEncodedInput.from_data( - b"compressed-data", [EncodedPayloadOptions.COMPRESSED], encoding_metadata + compressed_payload, [EncodedPayloadOptions.COMPRESSED] ) with pytest.raises(WorkflowPayloadCompressionException): @@ -404,10 +498,12 @@ async def test_payload_encoder_invalid_compression_metadata_is_error( @pytest.mark.asyncio async def test_payload_encoder_corrupted_compressed_data_is_error(): + compressed_payload = CompressedPayloadData.from_payload( + b"corrupted-data", ZstdCompressionConfig(level=3) + ) encoded = NetworkEncodedInput.from_data( - b"corrupted-data", + compressed_payload.model_dump_json().encode(), [EncodedPayloadOptions.COMPRESSED], - {"compression": {"algorithm": "zstd", "level": 3}}, ) with pytest.raises(zstandard.ZstdError): @@ -571,18 +667,15 @@ async def test_payload_encoder_encodes_event_content_without_offloading(): ) payload = json.dumps({"data": "x" * 20_000}).encode() - encoded, encoding_options, encoding_metadata = await encoder.encode_payload_content( + encoded, encoding_options = await encoder.encode_payload_content( payload, allow_offloading=False, force_full_encryption=True, ) - decoded = await decoder.decode_payload_content( - encoded, encoding_options, encoding_metadata - ) + decoded = await decoder.decode_payload_content(encoded, encoding_options) assert encoding_options == [ EncodedPayloadOptions.COMPRESSED, EncodedPayloadOptions.ENCRYPTED, ] - assert encoding_metadata == {"compression": {"algorithm": "zstd", "level": 3}} assert decoded == payload diff --git a/src/mistralai/extra/workflows/encoding/payload_compressor.py b/src/mistralai/extra/workflows/encoding/payload_compressor.py index 2f372ac4..e8d8c578 100644 --- a/src/mistralai/extra/workflows/encoding/payload_compressor.py +++ b/src/mistralai/extra/workflows/encoding/payload_compressor.py @@ -15,7 +15,6 @@ ) _ALGORITHM_CONFIG_ADAPTER: TypeAdapter[AlgorithmConfig] = TypeAdapter(AlgorithmConfig) -_COMPRESSION_METADATA_KEY = "compression" class Compressor(ABC): @@ -60,6 +59,14 @@ def decompress(self, data: bytes) -> bytes: return result +def compressor_from_config(algo_config: AlgorithmConfig) -> Compressor: + if isinstance(algo_config, ZstdCompressionConfig): + return ZstdCompressor(algo_config) + raise WorkflowPayloadCompressionException( + f"Unsupported compression algorithm: {algo_config.algorithm!r}" + ) + + @lru_cache(maxsize=8) def _build_compressor_for_config(config_json: str) -> Compressor: try: @@ -69,11 +76,7 @@ def _build_compressor_for_config(config_json: str) -> Compressor: f"Invalid compression config in payload: {exc}" ) from exc - if isinstance(algo_config, ZstdCompressionConfig): - return ZstdCompressor(algo_config) - raise WorkflowPayloadCompressionException( - f"Unsupported compression algorithm: {algo_config.algorithm!r}" - ) + return compressor_from_config(algo_config) def build_compressor( @@ -84,24 +87,3 @@ def build_compressor( return _build_compressor_for_config( compression_config.algorithm_config.model_dump_json() ) - - -def compression_metadata(compressor: Compressor) -> dict[str, object]: - return { - _COMPRESSION_METADATA_KEY: compressor.algorithm_config.model_dump(mode="json") - } - - -def compressor_from_metadata(metadata: dict[str, object]) -> Compressor: - config = metadata.get(_COMPRESSION_METADATA_KEY) - if not isinstance(config, dict): - raise WorkflowPayloadCompressionException( - "Missing compression config in payload metadata" - ) - try: - algo_config = _ALGORITHM_CONFIG_ADAPTER.validate_python(config) - except ValidationError as exc: - raise WorkflowPayloadCompressionException( - f"Invalid compression config in payload metadata: {exc}" - ) from exc - return _build_compressor_for_config(algo_config.model_dump_json()) diff --git a/src/mistralai/extra/workflows/encoding/payload_encoder.py b/src/mistralai/extra/workflows/encoding/payload_encoder.py index 0ef49de2..55ae68de 100644 --- a/src/mistralai/extra/workflows/encoding/payload_encoder.py +++ b/src/mistralai/extra/workflows/encoding/payload_encoder.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import binascii import functools import hashlib import json @@ -9,7 +10,7 @@ import urllib.parse from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union -from pydantic import BaseModel +from pydantic import BaseModel, ValidationError if TYPE_CHECKING: from cryptography.exceptions import InvalidTag @@ -25,6 +26,7 @@ from pydantic_core import from_json, to_json from mistralai.extra.workflows.encoding.config import ( + AlgorithmConfig, PayloadEncryptionConfig, PayloadEncryptionMode, PayloadOffloadingConfig, @@ -32,8 +34,7 @@ ) from mistralai.extra.workflows.encoding.payload_compressor import ( build_compressor, - compression_metadata, - compressor_from_metadata, + compressor_from_config, ) from .storage.blob_storage import get_blob_storage, BlobNotFoundError from mistralai.extra.workflows.encoding.models import ( @@ -44,6 +45,7 @@ WorkflowContext, ) from mistralai.extra.exceptions import ( + WorkflowPayloadCompressionException, WorkflowPayloadEncryptionException, WorkflowPayloadOffloadingException, ) @@ -55,6 +57,28 @@ class OffloadedPayloadData(BaseModel): key: str +class CompressedPayloadData(BaseModel): + compression: AlgorithmConfig + b64payload: str + + @classmethod + def from_payload( + cls, payload: bytes, compression: AlgorithmConfig + ) -> "CompressedPayloadData": + return cls( + compression=compression, + b64payload=base64.b64encode(payload).decode("utf-8"), + ) + + def get_payload(self) -> bytes: + try: + return base64.b64decode(self.b64payload, validate=True) + except binascii.Error as exc: + raise WorkflowPayloadCompressionException( + "Invalid compressed payload data" + ) from exc + + class PayloadEncoder: """This class is in charge of payload encoding/decoding operations such as: - Field-level or full-payload encryption @@ -252,17 +276,21 @@ def _compress(self, data: bytes) -> tuple[bytes, bool]: compressed = self.compressor.compress(data) if len(compressed) >= len(data): return data, False - return compressed, True - - def _compression_metadata(self) -> dict[str, object]: - assert self.compressor is not None, ( - "This should never be reached: compression metadata was requested " - "but PayloadEncoder.__init__ did not build a compressor" + compressed_payload = CompressedPayloadData.from_payload( + compressed, self.compressor.algorithm_config ) - return compression_metadata(self.compressor) + return compressed_payload.model_dump_json().encode(), True - def _decompress(self, data: bytes, encoding_metadata: dict[str, object]) -> bytes: - return compressor_from_metadata(encoding_metadata).decompress(data) + def _decompress(self, data: bytes) -> bytes: + try: + compressed_payload = CompressedPayloadData.model_validate_json(data) + except ValidationError as exc: + raise WorkflowPayloadCompressionException( + "Invalid compressed payload metadata" + ) from exc + return compressor_from_config(compressed_payload.compression).decompress( + compressed_payload.get_payload() + ) async def encode_payload_content( self, @@ -271,7 +299,7 @@ async def encode_payload_content( *, allow_offloading: bool = True, force_full_encryption: bool = False, - ) -> tuple[bytes, list[EncodedPayloadOptions], dict[str, object]]: + ) -> tuple[bytes, list[EncodedPayloadOptions]]: """Handle payload encoding. Encoding options are appended in the exact order in which transforms are @@ -281,7 +309,6 @@ async def encode_payload_content( data = data.encode() encoding_options: list[EncodedPayloadOptions] = [] - encoding_metadata: dict[str, object] = {} # Partial encryption needs the original JSON fields. It must run before # compression or offloading, which make field-level markers unavailable. @@ -299,7 +326,6 @@ async def encode_payload_content( data, compressed = self._compress(data) if compressed: encoding_options.append(EncodedPayloadOptions.COMPRESSED) - encoding_metadata.update(self._compression_metadata()) if allow_offloading and self.offloading_config is not None: data, offloaded = await self._handle_offloading(data, context) @@ -316,7 +342,7 @@ async def encode_payload_content( data = self._encrypt(data) encoding_options.append(EncodedPayloadOptions.ENCRYPTED) - return data, encoding_options, encoding_metadata + return data, encoding_options async def decode_payload_content( self, @@ -324,7 +350,6 @@ async def decode_payload_content( encoding_options: List[EncodedPayloadOptions], encoding_metadata: dict[str, object] | None = None, ) -> bytes: - encoding_metadata = encoding_metadata or {} # Decode in the exact reverse order of the encoding_options wire list. for option in reversed(encoding_options): if option == EncodedPayloadOptions.ENCRYPTED: @@ -332,7 +357,7 @@ async def decode_payload_content( elif option == EncodedPayloadOptions.PARTIALLY_ENCRYPTED: data, _ = await self._partially_decrypt_fields(data) elif option == EncodedPayloadOptions.COMPRESSED: - data = self._decompress(data, encoding_metadata) + data = self._decompress(data) elif option == EncodedPayloadOptions.OFFLOADED: if ( self.offloading_config is None @@ -391,14 +416,10 @@ async def encode_network_input( """This method MUST be called to format every payload send to Mistral Workflows control plane to ensure a proper encoding of the payload. """ - ( - encoded_data, - encoding_options, - encoding_metadata, - ) = await self.encode_payload_content(to_json(data), context) - network_input = NetworkEncodedInput.from_data( - encoded_data, encoding_options, encoding_metadata + encoded_data, encoding_options = await self.encode_payload_content( + to_json(data), context ) + network_input = NetworkEncodedInput.from_data(encoded_data, encoding_options) return network_input async def decode_network_result(self, data: Any) -> Any: From 8924e8c26be47d045470daae28539c00108bf001 Mon Sep 17 00:00:00 2001 From: Vibe Nuage Agent Date: Wed, 20 May 2026 17:52:14 +0000 Subject: [PATCH 08/16] temp: default payload compression on for staging preview TEMPORARY: enable PayloadCompressionConfig by default in WorkflowEncodingConfig so compression is active without env var wiring. Revert once validated and proper WorkerConfig plumbing is in. --- src/mistralai/extra/workflows/encoding/config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/mistralai/extra/workflows/encoding/config.py b/src/mistralai/extra/workflows/encoding/config.py index ccb73d28..1b9cb7dc 100644 --- a/src/mistralai/extra/workflows/encoding/config.py +++ b/src/mistralai/extra/workflows/encoding/config.py @@ -66,4 +66,8 @@ class PayloadCompressionConfig(BaseModel): class WorkflowEncodingConfig(BaseModel): payload_offloading: PayloadOffloadingConfig | None = None payload_encryption: PayloadEncryptionConfig | None = None - payload_compression: PayloadCompressionConfig | None = None + # TEMPORARY: default compression on to validate in staging preview. + # TODO: revert to None and wire through WorkerConfig env vars once validated. + payload_compression: PayloadCompressionConfig | None = Field( + default_factory=PayloadCompressionConfig + ) From 00b665f7405c97214df1e9e6a53766f98c75c189 Mon Sep 17 00:00:00 2001 From: Vibe Nuage Agent Date: Thu, 21 May 2026 08:27:30 +0000 Subject: [PATCH 09/16] Lower default compression threshold from 1MB to 1KB --- .../extra/tests/test_workflow_encoding.py | 16 ++++++++++++---- src/mistralai/extra/workflows/encoding/config.py | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/mistralai/extra/tests/test_workflow_encoding.py b/src/mistralai/extra/tests/test_workflow_encoding.py index 453a4952..70360bf2 100644 --- a/src/mistralai/extra/tests/test_workflow_encoding.py +++ b/src/mistralai/extra/tests/test_workflow_encoding.py @@ -191,7 +191,9 @@ async def test_payload_encoder_compresses_network_inputs(): ) assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] - compressed_payload = CompressedPayloadData.model_validate_json(encoded.get_payload()) + compressed_payload = CompressedPayloadData.model_validate_json( + encoded.get_payload() + ) assert compressed_payload.compression == ZstdCompressionConfig(level=3) decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) @@ -300,7 +302,9 @@ async def test_payload_encoder_skips_compression_when_not_smaller(): @pytest.mark.asyncio async def test_payload_encoder_skips_compression_without_config(): - encoder = PayloadEncoder(encoding_config=WorkflowEncodingConfig()) + encoder = PayloadEncoder( + encoding_config=WorkflowEncodingConfig(payload_compression=None) + ) payload = {"data": "x" * 20_000} encoded = await encoder.encode_network_input( @@ -376,7 +380,9 @@ async def test_payload_encoder_decodes_compressed_payload_with_decoder_config( ) assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] - compressed_payload = CompressedPayloadData.model_validate_json(encoded.get_payload()) + compressed_payload = CompressedPayloadData.model_validate_json( + encoded.get_payload() + ) assert compressed_payload.compression == ZstdCompressionConfig(level=22) assert decoded == payload @@ -452,7 +458,9 @@ async def test_payload_encoder_decodes_with_tampered_compression_level(): encoded = await encoder.encode_network_input( payload, WorkflowContext(namespace="test", execution_id="exec") ) - compressed_payload = CompressedPayloadData.model_validate_json(encoded.get_payload()) + compressed_payload = CompressedPayloadData.model_validate_json( + encoded.get_payload() + ) tampered_payload = compressed_payload.model_copy( update={"compression": ZstdCompressionConfig(level=1)} ) diff --git a/src/mistralai/extra/workflows/encoding/config.py b/src/mistralai/extra/workflows/encoding/config.py index 1b9cb7dc..ba52ab14 100644 --- a/src/mistralai/extra/workflows/encoding/config.py +++ b/src/mistralai/extra/workflows/encoding/config.py @@ -59,7 +59,7 @@ class ZstdCompressionConfig(BaseModel): class PayloadCompressionConfig(BaseModel): - min_size_bytes: int = 1024 * 1024 # 1MB + min_size_bytes: int = 1024 # 1KB algorithm_config: AlgorithmConfig = Field(default_factory=ZstdCompressionConfig) From 45a8cfbe6074bfb0aa952411516e004806679d31 Mon Sep 17 00:00:00 2001 From: Vibe Nuage Agent Date: Thu, 21 May 2026 09:13:27 +0000 Subject: [PATCH 10/16] Revert compression threshold back to 1MB --- src/mistralai/extra/workflows/encoding/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mistralai/extra/workflows/encoding/config.py b/src/mistralai/extra/workflows/encoding/config.py index ba52ab14..1b9cb7dc 100644 --- a/src/mistralai/extra/workflows/encoding/config.py +++ b/src/mistralai/extra/workflows/encoding/config.py @@ -59,7 +59,7 @@ class ZstdCompressionConfig(BaseModel): class PayloadCompressionConfig(BaseModel): - min_size_bytes: int = 1024 # 1KB + min_size_bytes: int = 1024 * 1024 # 1MB algorithm_config: AlgorithmConfig = Field(default_factory=ZstdCompressionConfig) From 37d843a12cbdab2670e27544e2a1837717062d6a Mon Sep 17 00:00:00 2001 From: Lucas Marandat <31749711+lucasmrdt@users.noreply.github.com> Date: Thu, 21 May 2026 18:54:33 +0200 Subject: [PATCH 11/16] Use msgpack for compressed workflow payloads (#546) * Use msgpack for compressed workflow payloads * Merge _require_msgpack with _require_zstandard pattern - Move _require_msgpack from payload_encoder.py to payload_compressor.py - Use import_module from importlib instead of __import__ - Return ModuleType instead of Any for consistency - Keep compression-related imports together in one place This consolidates the module import helpers for compression libraries (zstandard and msgpack) in a single location for better maintainability. * Make msgpack a core dependency --------- Co-authored-by: Vibe Nuage Agent --- pyproject.toml | 2 + .../extra/tests/test_workflow_encoding.py | 43 ++++++------ .../extra/workflows/encoding/models.py | 15 +---- .../workflows/encoding/payload_encoder.py | 46 +++++++------ uv.lock | 67 +++++++++++++++++++ 5 files changed, 119 insertions(+), 54 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 31a9e029..08232464 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "opentelemetry-api (>=1.33.1,<2.0.0)", "opentelemetry-semantic-conventions (>=0.60b1,<0.61)", "jsonpath-python >=1.0.6", # required for speakeasy generated path with pagination + "msgpack>=1.1.0,<2.0.0", ] [project.optional-dependencies] @@ -141,6 +142,7 @@ ignore_missing_imports = true [[tool.mypy.overrides]] module = [ "jsonpath.*", + "msgpack.*", "typing_inspect.*", "authlib.*", "websockets.*", diff --git a/src/mistralai/extra/tests/test_workflow_encoding.py b/src/mistralai/extra/tests/test_workflow_encoding.py index 70360bf2..94ce2d65 100644 --- a/src/mistralai/extra/tests/test_workflow_encoding.py +++ b/src/mistralai/extra/tests/test_workflow_encoding.py @@ -3,6 +3,7 @@ import gc import json +import msgpack import pytest import zstandard from pydantic import SecretStr @@ -42,18 +43,21 @@ ) -def _compressed_payload_json( +def _compressed_payload_msgpack( compressed_payload: CompressedPayloadData, *, invalid_compression: dict[str, object] | None = None, - invalid_base64: bool = False, + invalid_payload: object | None = None, ) -> bytes: - payload_data = compressed_payload.model_dump(mode="json") + payload_data: dict[str, object] = { + "compression": compressed_payload.compression.model_dump(mode="json"), + "payload": compressed_payload.payload, + } if invalid_compression is not None: payload_data["compression"] = invalid_compression - if invalid_base64: - payload_data["b64payload"] = f"{payload_data['b64payload']}!" - return json.dumps(payload_data).encode() + if invalid_payload is not None: + payload_data["payload"] = invalid_payload + return msgpack.packb(payload_data, use_bin_type=True) @pytest.fixture @@ -191,9 +195,7 @@ async def test_payload_encoder_compresses_network_inputs(): ) assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] - compressed_payload = CompressedPayloadData.model_validate_json( - encoded.get_payload() - ) + compressed_payload = CompressedPayloadData.from_msgpack(encoded.get_payload()) assert compressed_payload.compression == ZstdCompressionConfig(level=3) decoded = await encoder.decode_network_result(encoded.model_dump(mode="json")) @@ -235,7 +237,7 @@ async def test_payload_encoder_wraps_compression_config_in_payload_content(): encoded_data, encoding_options = await encoder.encode_payload_content( raw, WorkflowContext(namespace="test", execution_id="exec") ) - compressed_payload = CompressedPayloadData.model_validate_json(encoded_data) + compressed_payload = CompressedPayloadData.from_msgpack(encoded_data) assert encoding_options == [EncodedPayloadOptions.COMPRESSED] assert compressed_payload.compression == ZstdCompressionConfig(level=3) @@ -380,9 +382,7 @@ async def test_payload_encoder_decodes_compressed_payload_with_decoder_config( ) assert encoded.encoding_options == [EncodedPayloadOptions.COMPRESSED] - compressed_payload = CompressedPayloadData.model_validate_json( - encoded.get_payload() - ) + compressed_payload = CompressedPayloadData.from_msgpack(encoded.get_payload()) assert compressed_payload.compression == ZstdCompressionConfig(level=22) assert decoded == payload @@ -458,14 +458,12 @@ async def test_payload_encoder_decodes_with_tampered_compression_level(): encoded = await encoder.encode_network_input( payload, WorkflowContext(namespace="test", execution_id="exec") ) - compressed_payload = CompressedPayloadData.model_validate_json( - encoded.get_payload() - ) + compressed_payload = CompressedPayloadData.from_msgpack(encoded.get_payload()) tampered_payload = compressed_payload.model_copy( update={"compression": ZstdCompressionConfig(level=1)} ) tampered = NetworkEncodedInput.from_data( - tampered_payload.model_dump_json().encode(), encoded.encoding_options + tampered_payload.to_msgpack(), encoded.encoding_options ) decoded = await PayloadEncoder(WorkflowEncodingConfig()).decode_network_result( @@ -480,15 +478,18 @@ async def test_payload_encoder_decodes_with_tampered_compression_level(): "compressed_payload", [ b"compressed-data", - _compressed_payload_json( + _compressed_payload_msgpack( _COMPRESSED_TEST_PAYLOAD, invalid_compression={"algorithm": "lz4", "level": 1}, ), - _compressed_payload_json( + _compressed_payload_msgpack( _COMPRESSED_TEST_PAYLOAD, invalid_compression={"level": 3}, ), - _compressed_payload_json(_COMPRESSED_TEST_PAYLOAD, invalid_base64=True), + _compressed_payload_msgpack( + _COMPRESSED_TEST_PAYLOAD, + invalid_payload=123, + ), ], ) async def test_payload_encoder_invalid_compressed_payload_is_error( @@ -510,7 +511,7 @@ async def test_payload_encoder_corrupted_compressed_data_is_error(): b"corrupted-data", ZstdCompressionConfig(level=3) ) encoded = NetworkEncodedInput.from_data( - compressed_payload.model_dump_json().encode(), + compressed_payload.to_msgpack(), [EncodedPayloadOptions.COMPRESSED], ) diff --git a/src/mistralai/extra/workflows/encoding/models.py b/src/mistralai/extra/workflows/encoding/models.py index 984401c3..1f1c8546 100644 --- a/src/mistralai/extra/workflows/encoding/models.py +++ b/src/mistralai/extra/workflows/encoding/models.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, Literal, Optional +from typing import Literal, Optional import base64 from pydantic import BaseModel, Field @@ -36,10 +36,6 @@ class EncodedPayload(BaseModel): encoding_options: list[EncodedPayloadOptions] = Field( description="The encoding of the payload", default=[] ) - encoding_metadata: dict[str, Any] = Field( - description="Additional metadata required to decode the payload", - default_factory=dict, - ) payload: bytes = Field(description="The encoded payload") @@ -48,10 +44,6 @@ class NetworkEncodedBase(BaseModel): encoding_options: list[EncodedPayloadOptions] = Field( description="The encoding of the payload", default=[] ) - encoding_metadata: dict[str, Any] = Field( - description="Additional metadata required to decode the payload", - default_factory=dict, - ) def get_payload(self) -> bytes: return base64.b64decode(self.b64payload) @@ -66,7 +58,6 @@ def to_encoded_payload( return EncodedPayload( payload=base64.b64decode(self.b64payload), encoding_options=self.encoding_options, - encoding_metadata=self.encoding_metadata, context=WorkflowContext( namespace=namespace, execution_id=execution_id, @@ -79,19 +70,16 @@ def from_encoded_payload(encoded_payload: EncodedPayload) -> "NetworkEncodedInpu return NetworkEncodedInput( b64payload=base64.b64encode(encoded_payload.payload).decode("utf-8"), encoding_options=encoded_payload.encoding_options, - encoding_metadata=encoded_payload.encoding_metadata, ) @staticmethod def from_data( data: bytes, encoding_options: list[EncodedPayloadOptions], - encoding_metadata: dict[str, Any] | None = None, ) -> "NetworkEncodedInput": return NetworkEncodedInput( b64payload=base64.b64encode(data).decode("utf-8"), encoding_options=encoding_options, - encoding_metadata=encoding_metadata or {}, ) @@ -101,5 +89,4 @@ def from_encoded_payload(encoded_payload: EncodedPayload) -> "NetworkEncodedResu return NetworkEncodedResult( b64payload=base64.b64encode(encoded_payload.payload).decode("utf-8"), encoding_options=encoded_payload.encoding_options, - encoding_metadata=encoded_payload.encoding_metadata, ) diff --git a/src/mistralai/extra/workflows/encoding/payload_encoder.py b/src/mistralai/extra/workflows/encoding/payload_encoder.py index 023d7e4e..7d147900 100644 --- a/src/mistralai/extra/workflows/encoding/payload_encoder.py +++ b/src/mistralai/extra/workflows/encoding/payload_encoder.py @@ -1,7 +1,6 @@ from __future__ import annotations import base64 -import binascii import functools import hashlib import json @@ -10,6 +9,7 @@ import urllib.parse from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +import msgpack from pydantic import BaseModel, ValidationError if TYPE_CHECKING: @@ -60,24 +60,40 @@ class OffloadedPayloadData(BaseModel): class CompressedPayloadData(BaseModel): compression: AlgorithmConfig - b64payload: str + payload: bytes @classmethod def from_payload( cls, payload: bytes, compression: AlgorithmConfig ) -> "CompressedPayloadData": - return cls( - compression=compression, - b64payload=base64.b64encode(payload).decode("utf-8"), - ) + return cls(compression=compression, payload=payload) - def get_payload(self) -> bytes: + @classmethod + def from_msgpack(cls, data: bytes) -> "CompressedPayloadData": try: - return base64.b64decode(self.b64payload, validate=True) - except binascii.Error as exc: + unpacked = msgpack.unpackb(data, raw=False) + except Exception as exc: raise WorkflowPayloadCompressionException( "Invalid compressed payload data" ) from exc + try: + return cls.model_validate(unpacked) + except ValidationError as exc: + raise WorkflowPayloadCompressionException( + "Invalid compressed payload metadata" + ) from exc + + def to_msgpack(self) -> bytes: + return msgpack.packb( + { + "compression": self.compression.model_dump(mode="json"), + "payload": self.payload, + }, + use_bin_type=True, + ) + + def get_payload(self) -> bytes: + return self.payload class PayloadEncoder: @@ -280,15 +296,10 @@ def _compress(self, data: bytes) -> tuple[bytes, bool]: compressed_payload = CompressedPayloadData.from_payload( compressed, self.compressor.algorithm_config ) - return compressed_payload.model_dump_json().encode(), True + return compressed_payload.to_msgpack(), True def _decompress(self, data: bytes) -> bytes: - try: - compressed_payload = CompressedPayloadData.model_validate_json(data) - except ValidationError as exc: - raise WorkflowPayloadCompressionException( - "Invalid compressed payload metadata" - ) from exc + compressed_payload = CompressedPayloadData.from_msgpack(data) return compressor_from_config(compressed_payload.compression).decompress( compressed_payload.get_payload() ) @@ -359,7 +370,6 @@ async def decode_payload_content( self, data: bytes, encoding_options: List[EncodedPayloadOptions], - encoding_metadata: dict[str, object] | None = None, ) -> bytes: # Decode in the exact reverse order of the encoding_options wire list. for option in reversed(encoding_options): @@ -429,7 +439,6 @@ async def decode_event_payload( decrypted_bytes = await self.decode_payload_content( encrypted_bytes, encoding_options, - payload_data.get("encoding_metadata", {}), ) decrypted_value = json.loads(decrypted_bytes) @@ -492,7 +501,6 @@ async def decode_network_result(self, data: Any) -> Any: byte_results = await self.decode_payload_content( network_encoded_payload.get_payload(), network_encoded_payload.encoding_options, - network_encoded_payload.encoding_metadata, ) try: return from_json(byte_results) diff --git a/uv.lock b/uv.lock index 84476985..9416e89b 100644 --- a/uv.lock +++ b/uv.lock @@ -7,6 +7,10 @@ resolution-markers = [ "python_full_version < '3.11'", ] +[options] +exclude-newer = "2026-05-14T16:51:10.662539415Z" +exclude-newer-span = "P7D" + [[package]] name = "aioboto3" version = "12.4.0" @@ -1037,6 +1041,7 @@ dependencies = [ { name = "eval-type-backport" }, { name = "httpx" }, { name = "jsonpath-python" }, + { name = "msgpack" }, { name = "opentelemetry-api" }, { name = "opentelemetry-semantic-conventions" }, { name = "pydantic" }, @@ -1121,6 +1126,7 @@ requires-dist = [ { name = "mistralai", extras = ["workflow-payload-offloading-azure"], marker = "extra == 'workflow-payload-offloading'" }, { name = "mistralai", extras = ["workflow-payload-offloading-gcs"], marker = "extra == 'workflow-payload-offloading'" }, { name = "mistralai", extras = ["workflow-payload-offloading-s3"], marker = "extra == 'workflow-payload-offloading'" }, + { name = "msgpack", specifier = ">=1.1.0,<2.0.0" }, { name = "opentelemetry-api", specifier = ">=1.33.1,<2.0.0" }, { name = "opentelemetry-semantic-conventions", specifier = ">=0.60b1,<0.61" }, { name = "pydantic", specifier = ">=2.11.2" }, @@ -1182,6 +1188,67 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/75/bd9b7bb966668920f06b200e84454c8f3566b102183bc55c5473d96cb2b9/msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca", size = 20583, upload-time = "2025-03-14T23:51:03.016Z" }, ] +[[package]] +name = "msgpack" +version = "1.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/f2/bfb55a6236ed8725a96b0aa3acbd0ec17588e6a2c3b62a93eb513ed8783f/msgpack-1.1.2.tar.gz", hash = "sha256:3b60763c1373dd60f398488069bcdc703cd08a711477b5d480eecc9f9626f47e", size = 173581, upload-time = "2025-10-08T09:15:56.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/a2/3b68a9e769db68668b25c6108444a35f9bd163bb848c0650d516761a59c0/msgpack-1.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0051fffef5a37ca2cd16978ae4f0aef92f164df86823871b5162812bebecd8e2", size = 81318, upload-time = "2025-10-08T09:14:38.722Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e1/2b720cc341325c00be44e1ed59e7cfeae2678329fbf5aa68f5bda57fe728/msgpack-1.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a605409040f2da88676e9c9e5853b3449ba8011973616189ea5ee55ddbc5bc87", size = 83786, upload-time = "2025-10-08T09:14:40.082Z" }, + { url = "https://files.pythonhosted.org/packages/71/e5/c2241de64bfceac456b140737812a2ab310b10538a7b34a1d393b748e095/msgpack-1.1.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b696e83c9f1532b4af884045ba7f3aa741a63b2bc22617293a2c6a7c645f251", size = 398240, upload-time = "2025-10-08T09:14:41.151Z" }, + { url = "https://files.pythonhosted.org/packages/b7/09/2a06956383c0fdebaef5aa9246e2356776f12ea6f2a44bd1368abf0e46c4/msgpack-1.1.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:365c0bbe981a27d8932da71af63ef86acc59ed5c01ad929e09a0b88c6294e28a", size = 406070, upload-time = "2025-10-08T09:14:42.821Z" }, + { url = "https://files.pythonhosted.org/packages/0e/74/2957703f0e1ef20637d6aead4fbb314330c26f39aa046b348c7edcf6ca6b/msgpack-1.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:41d1a5d875680166d3ac5c38573896453bbbea7092936d2e107214daf43b1d4f", size = 393403, upload-time = "2025-10-08T09:14:44.38Z" }, + { url = "https://files.pythonhosted.org/packages/a5/09/3bfc12aa90f77b37322fc33e7a8a7c29ba7c8edeadfa27664451801b9860/msgpack-1.1.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:354e81bcdebaab427c3df4281187edc765d5d76bfb3a7c125af9da7a27e8458f", size = 398947, upload-time = "2025-10-08T09:14:45.56Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4f/05fcebd3b4977cb3d840f7ef6b77c51f8582086de5e642f3fefee35c86fc/msgpack-1.1.2-cp310-cp310-win32.whl", hash = "sha256:e64c8d2f5e5d5fda7b842f55dec6133260ea8f53c4257d64494c534f306bf7a9", size = 64769, upload-time = "2025-10-08T09:14:47.334Z" }, + { url = "https://files.pythonhosted.org/packages/d0/3e/b4547e3a34210956382eed1c85935fff7e0f9b98be3106b3745d7dec9c5e/msgpack-1.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:db6192777d943bdaaafb6ba66d44bf65aa0e9c5616fa1d2da9bb08828c6b39aa", size = 71293, upload-time = "2025-10-08T09:14:48.665Z" }, + { url = "https://files.pythonhosted.org/packages/2c/97/560d11202bcd537abca693fd85d81cebe2107ba17301de42b01ac1677b69/msgpack-1.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2e86a607e558d22985d856948c12a3fa7b42efad264dca8a3ebbcfa2735d786c", size = 82271, upload-time = "2025-10-08T09:14:49.967Z" }, + { url = "https://files.pythonhosted.org/packages/83/04/28a41024ccbd67467380b6fb440ae916c1e4f25e2cd4c63abe6835ac566e/msgpack-1.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:283ae72fc89da59aa004ba147e8fc2f766647b1251500182fac0350d8af299c0", size = 84914, upload-time = "2025-10-08T09:14:50.958Z" }, + { url = "https://files.pythonhosted.org/packages/71/46/b817349db6886d79e57a966346cf0902a426375aadc1e8e7a86a75e22f19/msgpack-1.1.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61c8aa3bd513d87c72ed0b37b53dd5c5a0f58f2ff9f26e1555d3bd7948fb7296", size = 416962, upload-time = "2025-10-08T09:14:51.997Z" }, + { url = "https://files.pythonhosted.org/packages/da/e0/6cc2e852837cd6086fe7d8406af4294e66827a60a4cf60b86575a4a65ca8/msgpack-1.1.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:454e29e186285d2ebe65be34629fa0e8605202c60fbc7c4c650ccd41870896ef", size = 426183, upload-time = "2025-10-08T09:14:53.477Z" }, + { url = "https://files.pythonhosted.org/packages/25/98/6a19f030b3d2ea906696cedd1eb251708e50a5891d0978b012cb6107234c/msgpack-1.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7bc8813f88417599564fafa59fd6f95be417179f76b40325b500b3c98409757c", size = 411454, upload-time = "2025-10-08T09:14:54.648Z" }, + { url = "https://files.pythonhosted.org/packages/b7/cd/9098fcb6adb32187a70b7ecaabf6339da50553351558f37600e53a4a2a23/msgpack-1.1.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bafca952dc13907bdfdedfc6a5f579bf4f292bdd506fadb38389afa3ac5b208e", size = 422341, upload-time = "2025-10-08T09:14:56.328Z" }, + { url = "https://files.pythonhosted.org/packages/e6/ae/270cecbcf36c1dc85ec086b33a51a4d7d08fc4f404bdbc15b582255d05ff/msgpack-1.1.2-cp311-cp311-win32.whl", hash = "sha256:602b6740e95ffc55bfb078172d279de3773d7b7db1f703b2f1323566b878b90e", size = 64747, upload-time = "2025-10-08T09:14:57.882Z" }, + { url = "https://files.pythonhosted.org/packages/2a/79/309d0e637f6f37e83c711f547308b91af02b72d2326ddd860b966080ef29/msgpack-1.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:d198d275222dc54244bf3327eb8cbe00307d220241d9cec4d306d49a44e85f68", size = 71633, upload-time = "2025-10-08T09:14:59.177Z" }, + { url = "https://files.pythonhosted.org/packages/73/4d/7c4e2b3d9b1106cd0aa6cb56cc57c6267f59fa8bfab7d91df5adc802c847/msgpack-1.1.2-cp311-cp311-win_arm64.whl", hash = "sha256:86f8136dfa5c116365a8a651a7d7484b65b13339731dd6faebb9a0242151c406", size = 64755, upload-time = "2025-10-08T09:15:00.48Z" }, + { url = "https://files.pythonhosted.org/packages/ad/bd/8b0d01c756203fbab65d265859749860682ccd2a59594609aeec3a144efa/msgpack-1.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:70a0dff9d1f8da25179ffcf880e10cf1aad55fdb63cd59c9a49a1b82290062aa", size = 81939, upload-time = "2025-10-08T09:15:01.472Z" }, + { url = "https://files.pythonhosted.org/packages/34/68/ba4f155f793a74c1483d4bdef136e1023f7bcba557f0db4ef3db3c665cf1/msgpack-1.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:446abdd8b94b55c800ac34b102dffd2f6aa0ce643c55dfc017ad89347db3dbdb", size = 85064, upload-time = "2025-10-08T09:15:03.764Z" }, + { url = "https://files.pythonhosted.org/packages/f2/60/a064b0345fc36c4c3d2c743c82d9100c40388d77f0b48b2f04d6041dbec1/msgpack-1.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c63eea553c69ab05b6747901b97d620bb2a690633c77f23feb0c6a947a8a7b8f", size = 417131, upload-time = "2025-10-08T09:15:05.136Z" }, + { url = "https://files.pythonhosted.org/packages/65/92/a5100f7185a800a5d29f8d14041f61475b9de465ffcc0f3b9fba606e4505/msgpack-1.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:372839311ccf6bdaf39b00b61288e0557916c3729529b301c52c2d88842add42", size = 427556, upload-time = "2025-10-08T09:15:06.837Z" }, + { url = "https://files.pythonhosted.org/packages/f5/87/ffe21d1bf7d9991354ad93949286f643b2bb6ddbeab66373922b44c3b8cc/msgpack-1.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2929af52106ca73fcb28576218476ffbb531a036c2adbcf54a3664de124303e9", size = 404920, upload-time = "2025-10-08T09:15:08.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/41/8543ed2b8604f7c0d89ce066f42007faac1eaa7d79a81555f206a5cdb889/msgpack-1.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:be52a8fc79e45b0364210eef5234a7cf8d330836d0a64dfbb878efa903d84620", size = 415013, upload-time = "2025-10-08T09:15:09.83Z" }, + { url = "https://files.pythonhosted.org/packages/41/0d/2ddfaa8b7e1cee6c490d46cb0a39742b19e2481600a7a0e96537e9c22f43/msgpack-1.1.2-cp312-cp312-win32.whl", hash = "sha256:1fff3d825d7859ac888b0fbda39a42d59193543920eda9d9bea44d958a878029", size = 65096, upload-time = "2025-10-08T09:15:11.11Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ec/d431eb7941fb55a31dd6ca3404d41fbb52d99172df2e7707754488390910/msgpack-1.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:1de460f0403172cff81169a30b9a92b260cb809c4cb7e2fc79ae8d0510c78b6b", size = 72708, upload-time = "2025-10-08T09:15:12.554Z" }, + { url = "https://files.pythonhosted.org/packages/c5/31/5b1a1f70eb0e87d1678e9624908f86317787b536060641d6798e3cf70ace/msgpack-1.1.2-cp312-cp312-win_arm64.whl", hash = "sha256:be5980f3ee0e6bd44f3a9e9dea01054f175b50c3e6cdb692bc9424c0bbb8bf69", size = 64119, upload-time = "2025-10-08T09:15:13.589Z" }, + { url = "https://files.pythonhosted.org/packages/6b/31/b46518ecc604d7edf3a4f94cb3bf021fc62aa301f0cb849936968164ef23/msgpack-1.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4efd7b5979ccb539c221a4c4e16aac1a533efc97f3b759bb5a5ac9f6d10383bf", size = 81212, upload-time = "2025-10-08T09:15:14.552Z" }, + { url = "https://files.pythonhosted.org/packages/92/dc/c385f38f2c2433333345a82926c6bfa5ecfff3ef787201614317b58dd8be/msgpack-1.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:42eefe2c3e2af97ed470eec850facbe1b5ad1d6eacdbadc42ec98e7dcf68b4b7", size = 84315, upload-time = "2025-10-08T09:15:15.543Z" }, + { url = "https://files.pythonhosted.org/packages/d3/68/93180dce57f684a61a88a45ed13047558ded2be46f03acb8dec6d7c513af/msgpack-1.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1fdf7d83102bf09e7ce3357de96c59b627395352a4024f6e2458501f158bf999", size = 412721, upload-time = "2025-10-08T09:15:16.567Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ba/459f18c16f2b3fc1a1ca871f72f07d70c07bf768ad0a507a698b8052ac58/msgpack-1.1.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fac4be746328f90caa3cd4bc67e6fe36ca2bf61d5c6eb6d895b6527e3f05071e", size = 424657, upload-time = "2025-10-08T09:15:17.825Z" }, + { url = "https://files.pythonhosted.org/packages/38/f8/4398c46863b093252fe67368b44edc6c13b17f4e6b0e4929dbf0bdb13f23/msgpack-1.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fffee09044073e69f2bad787071aeec727183e7580443dfeb8556cbf1978d162", size = 402668, upload-time = "2025-10-08T09:15:19.003Z" }, + { url = "https://files.pythonhosted.org/packages/28/ce/698c1eff75626e4124b4d78e21cca0b4cc90043afb80a507626ea354ab52/msgpack-1.1.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5928604de9b032bc17f5099496417f113c45bc6bc21b5c6920caf34b3c428794", size = 419040, upload-time = "2025-10-08T09:15:20.183Z" }, + { url = "https://files.pythonhosted.org/packages/67/32/f3cd1667028424fa7001d82e10ee35386eea1408b93d399b09fb0aa7875f/msgpack-1.1.2-cp313-cp313-win32.whl", hash = "sha256:a7787d353595c7c7e145e2331abf8b7ff1e6673a6b974ded96e6d4ec09f00c8c", size = 65037, upload-time = "2025-10-08T09:15:21.416Z" }, + { url = "https://files.pythonhosted.org/packages/74/07/1ed8277f8653c40ebc65985180b007879f6a836c525b3885dcc6448ae6cb/msgpack-1.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:a465f0dceb8e13a487e54c07d04ae3ba131c7c5b95e2612596eafde1dccf64a9", size = 72631, upload-time = "2025-10-08T09:15:22.431Z" }, + { url = "https://files.pythonhosted.org/packages/e5/db/0314e4e2db56ebcf450f277904ffd84a7988b9e5da8d0d61ab2d057df2b6/msgpack-1.1.2-cp313-cp313-win_arm64.whl", hash = "sha256:e69b39f8c0aa5ec24b57737ebee40be647035158f14ed4b40e6f150077e21a84", size = 64118, upload-time = "2025-10-08T09:15:23.402Z" }, + { url = "https://files.pythonhosted.org/packages/22/71/201105712d0a2ff07b7873ed3c220292fb2ea5120603c00c4b634bcdafb3/msgpack-1.1.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e23ce8d5f7aa6ea6d2a2b326b4ba46c985dbb204523759984430db7114f8aa00", size = 81127, upload-time = "2025-10-08T09:15:24.408Z" }, + { url = "https://files.pythonhosted.org/packages/1b/9f/38ff9e57a2eade7bf9dfee5eae17f39fc0e998658050279cbb14d97d36d9/msgpack-1.1.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6c15b7d74c939ebe620dd8e559384be806204d73b4f9356320632d783d1f7939", size = 84981, upload-time = "2025-10-08T09:15:25.812Z" }, + { url = "https://files.pythonhosted.org/packages/8e/a9/3536e385167b88c2cc8f4424c49e28d49a6fc35206d4a8060f136e71f94c/msgpack-1.1.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:99e2cb7b9031568a2a5c73aa077180f93dd2e95b4f8d3b8e14a73ae94a9e667e", size = 411885, upload-time = "2025-10-08T09:15:27.22Z" }, + { url = "https://files.pythonhosted.org/packages/2f/40/dc34d1a8d5f1e51fc64640b62b191684da52ca469da9cd74e84936ffa4a6/msgpack-1.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:180759d89a057eab503cf62eeec0aa61c4ea1200dee709f3a8e9397dbb3b6931", size = 419658, upload-time = "2025-10-08T09:15:28.4Z" }, + { url = "https://files.pythonhosted.org/packages/3b/ef/2b92e286366500a09a67e03496ee8b8ba00562797a52f3c117aa2b29514b/msgpack-1.1.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:04fb995247a6e83830b62f0b07bf36540c213f6eac8e851166d8d86d83cbd014", size = 403290, upload-time = "2025-10-08T09:15:29.764Z" }, + { url = "https://files.pythonhosted.org/packages/78/90/e0ea7990abea5764e4655b8177aa7c63cdfa89945b6e7641055800f6c16b/msgpack-1.1.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8e22ab046fa7ede9e36eeb4cfad44d46450f37bb05d5ec482b02868f451c95e2", size = 415234, upload-time = "2025-10-08T09:15:31.022Z" }, + { url = "https://files.pythonhosted.org/packages/72/4e/9390aed5db983a2310818cd7d3ec0aecad45e1f7007e0cda79c79507bb0d/msgpack-1.1.2-cp314-cp314-win32.whl", hash = "sha256:80a0ff7d4abf5fecb995fcf235d4064b9a9a8a40a3ab80999e6ac1e30b702717", size = 66391, upload-time = "2025-10-08T09:15:32.265Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f1/abd09c2ae91228c5f3998dbd7f41353def9eac64253de3c8105efa2082f7/msgpack-1.1.2-cp314-cp314-win_amd64.whl", hash = "sha256:9ade919fac6a3e7260b7f64cea89df6bec59104987cbea34d34a2fa15d74310b", size = 73787, upload-time = "2025-10-08T09:15:33.219Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b0/9d9f667ab48b16ad4115c1935d94023b82b3198064cb84a123e97f7466c1/msgpack-1.1.2-cp314-cp314-win_arm64.whl", hash = "sha256:59415c6076b1e30e563eb732e23b994a61c159cec44deaf584e5cc1dd662f2af", size = 66453, upload-time = "2025-10-08T09:15:34.225Z" }, + { url = "https://files.pythonhosted.org/packages/16/67/93f80545eb1792b61a217fa7f06d5e5cb9e0055bed867f43e2b8e012e137/msgpack-1.1.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:897c478140877e5307760b0ea66e0932738879e7aa68144d9b78ea4c8302a84a", size = 85264, upload-time = "2025-10-08T09:15:35.61Z" }, + { url = "https://files.pythonhosted.org/packages/87/1c/33c8a24959cf193966ef11a6f6a2995a65eb066bd681fd085afd519a57ce/msgpack-1.1.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a668204fa43e6d02f89dbe79a30b0d67238d9ec4c5bd8a940fc3a004a47b721b", size = 89076, upload-time = "2025-10-08T09:15:36.619Z" }, + { url = "https://files.pythonhosted.org/packages/fc/6b/62e85ff7193663fbea5c0254ef32f0c77134b4059f8da89b958beb7696f3/msgpack-1.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5559d03930d3aa0f3aacb4c42c776af1a2ace2611871c84a75afe436695e6245", size = 435242, upload-time = "2025-10-08T09:15:37.647Z" }, + { url = "https://files.pythonhosted.org/packages/c1/47/5c74ecb4cc277cf09f64e913947871682ffa82b3b93c8dad68083112f412/msgpack-1.1.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70c5a7a9fea7f036b716191c29047374c10721c389c21e9ffafad04df8c52c90", size = 432509, upload-time = "2025-10-08T09:15:38.794Z" }, + { url = "https://files.pythonhosted.org/packages/24/a4/e98ccdb56dc4e98c929a3f150de1799831c0a800583cde9fa022fa90602d/msgpack-1.1.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f2cb069d8b981abc72b41aea1c580ce92d57c673ec61af4c500153a626cb9e20", size = 415957, upload-time = "2025-10-08T09:15:40.238Z" }, + { url = "https://files.pythonhosted.org/packages/da/28/6951f7fb67bc0a4e184a6b38ab71a92d9ba58080b27a77d3e2fb0be5998f/msgpack-1.1.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d62ce1f483f355f61adb5433ebfd8868c5f078d1a52d042b0a998682b4fa8c27", size = 422910, upload-time = "2025-10-08T09:15:41.505Z" }, + { url = "https://files.pythonhosted.org/packages/f0/03/42106dcded51f0a0b5284d3ce30a671e7bd3f7318d122b2ead66ad289fed/msgpack-1.1.2-cp314-cp314t-win32.whl", hash = "sha256:1d1418482b1ee984625d88aa9585db570180c286d942da463533b238b98b812b", size = 75197, upload-time = "2025-10-08T09:15:42.954Z" }, + { url = "https://files.pythonhosted.org/packages/15/86/d0071e94987f8db59d4eeb386ddc64d0bb9b10820a8d82bcd3e53eeb2da6/msgpack-1.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:5a46bf7e831d09470ad92dff02b8b1ac92175ca36b087f904a0519857c6be3ff", size = 85772, upload-time = "2025-10-08T09:15:43.954Z" }, + { url = "https://files.pythonhosted.org/packages/81/f2/08ace4142eb281c12701fc3b93a10795e4d4dc7f753911d836675050f886/msgpack-1.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d99ef64f349d5ec3293688e91486c5fdb925ed03807f64d98d205d2713c60b46", size = 70868, upload-time = "2025-10-08T09:15:44.959Z" }, +] + [[package]] name = "multidict" version = "6.7.1" From 8a4f6801c5d20e59920047a30ddcbd10667118d2 Mon Sep 17 00:00:00 2001 From: Vibe Nuage Agent Date: Thu, 21 May 2026 17:09:54 +0000 Subject: [PATCH 12/16] Fix msgpack return typing --- .../extra/tests/test_workflow_encoding.py | 3 ++- .../extra/workflows/encoding/payload_encoder.py | 17 ++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/mistralai/extra/tests/test_workflow_encoding.py b/src/mistralai/extra/tests/test_workflow_encoding.py index 94ce2d65..1fd7efce 100644 --- a/src/mistralai/extra/tests/test_workflow_encoding.py +++ b/src/mistralai/extra/tests/test_workflow_encoding.py @@ -2,6 +2,7 @@ import gc import json +from typing import cast import msgpack import pytest @@ -57,7 +58,7 @@ def _compressed_payload_msgpack( payload_data["compression"] = invalid_compression if invalid_payload is not None: payload_data["payload"] = invalid_payload - return msgpack.packb(payload_data, use_bin_type=True) + return cast(bytes, msgpack.packb(payload_data, use_bin_type=True)) @pytest.fixture diff --git a/src/mistralai/extra/workflows/encoding/payload_encoder.py b/src/mistralai/extra/workflows/encoding/payload_encoder.py index 7d147900..25bbbe97 100644 --- a/src/mistralai/extra/workflows/encoding/payload_encoder.py +++ b/src/mistralai/extra/workflows/encoding/payload_encoder.py @@ -7,7 +7,7 @@ import logging import os import urllib.parse -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast import msgpack from pydantic import BaseModel, ValidationError @@ -84,12 +84,15 @@ def from_msgpack(cls, data: bytes) -> "CompressedPayloadData": ) from exc def to_msgpack(self) -> bytes: - return msgpack.packb( - { - "compression": self.compression.model_dump(mode="json"), - "payload": self.payload, - }, - use_bin_type=True, + return cast( + bytes, + msgpack.packb( + { + "compression": self.compression.model_dump(mode="json"), + "payload": self.payload, + }, + use_bin_type=True, + ), ) def get_payload(self) -> bytes: From a8271b2cab2308628c5992b7e33ee9a0d3f33b7d Mon Sep 17 00:00:00 2001 From: Vibe Nuage Agent Date: Fri, 22 May 2026 09:10:56 +0000 Subject: [PATCH 13/16] Address workflow compression review comments --- .../extra/tests/test_workflow_encoding.py | 11 ++--------- .../extra/workflows/encoding/config.py | 6 +----- .../workflows/encoding/payload_encoder.py | 18 ++++++------------ 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/src/mistralai/extra/tests/test_workflow_encoding.py b/src/mistralai/extra/tests/test_workflow_encoding.py index 1fd7efce..8aa4f2b1 100644 --- a/src/mistralai/extra/tests/test_workflow_encoding.py +++ b/src/mistralai/extra/tests/test_workflow_encoding.py @@ -677,15 +677,8 @@ async def test_payload_encoder_encodes_event_content_without_offloading(): ) payload = json.dumps({"data": "x" * 20_000}).encode() - encoded, encoding_options = await encoder.encode_payload_content( - payload, - allow_offloading=False, - force_full_encryption=True, - ) + encoded, encoding_options = await encoder.encode_event_payload_content(payload) decoded = await decoder.decode_payload_content(encoded, encoding_options) - assert encoding_options == [ - EncodedPayloadOptions.COMPRESSED, - EncodedPayloadOptions.ENCRYPTED, - ] + assert encoding_options == [EncodedPayloadOptions.COMPRESSED] assert decoded == payload diff --git a/src/mistralai/extra/workflows/encoding/config.py b/src/mistralai/extra/workflows/encoding/config.py index d986aa29..d54ad9fe 100644 --- a/src/mistralai/extra/workflows/encoding/config.py +++ b/src/mistralai/extra/workflows/encoding/config.py @@ -67,8 +67,4 @@ class PayloadCompressionConfig(BaseModel): class WorkflowEncodingConfig(BaseModel): payload_offloading: PayloadOffloadingConfig | None = None payload_encryption: PayloadEncryptionConfig | None = None - # TEMPORARY: default compression on to validate in staging preview. - # TODO: revert to None and wire through WorkerConfig env vars once validated. - payload_compression: PayloadCompressionConfig | None = Field( - default_factory=PayloadCompressionConfig - ) + payload_compression: PayloadCompressionConfig | None = None diff --git a/src/mistralai/extra/workflows/encoding/payload_encoder.py b/src/mistralai/extra/workflows/encoding/payload_encoder.py index 25bbbe97..bd123817 100644 --- a/src/mistralai/extra/workflows/encoding/payload_encoder.py +++ b/src/mistralai/extra/workflows/encoding/payload_encoder.py @@ -313,7 +313,6 @@ async def encode_payload_content( context: Optional[WorkflowContext] = None, *, allow_offloading: bool = True, - force_full_encryption: bool = False, ) -> tuple[bytes, list[EncodedPayloadOptions]]: """Handle payload encoding. @@ -328,8 +327,7 @@ async def encode_payload_content( # Partial encryption needs the original JSON fields. It must run before # compression or offloading, which make field-level markers unavailable. if ( - not force_full_encryption - and self.encryption_config is not None + self.encryption_config is not None and self.encryption_config.mode == PayloadEncryptionMode.PARTIAL ): data, partially_encrypted = await self._partially_encrypt_fields(data) @@ -350,9 +348,9 @@ async def encode_payload_content( # Full encryption intentionally remains the final transform. If the # payload was offloaded, this encrypts the small blob-reference envelope # rather than the blob bytes. - if self.encryption_config is not None and ( - force_full_encryption - or self.encryption_config.mode == PayloadEncryptionMode.FULL + if ( + self.encryption_config is not None + and self.encryption_config.mode == PayloadEncryptionMode.FULL ): data = self._encrypt(data) encoding_options.append(EncodedPayloadOptions.ENCRYPTED) @@ -360,14 +358,10 @@ async def encode_payload_content( return data, encoding_options async def encode_event_payload_content( - self, data: Union[bytes, str], force_full_encryption: bool = False + self, data: Union[bytes, str] ) -> tuple[bytes, list[EncodedPayloadOptions]]: """Encode event payload content without offloading.""" - return await self.encode_payload_content( - data, - allow_offloading=False, - force_full_encryption=force_full_encryption, - ) + return await self.encode_payload_content(data, allow_offloading=False) async def decode_payload_content( self, From 722d926191dba0648af7806072f8bb0b4b920d54 Mon Sep 17 00:00:00 2001 From: Vibe Nuage Agent Date: Mon, 25 May 2026 13:15:03 +0000 Subject: [PATCH 14/16] Clarify workflow event payload encoding --- .../extra/tests/test_workflow_encoding.py | 10 ++++- .../workflows/encoding/payload_encoder.py | 37 +++++++++++++++++-- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/mistralai/extra/tests/test_workflow_encoding.py b/src/mistralai/extra/tests/test_workflow_encoding.py index 8aa4f2b1..33260af1 100644 --- a/src/mistralai/extra/tests/test_workflow_encoding.py +++ b/src/mistralai/extra/tests/test_workflow_encoding.py @@ -677,8 +677,14 @@ async def test_payload_encoder_encodes_event_content_without_offloading(): ) payload = json.dumps({"data": "x" * 20_000}).encode() - encoded, encoding_options = await encoder.encode_event_payload_content(payload) + encoded, encoding_options = await encoder.encode_event_payload_content( + payload, + force_full_encryption=True, + ) decoded = await decoder.decode_payload_content(encoded, encoding_options) - assert encoding_options == [EncodedPayloadOptions.COMPRESSED] + assert encoding_options == [ + EncodedPayloadOptions.COMPRESSED, + EncodedPayloadOptions.ENCRYPTED, + ] assert decoded == payload diff --git a/src/mistralai/extra/workflows/encoding/payload_encoder.py b/src/mistralai/extra/workflows/encoding/payload_encoder.py index bd123817..2bccfd4a 100644 --- a/src/mistralai/extra/workflows/encoding/payload_encoder.py +++ b/src/mistralai/extra/workflows/encoding/payload_encoder.py @@ -358,10 +358,41 @@ async def encode_payload_content( return data, encoding_options async def encode_event_payload_content( - self, data: Union[bytes, str] + self, data: Union[bytes, str], force_full_encryption: bool = False ) -> tuple[bytes, list[EncodedPayloadOptions]]: - """Encode event payload content without offloading.""" - return await self.encode_payload_content(data, allow_offloading=False) + """Encode workflow event payload content without blob offloading. + + Event payloads may be emitted before event upload support is wired into + this SDK. Keep this path separate from network input encoding so future + event producers can compress and encrypt events without accidentally + offloading them. + """ + if isinstance(data, str): + data = data.encode() + + encoding_options: list[EncodedPayloadOptions] = [] + + if ( + not force_full_encryption + and self.encryption_config is not None + and self.encryption_config.mode == PayloadEncryptionMode.PARTIAL + ): + data, partially_encrypted = await self._partially_encrypt_fields(data) + if partially_encrypted: + encoding_options.append(EncodedPayloadOptions.PARTIALLY_ENCRYPTED) + + data, compressed = self._compress(data) + if compressed: + encoding_options.append(EncodedPayloadOptions.COMPRESSED) + + if self.encryption_config is not None and ( + force_full_encryption + or self.encryption_config.mode == PayloadEncryptionMode.FULL + ): + data = self._encrypt(data) + encoding_options.append(EncodedPayloadOptions.ENCRYPTED) + + return data, encoding_options async def decode_payload_content( self, From 0ed8b7c576975eece00cb24f45cd12f8f4ca3d71 Mon Sep 17 00:00:00 2001 From: Vibe Nuage Agent Date: Mon, 25 May 2026 13:20:32 +0000 Subject: [PATCH 15/16] Revert "Clarify workflow event payload encoding" This reverts commit 722d926191dba0648af7806072f8bb0b4b920d54. --- .../extra/tests/test_workflow_encoding.py | 10 +---- .../workflows/encoding/payload_encoder.py | 37 ++----------------- 2 files changed, 5 insertions(+), 42 deletions(-) diff --git a/src/mistralai/extra/tests/test_workflow_encoding.py b/src/mistralai/extra/tests/test_workflow_encoding.py index 33260af1..8aa4f2b1 100644 --- a/src/mistralai/extra/tests/test_workflow_encoding.py +++ b/src/mistralai/extra/tests/test_workflow_encoding.py @@ -677,14 +677,8 @@ async def test_payload_encoder_encodes_event_content_without_offloading(): ) payload = json.dumps({"data": "x" * 20_000}).encode() - encoded, encoding_options = await encoder.encode_event_payload_content( - payload, - force_full_encryption=True, - ) + encoded, encoding_options = await encoder.encode_event_payload_content(payload) decoded = await decoder.decode_payload_content(encoded, encoding_options) - assert encoding_options == [ - EncodedPayloadOptions.COMPRESSED, - EncodedPayloadOptions.ENCRYPTED, - ] + assert encoding_options == [EncodedPayloadOptions.COMPRESSED] assert decoded == payload diff --git a/src/mistralai/extra/workflows/encoding/payload_encoder.py b/src/mistralai/extra/workflows/encoding/payload_encoder.py index 2bccfd4a..bd123817 100644 --- a/src/mistralai/extra/workflows/encoding/payload_encoder.py +++ b/src/mistralai/extra/workflows/encoding/payload_encoder.py @@ -358,41 +358,10 @@ async def encode_payload_content( return data, encoding_options async def encode_event_payload_content( - self, data: Union[bytes, str], force_full_encryption: bool = False + self, data: Union[bytes, str] ) -> tuple[bytes, list[EncodedPayloadOptions]]: - """Encode workflow event payload content without blob offloading. - - Event payloads may be emitted before event upload support is wired into - this SDK. Keep this path separate from network input encoding so future - event producers can compress and encrypt events without accidentally - offloading them. - """ - if isinstance(data, str): - data = data.encode() - - encoding_options: list[EncodedPayloadOptions] = [] - - if ( - not force_full_encryption - and self.encryption_config is not None - and self.encryption_config.mode == PayloadEncryptionMode.PARTIAL - ): - data, partially_encrypted = await self._partially_encrypt_fields(data) - if partially_encrypted: - encoding_options.append(EncodedPayloadOptions.PARTIALLY_ENCRYPTED) - - data, compressed = self._compress(data) - if compressed: - encoding_options.append(EncodedPayloadOptions.COMPRESSED) - - if self.encryption_config is not None and ( - force_full_encryption - or self.encryption_config.mode == PayloadEncryptionMode.FULL - ): - data = self._encrypt(data) - encoding_options.append(EncodedPayloadOptions.ENCRYPTED) - - return data, encoding_options + """Encode event payload content without offloading.""" + return await self.encode_payload_content(data, allow_offloading=False) async def decode_payload_content( self, From 9563c1549fb8a00291af8917acee6105cd0f1c14 Mon Sep 17 00:00:00 2001 From: Vibe Nuage Agent Date: Mon, 25 May 2026 13:21:37 +0000 Subject: [PATCH 16/16] Make msgpack a workflow compression extra --- pyproject.toml | 3 ++- .../extra/workflows/encoding/payload_encoder.py | 14 +++++++++++++- uv.lock | 6 ++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eaf40f95..701d8684 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ dependencies = [ "opentelemetry-api (>=1.33.1,<2.0.0)", "opentelemetry-semantic-conventions (>=0.60b1,<0.61)", "jsonpath-python >=1.0.6", # required for speakeasy generated path with pagination - "msgpack>=1.1.0,<2.0.0", ] [project.optional-dependencies] @@ -51,6 +50,7 @@ workflow_payload_encryption = [ "cryptography>=41.0.0,<47.0.0", ] workflow_payload_compression = [ + "msgpack>=1.1.0,<2.0.0", "zstandard>=0.25.0,<0.26", ] @@ -63,6 +63,7 @@ dev = [ "invoke>=2.2.0,<3", "pyyaml>=6.0.2,<7", "mypy==1.15.0", + "msgpack>=1.1.0,<2.0.0", "opentelemetry-sdk (>=1.33.1,<2.0.0)", "pylint==3.2.3", "pytest>=8.2.2,<9", diff --git a/src/mistralai/extra/workflows/encoding/payload_encoder.py b/src/mistralai/extra/workflows/encoding/payload_encoder.py index bd123817..a90cb8aa 100644 --- a/src/mistralai/extra/workflows/encoding/payload_encoder.py +++ b/src/mistralai/extra/workflows/encoding/payload_encoder.py @@ -3,13 +3,14 @@ import base64 import functools import hashlib +from importlib import import_module import json import logging import os +from types import ModuleType import urllib.parse from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast -import msgpack from pydantic import BaseModel, ValidationError if TYPE_CHECKING: @@ -58,6 +59,15 @@ class OffloadedPayloadData(BaseModel): key: str +def _require_msgpack() -> ModuleType: + try: + return import_module("msgpack") + except ImportError: + raise WorkflowPayloadCompressionException( + "Payload compression requires installing mistralai[workflow_payload_compression]" + ) from None + + class CompressedPayloadData(BaseModel): compression: AlgorithmConfig payload: bytes @@ -70,6 +80,7 @@ def from_payload( @classmethod def from_msgpack(cls, data: bytes) -> "CompressedPayloadData": + msgpack = _require_msgpack() try: unpacked = msgpack.unpackb(data, raw=False) except Exception as exc: @@ -84,6 +95,7 @@ def from_msgpack(cls, data: bytes) -> "CompressedPayloadData": ) from exc def to_msgpack(self) -> bytes: + msgpack = _require_msgpack() return cast( bytes, msgpack.packb( diff --git a/uv.lock b/uv.lock index a7bbd2cc..531d3615 100644 --- a/uv.lock +++ b/uv.lock @@ -1041,7 +1041,6 @@ dependencies = [ { name = "eval-type-backport" }, { name = "httpx" }, { name = "jsonpath-python" }, - { name = "msgpack" }, { name = "opentelemetry-api" }, { name = "opentelemetry-semantic-conventions" }, { name = "pydantic" }, @@ -1063,6 +1062,7 @@ realtime = [ { name = "websockets" }, ] workflow-payload-compression = [ + { name = "msgpack" }, { name = "zstandard" }, ] workflow-payload-encryption = [ @@ -1091,6 +1091,7 @@ dev = [ { name = "griffe" }, { name = "invoke" }, { name = "mcp" }, + { name = "msgpack" }, { name = "mypy" }, { name = "opentelemetry-sdk" }, { name = "pylint" }, @@ -1126,7 +1127,7 @@ requires-dist = [ { name = "mistralai", extras = ["workflow-payload-offloading-azure"], marker = "extra == 'workflow-payload-offloading'" }, { name = "mistralai", extras = ["workflow-payload-offloading-gcs"], marker = "extra == 'workflow-payload-offloading'" }, { name = "mistralai", extras = ["workflow-payload-offloading-s3"], marker = "extra == 'workflow-payload-offloading'" }, - { name = "msgpack", specifier = ">=1.1.0,<2.0.0" }, + { name = "msgpack", marker = "extra == 'workflow-payload-compression'", specifier = ">=1.1.0,<2.0.0" }, { name = "opentelemetry-api", specifier = ">=1.33.1,<2.0.0" }, { name = "opentelemetry-semantic-conventions", specifier = ">=0.60b1,<0.61" }, { name = "pydantic", specifier = ">=2.11.2" }, @@ -1144,6 +1145,7 @@ dev = [ { name = "griffe", specifier = ">=1.7.3,<2" }, { name = "invoke", specifier = ">=2.2.0,<3" }, { name = "mcp", specifier = ">=1.0,<2" }, + { name = "msgpack", specifier = ">=1.1.0,<2.0.0" }, { name = "mypy", specifier = "==1.15.0" }, { name = "opentelemetry-sdk", specifier = ">=1.33.1,<2.0.0" }, { name = "pylint", specifier = "==3.2.3" },