diff --git a/python/bucketbase/__init__.py b/python/bucketbase/__init__.py index 934971f..28ab9d3 100644 --- a/python/bucketbase/__init__.py +++ b/python/bucketbase/__init__.py @@ -4,7 +4,7 @@ """ beware of the order of imports, as some of the imports are circular, like fs_bucket due to (named_lock_manager) """ -from bucketbase.ibucket import S3_NAME_CHARS_NO_SEP, IBucket, ShallowListing +from bucketbase.ibucket import S3_NAME_CHARS_NO_SEP, IBucket, ObjectVersion, ShallowListing from bucketbase.errors import DeleteError from bucketbase.cached_immutable_bucket import CachedImmutableBucket diff --git a/python/bucketbase/ibucket.py b/python/bucketbase/ibucket.py index 178fba3..63ef680 100644 --- a/python/bucketbase/ibucket.py +++ b/python/bucketbase/ibucket.py @@ -34,6 +34,18 @@ class ShallowListing: prefixes: slist[str] +@dataclass(frozen=True) +class ObjectVersion: + """ + Object version metadata returned by version-aware bucket implementations. + """ + + name: PurePosixPath + version_id: str + is_latest: bool + is_delete_marker: bool = False + + class ObjectStream(AbstractContextManager[BinaryIO]): def __init__(self, stream: BinaryIO, name: PurePosixPath) -> None: self._stream = stream @@ -351,6 +363,31 @@ def remove_objects(self, names: Iterable[PurePosixPath | str]) -> slist[DeleteEr """ raise NotImplementedError() + def list_object_versions(self, name: PurePosixPath | str) -> slist[ObjectVersion]: + """ + Lists versions for a single object name. + """ + raise NotImplementedError() + + def get_object_version(self, name: PurePosixPath | str, version_id: str) -> bytes: + """ + Retrieves a specific object version. Use get_object() to retrieve the current object version. + """ + with self.get_object_version_stream(name, version_id) as response: + return response.read() + + def get_object_version_stream(self, name: PurePosixPath | str, version_id: str) -> ObjectStream: + """ + Retrieves a stream for a specific object version. Use get_object_stream() to retrieve the current object version. + """ + raise NotImplementedError() + + def remove_object_with_versions(self, name: PurePosixPath | str) -> slist[DeleteError]: + """ + Deletes every listed version of a single object name, including delete markers when supported. + """ + raise NotImplementedError() + def open_write(self, name: PurePosixPath | str, timeout_sec: Optional[float] = None) -> AbstractContextManager[BinaryIO]: """ Returns a writable stream that, for MinIO, supports multipart upload functionality. diff --git a/python/bucketbase/memory_bucket.py b/python/bucketbase/memory_bucket.py index 52dc574..b75aa86 100644 --- a/python/bucketbase/memory_bucket.py +++ b/python/bucketbase/memory_bucket.py @@ -1,5 +1,6 @@ import io import multiprocessing +import uuid import weakref from contextlib import contextmanager from multiprocessing.managers import DictProxy, SyncManager @@ -11,7 +12,9 @@ from streamerate import stream as sstream from bucketbase import DeleteError -from bucketbase.ibucket import IBucket, ObjectStream, ShallowListing +from bucketbase.ibucket import IBucket, ObjectStream, ObjectVersion, ShallowListing + +MemoryObjectVersion = tuple[str, bytes | None, bool] class _NonClosingBytesIO(io.BytesIO): @@ -30,14 +33,21 @@ class MemoryBucket(IBucket): def __init__(self) -> None: self._objects: dict[str, bytes] | DictProxy[str, bytes] = {} + self._object_versions: dict[str, list[MemoryObjectVersion]] | DictProxy[str, list[MemoryObjectVersion]] = {} self._lock: Any = RLock() + def _store_object_version(self, name: str, content: bytes | None, is_delete_marker: bool) -> None: + versions = self._object_versions.get(name, []) + versions.append((uuid.uuid4().hex, content, is_delete_marker)) + self._object_versions[name] = versions + def put_object(self, name: PurePosixPath | str, content: Union[str, bytes, bytearray]) -> None: _name = self._validate_name(name) _content = self._encode_content(content) with self._lock: self._objects[_name] = _content + self._store_object_version(_name, _content, is_delete_marker=False) def put_object_stream(self, name: PurePosixPath | str, stream: BinaryIO) -> None: _content = stream.read() @@ -55,6 +65,20 @@ def get_object_stream(self, name: PurePosixPath | str) -> ObjectStream: content = self.get_object(name) return ObjectStream(io.BytesIO(content), PurePosixPath(name)) + def get_object_version(self, name: PurePosixPath | str, version_id: str) -> bytes: + _name = self._validate_name(name) + with self._lock: + for stored_version_id, content, is_delete_marker in self._object_versions.get(_name, []): + if stored_version_id == version_id: + if is_delete_marker or content is None: + raise FileNotFoundError(f"Object {_name} version {version_id} is a delete marker in MemoryObjectStore") + return content + raise FileNotFoundError(f"Object {_name} version {version_id} not found in MemoryObjectStore") + + def get_object_version_stream(self, name: PurePosixPath | str, version_id: str) -> ObjectStream: + content = self.get_object_version(name, version_id) + return ObjectStream(io.BytesIO(content), PurePosixPath(name)) + def list_objects(self, prefix: PurePosixPath | str = "") -> slist[PurePosixPath]: self._split_prefix(prefix) # validate prefix str_prefix = str(prefix) @@ -79,6 +103,21 @@ def shallow_list_objects(self, prefix: PurePosixPath | str = "") -> ShallowListi prefixes.add(common_prefix) return ShallowListing(objects=objects, prefixes=prefixes.to_list()) + def list_object_versions(self, name: PurePosixPath | str) -> slist[ObjectVersion]: + _name = self._validate_name(name) + with self._lock: + versions = self._object_versions.get(_name, []) + latest_index = len(versions) - 1 + return slist( + ObjectVersion( + name=PurePosixPath(_name), + version_id=version_id, + is_latest=index == latest_index, + is_delete_marker=is_delete_marker, + ) + for index, (version_id, _content, is_delete_marker) in reversed(list(enumerate(versions))) + ) + def exists(self, name: PurePosixPath | str) -> bool: _name = self._validate_name(name) with self._lock: @@ -93,8 +132,16 @@ def remove_objects(self, names: Iterable[PurePosixPath | str]) -> slist[DeleteEr obj = self._validate_name(obj) if obj in self._objects: self._objects.pop(obj) + self._store_object_version(obj, content=None, is_delete_marker=True) return delete_errors + def remove_object_with_versions(self, name: PurePosixPath | str) -> slist[DeleteError]: + _name = self._validate_name(name) + with self._lock: + self._objects.pop(_name, None) + self._object_versions.pop(_name, None) + return slist() + def get_size(self, name: PurePosixPath | str) -> int: _name = self._validate_name(name) @@ -141,6 +188,7 @@ def open_write_sync(self, name: PurePosixPath | str) -> Generator[BinaryIO, None if not exception_occurred: with self._lock: self._objects[_name] = content + self._store_object_version(_name, content, is_delete_marker=False) class SharedMemoryBucket(MemoryBucket): @@ -170,6 +218,7 @@ def __init__(self, manager: Optional[SyncManager] = None) -> None: # override parent's structures with managed ones. This is a small hack, but it's simple & clear enough self._objects: dict[str, bytes] | DictProxy[str, bytes] = manager.dict() + self._object_versions: dict[str, list[MemoryObjectVersion]] | DictProxy[str, list[MemoryObjectVersion]] = manager.dict() self._lock: Any = manager.RLock() def __getstate__(self) -> dict[str, Any]: diff --git a/python/bucketbase/minio_bucket.py b/python/bucketbase/minio_bucket.py index f33b889..f6a917e 100644 --- a/python/bucketbase/minio_bucket.py +++ b/python/bucketbase/minio_bucket.py @@ -18,7 +18,7 @@ from streamerate import stream as sstream from urllib3 import BaseHTTPResponse -from bucketbase.ibucket import IBucket, ObjectStream, ShallowListing +from bucketbase.ibucket import IBucket, ObjectStream, ObjectVersion, ShallowListing class MinioObjectStream(ObjectStream): @@ -125,18 +125,42 @@ def __init__(self, bucket_name: str, minio_client: Minio, part_size: int | None @classmethod def _get_object_name(cls, obj: Object) -> str: - return obj.name if cls._USES_NAME_ATTRIBUTE else obj.object_name + object_name = obj.name if cls._USES_NAME_ATTRIBUTE else obj.object_name + if object_name is None: + raise ValueError("Minio object listing item has no object name") + return object_name + + @staticmethod + def _read_response(response: BaseHTTPResponse) -> bytes: + try: + data = bytes() + for buffer in response.stream(amt=1024 * 1024): + data += buffer + return data + finally: + response.release_conn() + + @staticmethod + def _to_bool(value: object) -> bool: + return value if isinstance(value, bool) else str(value).lower() == "true" + + @classmethod + def _to_object_version(cls, obj: Object) -> ObjectVersion: + object_name = cls._get_object_name(obj) + version_id = obj.version_id + if version_id is None: + raise ValueError(f"Minio object listing item {object_name} has no version id") + return ObjectVersion( + name=PurePosixPath(object_name), + version_id=version_id, + is_latest=cls._to_bool(obj.is_latest), + is_delete_marker=obj.is_delete_marker, + ) def get_object(self, name: PurePosixPath | str) -> bytes: with self.get_object_stream(name) as response: assert isinstance(response, BaseHTTPResponse), f"Expected IOBase, got {type(response)}" - try: - data = bytes() - for buffer in response.stream(amt=1024 * 1024): - data += buffer - return data - finally: - response.release_conn() + return self._read_response(response) def get_object_stream(self, name: PurePosixPath | str) -> ObjectStream: _name = self._validate_name(name) @@ -150,6 +174,21 @@ def get_object_stream(self, name: PurePosixPath | str) -> ObjectStream: _name_path = PurePosixPath(_name) if isinstance(_name, str) else _name return MinioObjectStream(response, _name_path) + def get_object_version(self, name: PurePosixPath | str, version_id: str) -> bytes: + with self.get_object_version_stream(name, version_id) as response: + assert isinstance(response, BaseHTTPResponse), f"Expected IOBase, got {type(response)}" + return self._read_response(response) + + def get_object_version_stream(self, name: PurePosixPath | str, version_id: str) -> ObjectStream: + _name = self._validate_name(name) + try: + response: BaseHTTPResponse = self._minio_client.get_object(self._bucket_name, _name, version_id=version_id) + except minio.error.S3Error as e: + if e.code in ("MethodNotAllowed", "NoSuchKey", "NoSuchVersion"): + raise FileNotFoundError(f"Object {_name} version {version_id} not found in bucket {self._bucket_name} on Minio") from e + raise + return MinioObjectStream(response, PurePosixPath(_name)) + def fget_object(self, name: PurePosixPath | str, file_path: Path) -> None: """ Raises: @@ -198,6 +237,11 @@ def shallow_list_objects(self, prefix: PurePosixPath | str = "") -> ShallowListi objects = object_names.filter(lambda x: not x.endswith("/")).map(PurePosixPath).to_list() return ShallowListing(objects=objects, prefixes=prefixes) + def list_object_versions(self, name: PurePosixPath | str) -> slist[ObjectVersion]: + _name = self._validate_name(name) + listing_itr = self._minio_client.list_objects(bucket_name=self._bucket_name, prefix=_name, recursive=True, include_version=True) + return sstream(listing_itr).filter(lambda obj: self._get_object_name(obj) == _name).map(self._to_object_version).to_list() + def exists(self, name: PurePosixPath | str) -> bool: _name = self._validate_name(name) try: @@ -216,6 +260,14 @@ def remove_objects(self, names: Iterable[PurePosixPath | str]) -> slist[DeleteEr errors = slist(self._minio_client.remove_objects(self._bucket_name, delete_objects_stream)) return errors + def remove_object_with_versions(self, name: PurePosixPath | str) -> slist[DeleteError]: + versions = self.list_object_versions(name) + if versions.size() == 0: + return slist() + + delete_objects_stream = versions.map(lambda version: DeleteObject(str(version.name), version.version_id)) + return slist(self._minio_client.remove_objects(self._bucket_name, delete_objects_stream)) + def get_size(self, name: PurePosixPath | str) -> int: try: st = self._minio_client.stat_object(self._bucket_name, str(name)) diff --git a/python/poetry.lock b/python/poetry.lock index 0de835a..b5974b0 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -23,6 +23,7 @@ files = [ {file = "argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741"}, {file = "argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1"}, ] +markers = {main = "extra == \"minio\""} [package.dependencies] argon2-cffi-bindings = "*" @@ -62,6 +63,7 @@ files = [ {file = "argon2_cffi_bindings-25.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:da0c79c23a63723aa5d782250fbf51b768abca630285262fb5144ba5ae01e520"}, {file = "argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d"}, ] +markers = {main = "extra == \"minio\""} [package.dependencies] cffi = [ @@ -191,6 +193,7 @@ files = [ {file = "cffi-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9"}, {file = "cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529"}, ] +markers = {main = "extra == \"minio\" or platform_python_implementation == \"CPython\" and sys_platform == \"win32\""} [package.dependencies] pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} @@ -404,6 +407,7 @@ files = [ {file = "ciso8601-2.3.3-pp39-pypy39_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:475583568c06a5bc23a4de8c0521c39c2a46c2e189bae9a6c5efc25ab0605372"}, {file = "ciso8601-2.3.3.tar.gz", hash = "sha256:db5d78d9fb0de8686fbad1c1c2d168ed52efb6e8bf8774ae26226e5034a46dae"}, ] +markers = {main = "extra == \"minio\""} [[package]] name = "colorama" @@ -635,6 +639,8 @@ files = [ {file = "greenlet-3.2.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c2ca18a03a8cfb5b25bc1cbe20f3d9a4c80d8c3b13ba3df49ac3961af0b1018d"}, {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9fe0a28a7b952a21e2c062cd5756d34354117796c6d9215a87f55e38d15402c5"}, {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8854167e06950ca75b898b104b63cc646573aa5fef1353d4508ecdd1ee76254f"}, + {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f47617f698838ba98f4ff4189aef02e7343952df3a615f847bb575c3feb177a7"}, + {file = "greenlet-3.2.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af41be48a4f60429d5cad9d22175217805098a9ef7c40bfef44f7669fb9d74d8"}, {file = "greenlet-3.2.4-cp310-cp310-win_amd64.whl", hash = "sha256:73f49b5368b5359d04e18d15828eecc1806033db5233397748f4ca813ff1056c"}, {file = "greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2"}, {file = "greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246"}, @@ -644,6 +650,8 @@ files = [ {file = "greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8"}, {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52"}, {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:55e9c5affaa6775e2c6b67659f3a71684de4c549b3dd9afca3bc773533d284fa"}, + {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c9c6de1940a7d828635fbd254d69db79e54619f165ee7ce32fda763a9cb6a58c"}, + {file = "greenlet-3.2.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03c5136e7be905045160b1b9fdca93dd6727b180feeafda6818e6496434ed8c5"}, {file = "greenlet-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:9c40adce87eaa9ddb593ccb0fa6a07caf34015a29bf8d344811665b573138db9"}, {file = "greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd"}, {file = "greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb"}, @@ -653,6 +661,8 @@ files = [ {file = "greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0"}, {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0"}, {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f"}, + {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee7a6ec486883397d70eec05059353b8e83eca9168b9f3f9a361971e77e0bcd0"}, + {file = "greenlet-3.2.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:326d234cbf337c9c3def0676412eb7040a35a768efc92504b947b3e9cfc7543d"}, {file = "greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02"}, {file = "greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31"}, {file = "greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945"}, @@ -662,6 +672,8 @@ files = [ {file = "greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671"}, {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b"}, {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d25c5091190f2dc0eaa3f950252122edbbadbb682aa7b1ef2f8af0f8c0afefae"}, + {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6e343822feb58ac4d0a1211bd9399de2b3a04963ddeec21530fc426cc121f19b"}, + {file = "greenlet-3.2.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca7f6f1f2649b89ce02f6f229d7c19f680a6238af656f61e0115b24857917929"}, {file = "greenlet-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:554b03b6e73aaabec3745364d6239e9e012d64c68ccd0b8430c64ccc14939a8b"}, {file = "greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0"}, {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f"}, @@ -669,6 +681,8 @@ files = [ {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1"}, {file = "greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735"}, {file = "greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337"}, + {file = "greenlet-3.2.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2917bdf657f5859fbf3386b12d68ede4cf1f04c90c3a6bc1f013dd68a22e2269"}, + {file = "greenlet-3.2.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:015d48959d4add5d6c9f6c5210ee3803a830dce46356e3bc326d6776bde54681"}, {file = "greenlet-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:e37ab26028f12dbb0ff65f29a8d3d44a765c61e729647bf2ddfbbed621726f01"}, {file = "greenlet-3.2.4-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:b6a7c19cf0d2742d0809a4c05975db036fdff50cd294a93632d6a310bf9ac02c"}, {file = "greenlet-3.2.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:27890167f55d2387576d1f41d9487ef171849ea0359ce1510ca6e06c8bece11d"}, @@ -678,6 +692,8 @@ files = [ {file = "greenlet-3.2.4-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9913f1a30e4526f432991f89ae263459b1c64d1608c0d22a5c79c287b3c70df"}, {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b90654e092f928f110e0007f572007c9727b5265f7632c2fa7415b4689351594"}, {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:81701fd84f26330f0d5f4944d4e92e61afe6319dcd9775e39396e39d7c3e5f98"}, + {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:28a3c6b7cd72a96f61b0e4b2a36f681025b60ae4779cc73c1535eb5f29560b10"}, + {file = "greenlet-3.2.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:52206cd642670b0b320a1fd1cbfd95bca0e043179c1d8a045f2c6109dfe973be"}, {file = "greenlet-3.2.4-cp39-cp39-win32.whl", hash = "sha256:65458b409c1ed459ea899e939f0e1cdb14f58dbc803f2f93c5eab5694d32671b"}, {file = "greenlet-3.2.4-cp39-cp39-win_amd64.whl", hash = "sha256:d2e685ade4dafd447ede19c31277a224a239a0a1a4eca4e6390efedf20260cfb"}, {file = "greenlet-3.2.4.tar.gz", hash = "sha256:0dca0d95ff849f9a364385f36ab49f50065d76964944638be9691e1832e9f86d"}, @@ -944,6 +960,7 @@ files = [ {file = "minio-7.2.18-py3-none-any.whl", hash = "sha256:f23a6edbff8d0bc4b5c1a61b2628a01c5a3342aefc613ff9c276012e6321108f"}, {file = "minio-7.2.18.tar.gz", hash = "sha256:173402a5716099159c5659f9de75be204ebe248557b9f1cc9cf45aa70e9d3024"}, ] +markers = {main = "extra == \"minio\""} [package.dependencies] argon2-cffi = "*" @@ -975,6 +992,7 @@ groups = ["main", "dev"] files = [ {file = "multiminio-1.0.2-py3-none-any.whl", hash = "sha256:39cc4c4518c83dd8065e7ceb9155cf5895367296027cae2a4dc5d172109c4c8b"}, ] +markers = {main = "extra == \"minio\""} [package.dependencies] minio = ">=7.0.0" @@ -1111,7 +1129,6 @@ description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.10" groups = ["main", "dev"] -markers = "python_version < \"3.14\"" files = [ {file = "numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb"}, {file = "numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90"}, @@ -1169,6 +1186,7 @@ files = [ {file = "numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00"}, {file = "numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd"}, ] +markers = {main = "python_version < \"3.14\" and extra == \"minio\"", dev = "python_version < \"3.14\""} [[package]] name = "numpy" @@ -1177,7 +1195,6 @@ description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.11" groups = ["main", "dev"] -markers = "python_version >= \"3.14\"" files = [ {file = "numpy-2.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e78aecd2800b32e8347ce49316d3eaf04aed849cd5b38e0af39f829a4e59f5eb"}, {file = "numpy-2.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7fd09cc5d65bda1e79432859c40978010622112e9194e581e3415a3eccc7f43f"}, @@ -1254,6 +1271,7 @@ files = [ {file = "numpy-2.3.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:81b3a59793523e552c4a96109dde028aa4448ae06ccac5a76ff6532a85558a7f"}, {file = "numpy-2.3.4.tar.gz", hash = "sha256:a7d018bfedb375a8d979ac758b120ba846a7fe764911a64465fd87b8729f4a6a"}, ] +markers = {main = "python_version >= \"3.14\" and extra == \"minio\"", dev = "python_version >= \"3.14\""} [[package]] name = "packaging" @@ -1431,11 +1449,11 @@ description = "C parser in Python" optional = false python-versions = ">=3.8" groups = ["main", "dev"] -markers = "implementation_name != \"PyPy\"" files = [ {file = "pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934"}, {file = "pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2"}, ] +markers = {main = "implementation_name != \"PyPy\" and extra == \"minio\" or implementation_name != \"PyPy\" and platform_python_implementation == \"CPython\" and sys_platform == \"win32\"", dev = "implementation_name != \"PyPy\""} [[package]] name = "pycryptodome" @@ -1487,6 +1505,7 @@ files = [ {file = "pycryptodome-3.23.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:ce64e84a962b63a47a592690bdc16a7eaf709d2c2697ababf24a0def566899a6"}, {file = "pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef"}, ] +markers = {main = "extra == \"minio\""} [[package]] name = "pydantic" @@ -1694,6 +1713,7 @@ files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, ] +markers = {main = "extra == \"minio\""} [package.dependencies] six = ">=1.5" @@ -1709,6 +1729,7 @@ files = [ {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"}, {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, ] +markers = {main = "extra == \"minio\""} [[package]] name = "pywin32-ctypes" @@ -1948,6 +1969,7 @@ files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] +markers = {main = "extra == \"minio\""} [[package]] name = "streamerate" @@ -2091,6 +2113,7 @@ groups = ["main", "dev"] files = [ {file = "tsx-0.2.14-py3-none-any.whl", hash = "sha256:48e03b9a0fc4c9df6e08ae891eb592f5c03209d32b5c339636eb525d481f39f7"}, ] +markers = {main = "extra == \"minio\""} [package.dependencies] ciso8601 = ">=2.3.1" @@ -2275,4 +2298,4 @@ minio = ["minio", "multiminio"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0.0" -content-hash = "0829b5ede6ccd43eafeb57ff8965cb64ea71141c119cab155bb8c9ac3a69ebb8" +content-hash = "1c142bbaf14de3acc6d972bdb1d46204ab515b4e329c8183fb784282d8f82bf3" diff --git a/python/tests/bucket_tester.py b/python/tests/bucket_tester.py index 5059086..051efb0 100644 --- a/python/tests/bucket_tester.py +++ b/python/tests/bucket_tester.py @@ -79,6 +79,89 @@ def writable(self) -> bool: return True +class VersionedIBucketTester: # pylint: disable=too-many-public-methods + INVALID_PREFIXES = ["/", "/dir", "dir1//dir2", "dir1//", "star*1", "dir1/a\file.txt", "at@gmail", "sharp#1", "dollar$1", "comma,"] + PATH_WITH_2025_KEYS = "test-dir-with-2025-keys/" + + def __init__(self, storage: IBucket, test_case: TestCase) -> None: + self.storage = storage + self.test_case = test_case + # Next is a unique suffix to be used in the names of dirs and files, so they will be unique + self.us = uuid.uuid4().hex + + def cleanup(self) -> None: + # ToDo: Vlada - here we remove all versions + self.storage.remove_prefix(f"dir{self.us}") + + def test_full_cycle_object_versions_after_overwrite(self) -> None: + unique_dir = f"dir{self.us}" + path = PurePosixPath(f"{unique_dir}/versioned.txt") + self.storage.put_object(path, b"old content") + self.storage.put_object(path, b"new content") + + versions = self.storage.list_object_versions(path) + old_version_id = versions[1].version_id + + self.test_case.assertIsInstance(versions, slist) + self.test_case.assertEqual(2, len(versions)) + self.test_case.assertTrue(versions[0].is_latest) + self.test_case.assertFalse(versions[1].is_latest) + self.test_case.assertEqual(b"new content", self.storage.get_object(path)) + self.test_case.assertEqual(b"old content", self.storage.get_object_version(path, old_version_id)) + with self.storage.get_object_version_stream(path, old_version_id) as stream: + content = stream.read() + + self.test_case.assertEqual(b"old content", content) + + # test removeobjects + errors = self.storage.remove_objects([path]) + versions = self.storage.list_object_versions(path) + + self.test_case.assertEqual([], list(errors)) + self.test_case.assertFalse(self.storage.exists(path)) + self.test_case.assertTrue(versions[0].is_latest) + self.test_case.assertTrue(versions[0].is_delete_marker) + self.test_case.assertFalse(versions[1].is_latest) + self.test_case.assertEqual(b"old content", self.storage.get_object_version(path, old_version_id)) + with self.test_case.assertRaises(FileNotFoundError): + self.storage.get_object(path) + with self.test_case.assertRaises(FileNotFoundError): + self.storage.get_object_version(path, versions[0].version_id) + + # remove_object_with_versions + errors = self.storage.remove_object_with_versions(path) + + self.test_case.assertEqual([], list(errors)) + self.test_case.assertFalse(self.storage.exists(path)) + self.test_case.assertEqual([], list(self.storage.list_object_versions(path))) + with self.test_case.assertRaises(FileNotFoundError): + self.storage.get_object_version(path, old_version_id) + + def test_open_write_sync_creates_readable_version(self) -> None: + unique_dir = f"dir{self.us}" + path = PurePosixPath(f"{unique_dir}/open-write-sync-versioned.txt") + with self.storage.open_write_sync(path) as writer: + writer.write(b"streamed content") + + version_id = self.storage.list_object_versions(path)[0].version_id + + self.test_case.assertEqual(b"streamed content", self.storage.get_object_version(path, version_id)) + + def test_remove_objects_for_missing_name_does_not_create_version_history(self) -> None: + unique_dir = f"dir{self.us}" + path = PurePosixPath(f"{unique_dir}/missing-versioned.txt") + + errors = self.storage.remove_objects([path]) + + self.test_case.assertEqual([], list(errors)) + self.test_case.assertEqual([], list(self.storage.list_object_versions(path))) + + def test_invalid_names_raise_for_version_methods(self) -> None: + self.test_case.assertRaises(ValueError, self.storage.list_object_versions, "/") + self.test_case.assertRaises(ValueError, self.storage.get_object_version, "/", "v1") + self.test_case.assertRaises(ValueError, self.storage.remove_object_with_versions, "/") + + class IBucketTester: # pylint: disable=too-many-public-methods INVALID_PREFIXES = ["/", "/dir", "dir1//dir2", "dir1//", "star*1", "dir1/a\file.txt", "at@gmail", "sharp#1", "dollar$1", "comma,"] PATH_WITH_2025_KEYS = "test-dir-with-2025-keys/" diff --git a/python/tests/test_memory_bucket.py b/python/tests/test_memory_bucket.py index b7c28f1..18f134c 100644 --- a/python/tests/test_memory_bucket.py +++ b/python/tests/test_memory_bucket.py @@ -19,6 +19,18 @@ def test_put_and_get_object(self): def test_put_and_get_object_stream(self): self.tester.test_put_and_get_object_stream() + def test_full_cycle_object_versions_after_overwrite(self): + self.tester.test_full_cycle_object_versions_after_overwrite() + + def test_open_write_sync_creates_readable_version(self): + self.tester.test_open_write_sync_creates_readable_version() + + def test_remove_objects_for_missing_name_does_not_create_version_history(self): + self.tester.test_remove_objects_for_missing_name_does_not_create_version_history() + + def test_invalid_names_raise_for_version_methods(self): + self.tester.test_invalid_names_raise_for_version_methods() + def test_list_objects(self): self.tester.test_list_objects() @@ -130,6 +142,18 @@ def test_put_and_get_object(self): def test_put_and_get_object_stream(self): self.tester.test_put_and_get_object_stream() + def test_full_cycle_object_versions_after_overwrite(self): + self.tester.test_full_cycle_object_versions_after_overwrite() + + def test_open_write_sync_creates_readable_version(self): + self.tester.test_open_write_sync_creates_readable_version() + + def test_remove_objects_for_missing_name_does_not_create_version_history(self): + self.tester.test_remove_objects_for_missing_name_does_not_create_version_history() + + def test_invalid_names_raise_for_version_methods(self): + self.tester.test_invalid_names_raise_for_version_methods() + def test_list_objects(self): self.tester.test_list_objects() diff --git a/python/tests/test_minio_bucket.py b/python/tests/test_minio_bucket.py index 91a5ca3..d5678d7 100644 --- a/python/tests/test_minio_bucket.py +++ b/python/tests/test_minio_bucket.py @@ -1,14 +1,141 @@ import io +import uuid +from pathlib import PurePosixPath +from typing import Iterable, Iterator from unittest import TestCase from unittest.mock import patch +from minio.commonconfig import ENABLED +from minio.datatypes import Object +from minio.deleteobjects import DeleteError, DeleteObject +from minio.error import S3Error from minio.helpers import MAX_PART_SIZE, MIN_PART_SIZE +from minio.versioningconfig import VersioningConfig +from urllib3 import HTTPResponse +from bucketbase.ibucket import ObjectVersion from bucketbase.minio_bucket import MinioBucket, build_minio_client -from tests.bucket_tester import IBucketTester +from tests.bucket_tester import IBucketTester, VersionedIBucketTester from tests.config import CONFIG +class FakeVersionedMinioClient: + def __init__(self) -> None: + self.list_objects_response: list[Object] = [] + self.get_object_responses_by_version: dict[str | None, HTTPResponse] = {} + self.get_object_error: S3Error | None = None + self.remove_errors: list[DeleteError] = [] + self.list_objects_calls: list[dict[str, object]] = [] + self.get_object_calls: list[dict[str, object]] = [] + self.remove_objects_calls: list[tuple[str, list[DeleteObject]]] = [] + + def list_objects(self, **kwargs: object) -> Iterator[Object]: + self.list_objects_calls.append(kwargs) + return iter(self.list_objects_response) + + def get_object(self, bucket_name: str, object_name: str, version_id: str | None = None) -> HTTPResponse: + self.get_object_calls.append({"bucket_name": bucket_name, "object_name": object_name, "version_id": version_id}) + if self.get_object_error is not None: + raise self.get_object_error + return self.get_object_responses_by_version[version_id] + + def remove_objects(self, bucket_name: str, delete_object_list: Iterable[DeleteObject], bypass_governance_mode: bool = False) -> Iterator[DeleteError]: + self.remove_objects_calls.append((bucket_name, list(delete_object_list))) + return iter(self.remove_errors) + + +class TestMinioBucketVersionMethods(TestCase): + def setUp(self) -> None: + self.client = FakeVersionedMinioClient() + self.bucket = MinioBucket(bucket_name="test-bucket", minio_client=self.client) + + @staticmethod + def _make_object(name: str, version_id: str | None, is_latest: object = "false", is_delete_marker: bool = False) -> Object: + return Object( + bucket_name="test-bucket", + object_name=name, + version_id=version_id, + is_latest=is_latest, + is_delete_marker=is_delete_marker, + ) + + @staticmethod + def _make_response(content: bytes) -> HTTPResponse: + return HTTPResponse(body=io.BytesIO(content), headers={"content-length": str(len(content))}, preload_content=False) + + @staticmethod + def _make_s3_error(code: str) -> S3Error: + return S3Error(HTTPResponse(status=404), code, code, "resource", "request-id", "host-id") + + def test_list_object_versions_filters_exact_name(self) -> None: + self.client.list_objects_response = [ + self._make_object("dir/file.txt", "v2", is_latest="true"), + self._make_object("dir/file.txt.bak", "other", is_latest="true"), + self._make_object("dir/file.txt", "v1", is_delete_marker=True), + ] + + versions = self.bucket.list_object_versions("dir/file.txt") + + self.assertEqual( + [ + ObjectVersion(name=PurePosixPath("dir/file.txt"), version_id="v2", is_latest=True, is_delete_marker=False), + ObjectVersion(name=PurePosixPath("dir/file.txt"), version_id="v1", is_latest=False, is_delete_marker=True), + ], + list(versions), + ) + self.assertEqual( + {"bucket_name": "test-bucket", "prefix": "dir/file.txt", "recursive": True, "include_version": True}, + self.client.list_objects_calls[0], + ) + + def test_list_object_versions_requires_version_id(self) -> None: + self.client.list_objects_response = [self._make_object("dir/file.txt", None)] + + with self.assertRaisesRegex(ValueError, "has no version id"): + self.bucket.list_object_versions("dir/file.txt") + + def test_get_object_version_reads_specific_version(self) -> None: + self.client.get_object_responses_by_version["v1"] = self._make_response(b"old content") + + content = self.bucket.get_object_version("dir/file.txt", "v1") + + self.assertEqual(b"old content", content) + self.assertEqual([{"bucket_name": "test-bucket", "object_name": "dir/file.txt", "version_id": "v1"}], self.client.get_object_calls) + + def test_get_object_version_missing_version_raises_file_not_found(self) -> None: + self.client.get_object_error = self._make_s3_error("NoSuchVersion") + + with self.assertRaises(FileNotFoundError): + self.bucket.get_object_version("dir/file.txt", "missing") + + def test_get_object_version_delete_marker_raises_file_not_found(self) -> None: + self.client.get_object_error = self._make_s3_error("MethodNotAllowed") + + with self.assertRaises(FileNotFoundError): + self.bucket.get_object_version("dir/file.txt", "delete-marker-version") + + def test_remove_object_all_versions_deletes_listed_versions(self) -> None: + self.client.list_objects_response = [ + self._make_object("dir/file.txt", "v2", is_latest=True), + self._make_object("dir/file.txt", "v1", is_delete_marker=True), + ] + + errors = self.bucket.remove_object_with_versions("dir/file.txt") + + self.assertEqual([], list(errors)) + self.assertEqual("test-bucket", self.client.remove_objects_calls[0][0]) + self.assertEqual( + [("dir/file.txt", "v2"), ("dir/file.txt", "v1")], + [(obj.name, obj.version_id) for obj in self.client.remove_objects_calls[0][1]], + ) + + def test_remove_object_all_versions_without_versions_does_not_delete(self) -> None: + errors = self.bucket.remove_object_with_versions("dir/file.txt") + + self.assertEqual([], list(errors)) + self.assertEqual([], self.client.remove_objects_calls) + + class TestIntegratedMinioBucket(TestCase): def setUp(self) -> None: self.assertIsNotNone(CONFIG.MINIO_PUBLIC_SERVER, "MINIO_PUBLIC_SERVER not set") @@ -20,7 +147,9 @@ def setUp(self) -> None: self.bucket = MinioBucket(bucket_name=CONFIG.MINIO_DEV_TESTS_BUCKET, minio_client=self.minio_client) if not self.minio_client.bucket_exists(CONFIG.MINIO_DEV_TESTS_BUCKET): self.minio_client.make_bucket(bucket_name=CONFIG.MINIO_DEV_TESTS_BUCKET) + self.versioned_bucket = MinioBucket("versioned") self.tester = IBucketTester(self.bucket, self) + self.versioned_tester = IBucketTester(self.versioned_bucket, self) def tearDown(self) -> None: self.tester.cleanup() @@ -114,3 +243,120 @@ def test_regression_exception_thrown_in_open_write_context_by_AMX(self) -> None: def test_regression_infinite_cycle_on_unentered_open_write_context(self): self.tester.test_regression_infinite_cycle_on_unentered_open_write_context() + + +class TestIntegratedVersionedMinioBucket(TestCase): + def setUp(self) -> None: + self.assertIsNotNone(CONFIG.MINIO_PUBLIC_SERVER, "MINIO_PUBLIC_SERVER not set") + self.assertIsNotNone(CONFIG.MINIO_ACCESS_KEY, "MINIO_ACCESS_KEY not set") + self.assertIsNotNone(CONFIG.MINIO_SECRET_KEY, "MINIO_SECRET_KEY not set") + self.minio_client = build_minio_client( + endpoints=CONFIG.MINIO_PUBLIC_SERVER, access_key=CONFIG.MINIO_ACCESS_KEY, secret_key=CONFIG.MINIO_SECRET_KEY, timeout=30 + ) + self.bucket = MinioBucket(bucket_name=CONFIG.MINIO_DEV_TESTS_BUCKET, minio_client=self.minio_client) + if not self.minio_client.bucket_exists(CONFIG.MINIO_DEV_TESTS_BUCKET): + self.minio_client.make_bucket(bucket_name=CONFIG.MINIO_DEV_TESTS_BUCKET) + self.tester = VersionedIBucketTester(self.bucket, self) + + def tearDown(self) -> None: + self.tester.cleanup() + + def test_full_cycle_object_versions_after_overwrite(self): + self.tester.test_full_cycle_object_versions_after_overwrite() + + def test_open_write_sync_creates_readable_version(self): + self.tester.test_open_write_sync_creates_readable_version() + + def test_remove_objects_for_missing_name_does_not_create_version_history(self): + self.tester.test_remove_objects_for_missing_name_does_not_create_version_history() + + def test_invalid_names_raise_for_version_methods(self): + self.tester.test_invalid_names_raise_for_version_methods() + + +## @advlad this is gonna be removed +class TestIntegratedMinioBucketVersioning(TestCase): + def setUp(self) -> None: + self.assertIsNotNone(CONFIG.MINIO_PUBLIC_SERVER, "MINIO_PUBLIC_SERVER not set") + self.assertIsNotNone(CONFIG.MINIO_ACCESS_KEY, "MINIO_ACCESS_KEY not set") + self.assertIsNotNone(CONFIG.MINIO_SECRET_KEY, "MINIO_SECRET_KEY not set") + self.minio_client = build_minio_client( + endpoints=CONFIG.MINIO_PUBLIC_SERVER, access_key=CONFIG.MINIO_ACCESS_KEY, secret_key=CONFIG.MINIO_SECRET_KEY, timeout=30 + ) + self.bucket_name = self._make_locked_bucket_name() + self.minio_client.make_bucket(bucket_name=self.bucket_name, object_lock=True) + self.bucket = MinioBucket(bucket_name=self.bucket_name, minio_client=self.minio_client) + self._ensure_bucket_versioning_enabled() + self.tester = IBucketTester(self.bucket, self) + + def tearDown(self) -> None: + self._remove_all_bucket_versions() + self.minio_client.remove_bucket(self.bucket_name) + + @staticmethod + def _make_locked_bucket_name() -> str: + suffix = f"-versioning-{uuid.uuid4().hex[:12]}" + prefix = CONFIG.MINIO_DEV_TESTS_BUCKET[: 63 - len(suffix)].rstrip(".-") + return f"{prefix or 'bucketbase'}{suffix}" + + def _remove_all_bucket_versions(self) -> None: + objects = list(self.minio_client.list_objects(self.bucket_name, recursive=True, include_version=True)) + if not objects: + return + + delete_objects = [DeleteObject(MinioBucket._get_object_name(obj), obj.version_id) for obj in objects] + errors = list(self.minio_client.remove_objects(self.bucket_name, delete_objects)) + self.assertEqual([], errors) + + def _ensure_bucket_versioning_enabled(self) -> None: + versioning_config = self.minio_client.get_bucket_versioning(self.bucket_name) + if versioning_config.status != ENABLED: + self.minio_client.set_bucket_versioning(self.bucket_name, VersioningConfig(ENABLED)) + + def _put_two_object_versions(self, path: PurePosixPath) -> tuple[str, str]: + self.bucket.put_object(path, b"old content") + self.bucket.put_object(path, b"new content") + + versions = list(self.bucket.list_object_versions(path)) + latest_versions = [version for version in versions if version.is_latest and not version.is_delete_marker] + old_versions = [version for version in versions if not version.is_latest and not version.is_delete_marker] + + self.assertEqual(2, len(versions)) + self.assertEqual(1, len(latest_versions)) + self.assertEqual(1, len(old_versions)) + return old_versions[0].version_id, latest_versions[0].version_id + + def test_list_object_versions(self) -> None: + path = PurePosixPath(f"dir{self.tester.us}/integrated-version-list.txt") + + old_version_id, latest_version_id = self._put_two_object_versions(path) + versions = list(self.bucket.list_object_versions(path)) + + self.assertEqual(2, len(versions)) + self.assertEqual({old_version_id, latest_version_id}, {version.version_id for version in versions}) + self.assertTrue(all(version.name == path for version in versions)) + self.assertEqual(1, len([version for version in versions if version.is_latest])) + self.assertFalse(any(version.is_delete_marker for version in versions)) + + def test_get_object_version(self) -> None: + path = PurePosixPath(f"dir{self.tester.us}/integrated-version-read.txt") + + old_version_id, latest_version_id = self._put_two_object_versions(path) + + self.assertEqual(b"new content", self.bucket.get_object(path)) + self.assertEqual(b"old content", self.bucket.get_object_version(path, old_version_id)) + self.assertEqual(b"new content", self.bucket.get_object_version(path, latest_version_id)) + + def test_remove_object_all_versions(self) -> None: + path = PurePosixPath(f"dir{self.tester.us}/integrated-version-delete.txt") + + self._put_two_object_versions(path) + self.bucket.remove_objects([path]) + versions_before_delete = list(self.bucket.list_object_versions(path)) + + errors = self.bucket.remove_object_with_versions(path) + + self.assertEqual([], list(errors)) + self.assertTrue(any(version.is_delete_marker for version in versions_before_delete)) + self.assertFalse(self.bucket.exists(path)) + self.assertEqual([], list(self.bucket.list_object_versions(path)))