diff --git a/pyproject.toml b/pyproject.toml index adf72ee..ba1006f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,9 +76,7 @@ quote-style = "double" [tool.pyrefly] project-includes = ["src", "tests"] min-severity = "info" -preset = "strict" -strict-callable-subtyping = true -errors = { implicit-any = true, implicit-any-attribute = true, implicit-any-empty-container = true, implicit-any-parameter = true, implicit-any-type-argument = true, unannotated-parameter = true, unannotated-attribute = true, unannotated-return = true, unannotated-protocol-member = true, bad-override = true, bad-override-mutable-attribute = true, bad-override-param-name = true, bad-param-name-override = true, missing-override-decorator = true, inconsistent-inheritance = true, unused-ignore = true, unused-coroutine = true, redundant-cast = true, unreachable = true, redefinition = true, deprecated = true } +preset = "all" [tool.commitizen] name = "cz_customize" diff --git a/src/composekit/container.py b/src/composekit/container.py new file mode 100644 index 0000000..4359eae --- /dev/null +++ b/src/composekit/container.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, fields + +Volume = str | dict[str, object] + + +@dataclass +class Container: + image: str + folder: str | None = None + name: str | None = None + restart: str | None = None + privileged: bool | None = None + network: str | None = None + network_mode: str | None = None + working_dir: str | None = None + command: str | None = None + entrypoint: str | None = None + user: str | None = None + shm_size: str | None = None + cap_add: list[str] | None = None + cap_drop: list[str] | None = None + group_add: list[str] | None = None + sysctls: list[str] | None = None + devices: list[str] | None = None + volumes: list[Volume] | None = None + tmpfs: list[str] | None = None + environment: list[str] | None = None + ports: list[str] | None = None + depends_on: list[str] | None = None + labels: list[str] | dict[str, str] | None = None + healthcheck: dict[str, object] | None = None + + @classmethod + def fields(cls) -> list[str]: + return list(field.name for field in fields(cls)) + + @classmethod + def from_dict(cls, data: Mapping[str, object]) -> Container: + image = data.get("image") + if not isinstance(image, str): + raise TypeError("image must be a string") + + container = cls(image=image) + for key, value in data.items(): + if key == "image": + continue + + if key in cls.fields(): + setattr(container, key, value) + + return container + + def to_dict(self) -> dict[str, object]: + return { + key: getattr(self, key) + for key in self.fields() + if getattr(self, key) is not None + } + + +def load_containers(documents: Iterable[object]) -> list[Container]: + containers: list[Container] = [] + for document in documents: + if not isinstance(document, Mapping): + raise TypeError("container entry must be a mapping") + + data: dict[str, object] = {} + for key, value in document.items(): + if not isinstance(key, str): + raise TypeError("container keys must be strings") + + data[key] = value + + containers.append(Container.from_dict(data)) + + return containers diff --git a/src/composekit/generate.py b/src/composekit/generate.py index 80ea167..261307f 100755 --- a/src/composekit/generate.py +++ b/src/composekit/generate.py @@ -5,11 +5,12 @@ import shutil from collections.abc import Sequence from importlib.resources import files -from typing import Any, ClassVar +from typing import ClassVar try: import yaml + from composekit.container import Container, Volume, load_containers from composekit.utils import Config as _Config from composekit.utils import iter_container_files, open_repo except ImportError as err: @@ -20,7 +21,7 @@ class Config(_Config): config_paths = ("config/generate.yaml",) - default_values: ClassVar[dict[str, str | bool]] = { + default_values: ClassVar[dict[str, object]] = { "containers_folder": "containers", "composes_folder": "composes", "network_name": "cloud", @@ -34,32 +35,6 @@ class Config(_Config): } -OPTIONS = ( - "folder", - "name", - "image", - "privileged", - "network", - "working_dir", - "command", - "network_mode", - "user", - "entrypoint", - "cap_add", - "cap_drop", - "sysctls", - "labels", - "devices", - "volumes", - "tmpfs", - "environment", - "group_add", - "depends_on", - "healthcheck", - "ports", - "shm_size", -) - MOUNT_OPTIONS = { "rw", "ro", @@ -77,14 +52,10 @@ class Config(_Config): } -def get_folder_name( - name: str, container: dict[str, Any], config: Config -) -> str: - folder = str(container.get("folder", name)) +def get_folder_name(name: str, container: Container, config: Config) -> str: + folder = container.folder or name mode = config["capitalize_folder_name"] - if mode == "full" or ( - mode == "non_custom" and not container.get("folder") - ): + if mode == "full" or (mode == "non_custom" and not container.folder): folder = capitalize_name(folder) return folder @@ -94,7 +65,7 @@ def capitalize_name(name: str) -> str: return name[0].upper() + name[1:] -def is_custom_bind(volume: str | dict[str, Any]) -> bool: +def is_custom_bind(volume: Volume) -> bool: if isinstance(volume, dict): return True @@ -112,13 +83,13 @@ def is_custom_bind(volume: str | dict[str, Any]) -> bool: def handle_volumes( config: Config, folder: str, - volumes: Sequence[str | dict[str, Any]], + volumes: Sequence[Volume], used_volumes: list[str], -) -> list[str | dict[str, Any]]: +) -> list[Volume]: bind_path = str(config["bind_path"]) use_full_directory = bool(config["use_full_directory"]) custom_binds = list(filter(is_custom_bind, volumes)) - result: list[str | dict[str, Any]] = [] + result: list[Volume] = [] for volume in volumes: if isinstance(volume, dict): @@ -164,26 +135,28 @@ def duplicate_entries(entries: list[str]) -> list[str]: def generate( - name: str, container: dict[str, Any], config: Config -) -> dict[str, Any]: + name: str, container: Container, config: Config +) -> dict[str, object]: folder = get_folder_name(name, container, config) restart_policy = str(config["restart_policy"]) network = str(config["network_name"]) used_volumes: list[str] = [] - result: dict[str, Any] = { - "image": container.pop("image"), + result: dict[str, object] = { + "image": container.image, "hostname": name, "container_name": name, - "restart": container.get("restart", restart_policy), + "restart": container.restart or restart_policy, } - for option in OPTIONS: - if option not in container or option in ("folder", "name"): + for option in Container.fields(): + if option in ("folder", "name", "image"): continue - value = container[option] + value = getattr(container, option) + if value is None: + continue match option: case "devices": @@ -197,8 +170,8 @@ def generate( case _: result[option] = value - if "network_mode" not in container: - result.setdefault("networks", []).append(network) + if not container.network_mode: + result["networks"] = [network] return result @@ -244,7 +217,7 @@ def main(args: argparse.Namespace) -> None: gateway=gateway, ) ) - main_template["services"] = dict[str, Any]() + main_template["services"] = dict[str, object]() composes_template = ( templates.joinpath("composes.yaml").read_text().lstrip() @@ -260,21 +233,21 @@ def main(args: argparse.Namespace) -> None: for path in paths: used_names: list[str] = [] - service: dict[str, dict[str, Any]] = yaml.safe_load( + service: dict[str, dict[str, object]] = yaml.safe_load( service_template.format(network=network) ) - service["services"] = dict[str, Any]() + service["services"] = dict[str, object]() with open(path) as file: - containers: list[dict[str, Any]] = list(yaml.safe_load_all(file)) + containers = load_containers(yaml.safe_load_all(file)) for container in containers: - name = str(container.get("name", path.stem)) + name = container.name or path.stem if name in used_names: number = str(used_names.count(name) + 1) - container["name"] = name = f"{name}_{number}" - if container.get("folder"): - container["folder"] += number + container.name = name = f"{name}_{number}" + if container.folder: + container.folder += number used_names.append(name) service["services"][name] = generate(name, container, config) diff --git a/src/composekit/sort.py b/src/composekit/sort.py index 80e1f72..10af990 100644 --- a/src/composekit/sort.py +++ b/src/composekit/sort.py @@ -3,9 +3,9 @@ import argparse import asyncio from pathlib import Path -from typing import Any -from .generate import OPTIONS, Config +from .container import Container, load_containers +from .generate import Config try: import yaml @@ -24,17 +24,21 @@ async def process_file( git_lock: asyncio.Lock, ) -> None: with open(path) as file: - containers: list[dict[str, Any]] = list(yaml.safe_load_all(file)) + containers = load_containers(yaml.safe_load_all(file)) - for i, container in enumerate(containers): - sorted_dict = {k: container[k] for k in OPTIONS if k in container} - if list(container.keys()) == list(sorted_dict.keys()): - continue + sorted_containers: list[dict[str, object]] = [] + changed = False - containers[i] = sorted_dict + for container in containers: + data = container.to_dict() + sorted_dict = {k: data[k] for k in Container.fields() if k in data} + sorted_containers.append(sorted_dict) + changed = changed or list(data.keys()) != list(sorted_dict.keys()) + + if changed: async with git_lock: with open(path, "w") as file: - yaml.dump_all(containers, file, sort_keys=False) + yaml.dump_all(sorted_containers, file, sort_keys=False) if repo is not None: repo.index.add(path) diff --git a/src/composekit/update.py b/src/composekit/update.py index 922a0c3..adaf4b3 100755 --- a/src/composekit/update.py +++ b/src/composekit/update.py @@ -6,7 +6,7 @@ import re import sys from pathlib import Path -from typing import Any, ClassVar +from typing import ClassVar try: import httpx @@ -14,6 +14,7 @@ from git import Repo from packaging.version import InvalidVersion, Version + from composekit.container import Container, load_containers from composekit.utils import Config as _Config from composekit.utils import iter_container_files, list_tags, open_repo except ImportError as err: @@ -29,7 +30,7 @@ class Config(_Config): config_paths = ("config/update.yaml", "config/update.private.yaml") - default_values: ClassVar[dict[str, str | int]] = { + default_values: ClassVar[dict[str, object]] = { "containers_folder": "containers", "default_registry": "docker.io", "limit": 40, @@ -87,25 +88,49 @@ def parse_version(version: str | None) -> Version | None: return None +def get_update_options( + config: Config, full_image: str, user: str | None, image: str +) -> dict[str, object]: + for item in (full_image, f"{user}/{image}", image): + config_data = config[item] + if isinstance(config_data, dict): + return { + key: value + for key, value in config_data.items() + if isinstance(key, str) + } + + return {} + + async def find_versions( config: Config, - container: dict[str, Any], + options: dict[str, object], client: httpx.AsyncClient, registry: str | None, user: str | None, image: str, ) -> list[str]: - limit = int(container.get("limit") or config["limit"]) + limit_config = options.get("limit") or config["limit"] + limit = ( + limit_config + if isinstance(limit_config, int) + else int(limit_config) + if isinstance(limit_config, str) and limit_config.isdigit() + else 10 + ) full_image = "/".join(filter(None, [registry, user, image])) try: user = "library" if user is None else user + username = options.get("username") + password = options.get("password") if tags := await list_tags( client, registry, f"{user}/{image}", - container.get("username"), - container.get("password"), + username if isinstance(username, str) else None, + password if isinstance(password, str) else None, ): return tags[-limit:] @@ -118,34 +143,26 @@ async def find_versions( async def update( config: Config, - container: dict[str, Any], + container: Container, client: httpx.AsyncClient, ) -> tuple[str, str, str] | None: - if not (result := parse_image(str(container["image"]))): + if not (result := parse_image(container.image)): return None registry, user, image, version = result full_image = "/".join(filter(None, [registry, user, image])) registry = registry or str(config["default_registry"]) - container = next( - ( - _data - for item in [ - full_image, - f"{user}/{image}", - image, - ] - if (_data := config[item]) and isinstance(_data, dict) - ), - {}, - ) + options = get_update_options(config, full_image, user, image) - if container.get("update") is False: + if options.get("update") is False: logging.info(f"{full_image}: Update is disabled.") return None - version_regex = container.get("version_regex") + version_regex_config = options.get("version_regex") + version_regex = ( + version_regex_config if isinstance(version_regex_config, str) else None + ) if not ( current_version := parse_version( @@ -159,7 +176,7 @@ async def update( if not ( raw_versions := await find_versions( - config, container, client, registry, user, image + config, options, client, registry, user, image ) ): return None @@ -189,18 +206,22 @@ async def process_file( git_lock: asyncio.Lock, ) -> None: with open(path) as file: - containers: list[dict[str, Any]] = list(yaml.safe_load_all(file)) + containers = load_containers(yaml.safe_load_all(file)) for container in containers: if not (result := await update(config, container, client)): continue full_image, image, newest_version = result - container["image"] = f"{full_image}:{newest_version}" + container.image = f"{full_image}:{newest_version}" async with git_lock: with open(path, "w") as file: - yaml.dump_all(containers, file, sort_keys=False) + yaml.dump_all( + [item.to_dict() for item in containers], + file, + sort_keys=False, + ) if repo is not None: repo.index.add(path) @@ -224,9 +245,11 @@ async def process() -> None: git_lock = asyncio.Lock() containers_folder = str(config["containers_folder"]) - + timeout = ( + int(_value) if (_value := str(config["timeout"])).isdigit() else 10 + ) async with httpx.AsyncClient( - timeout=int(config["timeout"]), follow_redirects=True + timeout=timeout, follow_redirects=True ) as client: await asyncio.gather( *( diff --git a/src/composekit/utils/config.py b/src/composekit/utils/config.py index cff442c..1956543 100644 --- a/src/composekit/utils/config.py +++ b/src/composekit/utils/config.py @@ -1,13 +1,13 @@ import os -from typing import Any, ClassVar +from typing import ClassVar import yaml class Config: - config: dict[str, Any] + config: dict[str, object] config_paths: ClassVar[tuple[str, ...]] = () - default_values: ClassVar[dict[str, Any]] = {} + default_values: ClassVar[dict[str, object]] = {} def __init__(self) -> None: self.config = {} @@ -21,10 +21,10 @@ def load(self, *paths: str) -> None: with open(path) as file: self.config.update(yaml.safe_load(file) or {}) - def __setitem__(self, key: str, value: Any) -> None: + def __setitem__(self, key: str, value: object) -> None: self.config[key] = value - def __getitem__(self, key: str) -> Any: + def __getitem__(self, key: str) -> object | None: return ( os.getenv(key.upper()) or self.config.get(key.lower()) diff --git a/tests/test_generate.py b/tests/test_generate.py index 1969b36..2d6241f 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -2,11 +2,11 @@ import tempfile import unittest from pathlib import Path -from typing import Any from unittest.mock import MagicMock import yaml +from composekit.container import Container from composekit.generate import ( Config, capitalize_name, @@ -40,7 +40,7 @@ def test_is_custom_bind(self) -> None: def test_handle_volumes_basic(self) -> None: config = make_mock_config() volumes = ["/volume", "/volume2"] - container: dict[str, Any] = {} + container = Container(image="nginx") folder = get_folder_name("container", container, config) result = handle_volumes(config, folder, volumes, []) expected = [ @@ -52,7 +52,7 @@ def test_handle_volumes_basic(self) -> None: def test_handle_volumes_with_custom_binds(self) -> None: config = make_mock_config() volumes = ["/volume:/volume", "/volume2:/volume2"] - container: dict[str, Any] = {} + container = Container(image="nginx") folder = get_folder_name("container", container, config) result = handle_volumes(config, folder, volumes, []) self.assertEqual(result, ["/volume:/volume", "/volume2:/volume2"]) @@ -60,7 +60,7 @@ def test_handle_volumes_with_custom_binds(self) -> None: def test_handle_volumes_with_mount_options_and_custom_name(self) -> None: config = make_mock_config() volumes = ["/volume:ro;config", "/volume2:rw;data"] - container: dict[str, Any] = {} + container = Container(image="nginx") folder = get_folder_name("container", container, config) result = handle_volumes(config, folder, volumes, []) self.assertEqual( @@ -82,7 +82,7 @@ def test_capitalize_name(self) -> None: def test_generate_minimal(self) -> None: config = make_mock_config() - container = {"image": "nginx"} + container = Container(image="nginx") result = generate("web", container, config) self.assertEqual(result["image"], "nginx") self.assertEqual(result["hostname"], "web") @@ -116,12 +116,29 @@ def test_main_handles_duplicate_containers_without_folder(self) -> None: list(compose["services"].keys()), ["container", "container_2"] ) + def test_generate_accepts_container(self) -> None: + config = make_mock_config() + container = Container(image="nginx", volumes=["/config"]) + result = generate("web", container, config) + self.assertEqual(result["image"], "nginx") + self.assertEqual(result["volumes"], ["/bind/web:/config"]) + self.assertEqual(result["networks"], ["cloud"]) + + def test_container_to_dict_uses_field_order(self) -> None: + container = Container.from_dict( + {"folder": "web", "image": "nginx", "ports": ["80:80"]} + ) + self.assertEqual( + list(container.to_dict()), + ["image", "folder", "ports"], + ) + def test_handle_volumes_with_full_capitalize(self) -> None: config = Config() config["bind_path"] = "${BIND_PATH}" config["capitalize_folder_name"] = "full" volumes = ["/volume", "/volume2"] - container: dict[str, Any] = {} + container = Container(image="nginx") folder = get_folder_name("container", container, config) result = handle_volumes(config, folder, volumes, []) self.assertEqual( @@ -137,7 +154,7 @@ def test_handle_volumes_with_non_custom_capitalize(self) -> None: config["bind_path"] = "${BIND_PATH}" config["capitalize_folder_name"] = "non_custom" volumes = ["/volume", "/volume2"] - container: dict[str, Any] = {"folder": "container"} + container = Container(image="nginx", folder="container") folder = get_folder_name("container", container, config) result = handle_volumes(config, folder, volumes, []) self.assertEqual( diff --git a/tests/test_update.py b/tests/test_update.py index 0ada37d..155bc9f 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -1,10 +1,10 @@ import unittest -from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import httpx from packaging.version import Version +from composekit.container import Container from composekit.update import ( extract_version, find_versions, @@ -76,7 +76,7 @@ class TestUpdate(unittest.IsolatedAsyncioTestCase): async def test_find_versions_mocked(self) -> None: config = MagicMock() config.__getitem__.side_effect = lambda key: {"limit": 2}[key] - container: dict[str, Any] = {} + options: dict[str, object] = {} registry = None user = "user" image = "image" @@ -86,7 +86,7 @@ async def test_find_versions_mocked(self) -> None: mock_list_tags.return_value = ["1.0.0", "1.1.0", "1.2.0"] async with httpx.AsyncClient() as client: result = await find_versions( - config, container, client, registry, user, image + config, options, client, registry, user, image ) self.assertTrue( result == ["1.1.0", "1.2.0"] @@ -100,10 +100,10 @@ async def test_update_new_version(self) -> None: "limit": 10, "timeout": 5, "user/image": {"update": True}, - "user": dict[str, Any](), - "image": dict[str, Any](), + "user": dict[str, object](), + "image": dict[str, object](), }[key] - container = {"image": "user/image:1.0.0"} + container = Container(image="user/image:1.0.0") with patch( "composekit.update.find_versions", new_callable=AsyncMock ) as mock_find: @@ -116,6 +116,26 @@ async def test_update_new_version(self) -> None: self.assertTrue(full_image.endswith("user/image")) self.assertEqual(image, "image") + async def test_update_accepts_container(self) -> None: + config = MagicMock() + config.__getitem__.side_effect = lambda key: { + "default_registry": "docker.io", + "limit": 10, + "timeout": 5, + "user/image": {"update": True}, + "user": dict[str, object](), + "image": dict[str, object](), + }[key] + container = Container(image="user/image:1.0.0") + with patch( + "composekit.update.find_versions", new_callable=AsyncMock + ) as mock_find: + mock_find.return_value = ["1.0.1", "1.0.2"] + result = await update(config, container, AsyncMock()) + if result is None: + self.fail("expected update result") + self.assertEqual(result[2], "1.0.2") + async def test_update_disabled(self) -> None: config = MagicMock() config.__getitem__.side_effect = lambda key: { @@ -124,6 +144,6 @@ async def test_update_disabled(self) -> None: "timeout": 5, "user/image": {"update": False}, }[key] - container = {"image": "user/image:1.0.0"} + container = Container(image="user/image:1.0.0") result = await update(config, container, AsyncMock()) self.assertIsNone(result)