diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..62ad3d5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,66 @@ +name: '🐛 Bug report' +description: Report an issue with crpy. +labels: [bug] + +body: + - type: checkboxes + id: checks + attributes: + label: Checks + options: + - label: I have checked that this issue has not already been reported. + required: true + - label: I have confirmed this bug exists on the latest version of crpy. + required: true + + - type: textarea + id: example + attributes: + label: Reproducible example + description: > + Please follow [this guide](https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) on how to + provide a minimal, copy-pastable example. Include the (wrong) output if applicable. + value: | + ```python + + ``` + validations: + required: true + + - type: textarea + id: logs + attributes: + label: Log output + description: > + Include the stack trace, if available, of the problem being reported. + render: shell + + - type: textarea + id: problem + attributes: + label: Issue description + description: > + Provide any additional information you think might be relevant. + validations: + required: true + + - type: textarea + id: expected-behavior + attributes: + label: Expected behavior + description: > + Describe or show a code example of the expected behavior. + validations: + required: true + + - type: textarea + id: version + attributes: + label: Installed versions + description: > + Describe which version (or if running to git version, which commit) of the Python library. + value: > + - crpy version: + - Python version: + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..ddd16a1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,14 @@ +name: '✨ Feature request' +description: Suggest a new feature or enhancement for crpy. +labels: [enhancement] + +body: + - type: textarea + id: description + attributes: + label: Description + description: > + Describe the feature or enhancement and explain why it should be implemented. + Include a code example if applicable. + validations: + required: true diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 39e3430..281a27c 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -3,6 +3,7 @@ on: # see https://github.com/orgs/community/discussions/25029#discussioncomment-3246275 release: types: [published] + workflow_dispatch: jobs: pypi-publish: name: Upload release to PyPI diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f0e79d8..74e74cc 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,4 +1,4 @@ -name: Run tests +name: Tests on: pull_request: @@ -6,12 +6,11 @@ on: branches: [ main ] jobs: - build: - + test: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v3 @@ -22,14 +21,18 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install ruff pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi - name: Lint with ruff run: | # stop the build if there are Python syntax errors or undefined names - ruff --select=E9,F63,F7,F82 --target-version=py37 . + ruff check --select=E9,F63,F7,F82 --target-version=py39 . # default set of ruff rules with GitHub Annotations - ruff --target-version=py37 . + ruff check --target-version=py39 . - name: Test with pytest run: | pytest + # - name: Upload coverage reports to Codecov + # uses: codecov/codecov-action@v4 + # with: + # token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/try-build.yaml b/.github/workflows/try-build.yaml new file mode 100644 index 0000000..062f10c --- /dev/null +++ b/.github/workflows/try-build.yaml @@ -0,0 +1,18 @@ +name: Test package for import errors and dependency installation +on: + push: + branches: [ main ] +jobs: + try-build: + name: Tries to build the package and import it + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install locally + run: python3 -m pip install . + - name: Test library import + run: cd / && python3 -c "from crpy import RegistryInfo" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 97f8578..4f7f695 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,30 +1,16 @@ repos: - repo: 'https://github.com/charliermarsh/ruff-pre-commit' - rev: v0.0.285 + rev: v0.12.4 hooks: - - id: ruff + - id: ruff-check args: - - '--line-length=120' - '--fix' - '--exit-non-zero-on-fix' + - id: ruff-format - repo: 'https://github.com/pre-commit/pre-commit-hooks' - rev: v4.4.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer + - id: check-yaml - id: check-added-large-files - - repo: 'https://github.com/pycqa/isort' - rev: 5.12.0 - hooks: - - id: isort - name: isort (python) - args: - - '--profile' - - black - - '--filter-files' - - repo: 'https://github.com/psf/black' - rev: 23.7.0 - hooks: - - id: black - args: - - '--line-length=120' diff --git a/.ruff.toml b/.ruff.toml deleted file mode 100644 index 301f4fb..0000000 --- a/.ruff.toml +++ /dev/null @@ -1,3 +0,0 @@ -line-length = 120 -[mccabe] -max-complexity = 18 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7e8619e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.13-slim-bookworm + +WORKDIR /app +COPY . . +RUN pip install . && pip cache purge && rm -rf /app/* + +ENTRYPOINT ["crpy"] diff --git a/README.md b/README.md index 86595a4..cf0e721 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,21 @@ The script creates a cache directory (~/.crpy/) to store layers already download It was based on a simpler version called [sdenel/docker-pull-push](https://github.com/sdenel/docker-pull-push), but has since received so many changes that it does not resemble the original code anymore. -# Basic usage +# Installation + +You can install it from the official pip repository: + +```bash +pip install crpy +``` + +If you want to live on the edge and have the latest development features, install it directly from the repo: + +```bash +pip install git+https://github.com/bvanelli/crpy.git +``` + +# Basic CLI usage TODO: Fill in once the "final" version of the API is stable. For a preview of the options, here is the help command: @@ -46,18 +60,58 @@ optional arguments: For reporting issues visit https://github.com/bvanelli/crpy ``` -# Installation +One of the original intended usages was to run it CI to cache dependencies docker image (i.e. for Gitlab). In this +case, we can check if the image already exists on the remote repository: -You can install it from the official pip repository: +```bash +$ crpy manifest alpine:1.2.3 +Authenticated at index.docker.io/library/alpine:latest +{'errors': [{'code': 'MANIFEST_UNKNOWN', 'message': 'manifest unknown', 'detail': 'unknown tag=1.2.3'}]} +``` + +You are also able to download images and save them to disk: ```bash -pip install crpy +$ crpy pull alpine:latest alpine.tar.gz +latest: Pulling from index.docker.io/library/alpine +Authenticated at index.docker.io/library/alpine:latest +Using cache for layer 9824c27679d3 +9824c27679d3: Pull complete +Downloaded image from index.docker.io/library/alpine:latest ``` -If you want to live on the edge and have the latest development features, install it directly from the repo: +On can then push this image to another repository: ```bash -pip install git+https://github.com/bvanelli/crpy.git +$ crpy push alpine.tar.gz bvanelli/test:latest +crpy push alpine.tar.gz bvanelli/test:latest +The push refers to repository +Authenticated at index.docker.io/bvanelli/test:latest +Authenticated at index.docker.io/bvanelli/test:latest +9824c27679d3: Pushed +Pushed latest: digest: sha256:3f372403810ab0506dda12549f1035804192ef02fb36040c036845f90bd6bfe2 +``` + +Let's now list the tags available at this repository: + +```bash +$ crpy tags bvanelli/test +Authenticated at index.docker.io/bvanelli/test:latest +1.0.0 +latest +``` + +And delete one of the tags. I show this example because both tags were the same, and deleting one will delete them both, +so use this command with caution: + +```bash +$ crpy delete bvanelli/test:1.0.0 +crpy delete bvanelli/test:1.0.0 +Authenticated at index.docker.io/bvanelli/test:1.0.0 +Authenticated at index.docker.io/bvanelli/test:1.0.0 +b'' +$ crpy tags bvanelli/test +Authenticated at index.docker.io/bvanelli/test:latest ``` # Why creating this package? diff --git a/codecov.yaml b/codecov.yaml new file mode 100644 index 0000000..5077db3 --- /dev/null +++ b/codecov.yaml @@ -0,0 +1,5 @@ +coverage: + status: + project: + default: + threshold: 5% diff --git a/crpy/__init__.py b/crpy/__init__.py index e69de29..539249d 100644 --- a/crpy/__init__.py +++ b/crpy/__init__.py @@ -0,0 +1,14 @@ +from crpy.registry import RegistryInfo +from crpy.image import Blob, Image +from crpy.common import HTTPConnectionError, UnauthorizedError, BaseCrpyError +from crpy.version import __version__ + +__all__ = [ + "RegistryInfo", + "Blob", + "Image", + "HTTPConnectionError", + "UnauthorizedError", + "BaseCrpyError", + "__version__", +] diff --git a/crpy/cmd.py b/crpy/cmd.py index 0245605..1b7cfc5 100644 --- a/crpy/cmd.py +++ b/crpy/cmd.py @@ -130,6 +130,12 @@ async def _auth(args): print(table) +async def _version(_args) -> None: + from crpy import __version__ + + print(__version__) + + def main(*args): parser = argparse.ArgumentParser( prog="crpy", @@ -260,6 +266,9 @@ def main(*args): default="index.docker.io", ) delete.set_defaults(func=_delete) + # version + version = subparsers.add_parser("version", help="Displays the application version.") + version.set_defaults(func=_version) arguments = parser.parse_args(args if args else None) diff --git a/crpy/image.py b/crpy/image.py index 2206379..056f019 100644 --- a/crpy/image.py +++ b/crpy/image.py @@ -89,7 +89,7 @@ def layers(self, layers: List[INPUT_TYPES]): def to_disk(self, filename: pathlib.Path, tags: List[str] = None): with tempfile.TemporaryDirectory() as temp_dir: web_manifest = self.manifest.as_dict() - config_filename = f'{web_manifest["config"]["digest"].split(":")[1]}.json' + config_filename = f"{web_manifest['config']['digest'].split(':')[1]}.json" with open(f"{temp_dir}/{config_filename}", "wb") as outfile: outfile.write(self.config.as_bytes()) @@ -101,6 +101,10 @@ def to_disk(self, filename: pathlib.Path, tags: List[str] = None): os.makedirs(f"{temp_dir}/{layer_folder}", exist_ok=True) with open(f"{temp_dir}/{path}", "wb") as f: f.write(layer_bytes) + # add version for backwards compatibility + # https://github.com/moby/moby/blob/daa4618da826fb1de4fc2478d88196edbba49b2f/image/spec/v1.md + with open(f"{temp_dir}/{layer_folder}/VERSION", "w") as f: + f.write("1.0") layer_path_l.append(path) manifest = [{"Config": config_filename, "RepoTags": tags or [], "Layers": layer_path_l}] diff --git a/crpy/registry.py b/crpy/registry.py index c336cb0..de5b126 100644 --- a/crpy/registry.py +++ b/crpy/registry.py @@ -237,6 +237,14 @@ async def get_manifest(self, fat: bool = False, reference: Optional[str] = None) return response async def get_manifest_from_architecture(self, architecture: Union[str, Platform, None] = None) -> dict: + """ + Returns the manifest as a dictionary for a given architecture. If no architecture is specified, it will return + the default manifest, which **might contain multiple entries**. If you want to only get the default manifest, + use `get_default_manifest()` instead. + + :param architecture: optional architecture for the image. + :return: dictionary representation of the manifest. + """ if isinstance(architecture, Platform): architecture = architecture.value elif isinstance(architecture, str): @@ -265,6 +273,22 @@ async def get_manifest_from_architecture(self, architecture: Union[str, Platform manifest = await self.get_manifest() return manifest.json() + @alru_cache + async def get_default_manifest(self, architecture: Union[str, Platform] = None) -> dict: + """ + Same as `get_manifest_from_architecture()`, but will return one and only one manifest. If the architecture is + not specified, it will return the default manifest. + + :param architecture: optional architecture for the image. + :return: dictionary representation of the manifest. + """ + manifest = await self.get_manifest_from_architecture(architecture) + # manifest should be a single entry - if not the case (i.e., oci images), retrieve the default amd64 version + if manifest["mediaType"] in (_ociv1_index_mimetype, _schema2_list_mimetype): + default_platform = Platform.from_dict(manifest["manifests"][0]["platform"]) + manifest = await self.get_manifest_from_architecture(default_platform) + return manifest + @alru_cache async def get_config(self, architecture: Union[str, Platform] = None) -> Response: """ @@ -275,11 +299,7 @@ async def get_config(self, architecture: Union[str, Platform] = None) -> Respons will be pulled. :return: Response object with status code, raw data and response headers. """ - manifest = await self.get_manifest_from_architecture(architecture) - # manifest should be a single entry - if not the case (i.e. oci images, retrieve the amd64 version - if manifest["mediaType"] in (_ociv1_index_mimetype, _schema2_list_mimetype): - default_platform = Platform.from_dict(manifest["manifests"][0]["platform"]) - manifest = await self.get_manifest_from_architecture(default_platform) + manifest = await self.get_default_manifest(architecture) config_digest = manifest["config"]["digest"] response = await self._request_with_auth( f"{self.blobs_url()}/{config_digest}", method="get", headers=self._headers @@ -294,7 +314,7 @@ async def get_layers(self, architecture: Union[str, Platform, None] = None) -> L will be pulled. :return: """ - manifest = await self.get_manifest_from_architecture(architecture) + manifest = await self.get_default_manifest(architecture) layers = [m["digest"] for m in manifest["layers"]] return layers @@ -356,7 +376,7 @@ async def pull( """ print(f"{self.tag}: Pulling from {self.registry}/{self.repository}") image = Image() - image.manifest = await self.get_manifest_from_architecture(architecture) + image.manifest = await self.get_default_manifest(architecture) raw_config = await self.get_config(architecture) image.config = raw_config.data for layer in await self.get_layers(architecture): diff --git a/crpy/version.py b/crpy/version.py new file mode 100644 index 0000000..0a67244 --- /dev/null +++ b/crpy/version.py @@ -0,0 +1,2 @@ +__version_info__ = ("0", "2", "0") +__version__ = ".".join(__version_info__) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..bcb15e0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,51 @@ +[build-system] +requires = ["setuptools>=61.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "crpy" +description = "Simple and straight forward async library for interacting with container registry API." +readme = "README.md" +authors = [ + { name = "Brunno Vanelli", email = "brunnovanelli@gmail.com" } +] +requires-python = ">=3.9.0" +dependencies = [ + "requests>=2", + "async-lru>=2", + "aiohttp>=3", + "async-lru>=2", + "rich>=13.5", +] +classifiers = [ + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +keywords = ["docker", "oci", "container-registry", "api"] +dynamic = ["version"] + +[project.urls] +Homepage = "https://github.com/bvanelli/crpy" +Repository = "https://github.com/bvanelli/crpy.git" +"Bug Tracker" = "https://github.com/bvanelli/crpy/issues" + +[project.scripts] +crpy = "crpy.cmd:main" + +[tool.pytest.ini_options] +asyncio_default_fixture_loop_scope = "function" +asyncio_mode = "auto" + +[tool.setuptools.packages.find] +exclude = ["docs*", "tests*", "examples*"] + +[tool.setuptools.dynamic] +version = { attr = "crpy.version.__version__" } + +[tool.ruff] +line-length = 120 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..66c9786 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,3 @@ +pytest-asyncio>=1 +ruff>=0.12 +pytest>=8 diff --git a/setup.py b/setup.py deleted file mode 100644 index 1b0807d..0000000 --- a/setup.py +++ /dev/null @@ -1,21 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="crpy", - version="0.0.1", - packages=find_packages(), - description="Simple and straight forward async library for interacting with container registry API.", - long_description=open("README.md").read(), - long_description_content_type="text/markdown", - author="Brunno Vanelli", - author_email="brunnovanelli@gmail.com", - url="https://github.com/bvanelli/crpy", - zip_safe=False, - project_urls={ - "Issues": "https://github.com/bvanelli/crpy/issues", - }, - entry_points=""" - [console_scripts] - crpy=crpy.cmd:main - """, -) diff --git a/tests/test_auth.py b/tests/test_auth.py index 852178f..b96d139 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,9 +1,6 @@ -import pytest - from crpy.registry import RegistryInfo -@pytest.mark.asyncio async def test_auth(): registry = RegistryInfo.from_url("alpine") token = await registry.auth() diff --git a/tests/test_registry_endpoints.py b/tests/test_registry_endpoints.py index 3f56a23..1f46288 100644 --- a/tests/test_registry_endpoints.py +++ b/tests/test_registry_endpoints.py @@ -1,13 +1,11 @@ import io import tarfile -import pytest from crpy.common import Platform, compute_sha256 from crpy.registry import RegistryInfo -@pytest.mark.asyncio async def test_pull_docker_io(): file = io.BytesIO() ri = RegistryInfo.from_url("index.docker.io/library/alpine:3.18.2") @@ -20,11 +18,10 @@ async def test_pull_docker_io(): assert len([layer for layer in content if layer.endswith("layer.tar")]) -@pytest.mark.asyncio async def test_api_calls(): ri = RegistryInfo.from_url("index.docker.io/library/alpine:3.18.2") fat_manifest = (await ri.get_manifest(fat=True)).json() - manifest = (await ri.get_manifest()).json() + manifest = await ri.get_default_manifest() assert manifest["config"]["digest"] == "sha256:c1aabb73d2339c5ebaa3681de2e9d9c18d57485045a4e311d9f8004bec208d67" # the digest of the fat manifest should match the one in # https://hub.docker.com/layers/library/alpine/3.18.2/images/ @@ -47,7 +44,6 @@ async def test_api_calls(): assert sha_256_layer == layers[0] -@pytest.mark.asyncio async def test_list_tags(): ri = RegistryInfo.from_url("index.docker.io/library/alpine") tags = await ri.list_tags()