diff --git a/README.md b/README.md index cf0e721..9943db1 100644 --- a/README.md +++ b/README.md @@ -25,16 +25,18 @@ pip install git+https://github.com/bvanelli/crpy.git # Basic CLI usage -TODO: Fill in once the "final" version of the API is stable. For a preview of the options, here is the help command: +For a preview of the options, here is the help command: ``` usage: crpy [-h] [-k] [-p PROXY] - {pull,push,login,logout,auth,manifest,config,commands,layer,repositories,tags,delete} ... + {pull,push,login,logout,auth,manifest,config,commands,layer,repositories,tags,delete,resolve,version} + ... -Package that can do basic docker command like pull and push without installing the docker virtual machine +Package that can do basic docker command like pull and push without installing +the docker virtual machine positional arguments: - {pull,push,login,logout,auth,manifest,config,commands,layer,repositories,tags,delete} + {pull,push,login,logout,auth,manifest,config,commands,layer,repositories,tags,delete,resolve,version} pull Pulls a docker image from a remove repo. push Pushes a docker image from a remove repo. login Logs in on a remote repo @@ -42,20 +44,27 @@ positional arguments: auth Shows authenticated repositories manifest Inspects a docker registry metadata. config Inspects a docker registry metadata. - commands Inspects a docker registry build commands. These are the same as when you check individual - image layers on Docker hub. + commands Inspects a docker registry build commands. These are + the same as when you check individual image layers on + Docker hub. layer Inspects a docker registry layer. repositories List the repositories on the registry. tags List the tags on a repository. delete Deletes a tag in a remote repo. + resolve Dry-run a pull to discover every endpoint (registry, + auth, CDN) and resolve their IPs. Useful for + configuring firewall rules, proxy allowlists, or DNS + policies in restricted networks. + version Displays the application version. -optional arguments: +options: -h, --help show this help message and exit - -k, --insecure Use insecure registry. Ignores the validation of the certificate (useful for development - registries). + -k, --insecure Use insecure registry. Ignores the validation of the + certificate (useful for development registries). -p PROXY, --proxy PROXY - Proxy for all requests. If your proxy contains authentication, pass it on the request in the - usual format "http://user:pass@some.proxy.com" + Proxy for all requests. If your proxy contains + authentication, pass it on the request in the usual + format "http://user:pass@some.proxy.com" For reporting issues visit https://github.com/bvanelli/crpy ``` @@ -114,6 +123,38 @@ $ crpy tags bvanelli/test Authenticated at index.docker.io/bvanelli/test:latest ``` +You can also discover every network endpoint that a pull would contact, without actually downloading any data. This is +useful when you need to configure firewall rules, proxy allowlists, or DNS policies in restricted networks — container +pulls often hit multiple hosts (registry, auth server, CDN) that all need to be reachable: + +```bash +$ crpy resolve alpine:latest + Endpoints for alpine:latest +┏━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Role ┃ Hostname ┃ IPs ┃ URL ┃ +┡━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ registry │ index.docker.io │ 100.50.185.129, │ https://index.docker.io/v2/li │ +│ │ │ 174.129.222.113, 3.81.188.6, │ brary/alpine/manifests/latest │ +│ │ │ 44.208.12.140, 52.71.174.30, │ │ +│ │ │ 52.86.153.188, 54.147.201.31, │ │ +│ │ │ 54.196.196.77 │ │ +│ auth │ auth.docker.io │ 104.18.43.178, 172.64.144.78 │ https://auth.docker.io/token? │ +│ │ │ │ service=registry.docker.io&sc │ +│ │ │ │ ope=repository:library/alpine │ +│ │ │ │ :pull │ +│ config │ index.docker.io │ 100.50.185.129, │ https://index.docker.io/v2/li │ +│ │ │ 174.129.222.113, 3.81.188.6, │ brary/alpine/blobs/sha256:a40 │ +│ │ │ 44.208.12.140, 52.71.174.30, │ c03cbb81c59bfb0e0887ab0b18597 │ +│ │ │ 52.86.153.188, 54.147.201.31, │ 27075da7b9cc576a1cec2c771f38c │ +│ │ │ 54.196.196.77 │ 5fb │ +│ layer-0 │ index.docker.io │ 100.50.185.129, │ https://index.docker.io/v2/li │ +│ │ │ 174.129.222.113, 3.81.188.6, │ brary/alpine/blobs/sha256:589 │ +│ │ │ 44.208.12.140, 52.71.174.30, │ 002ba0eaed121a1dbf42f6648f29e │ +│ │ │ 52.86.153.188, 54.147.201.31, │ 5be55d5c8a6ee0f8eaa0285cc21ac │ +│ │ │ 54.196.196.77 │ 153 │ +└──────────┴─────────────────┴───────────────────────────────┴───────────────────────────────┘ +``` + # Why creating this package? Essentially, I wanted to learn how docker handles docker image pushing and pulling, and I ended up also implementing @@ -123,27 +164,27 @@ what is going on under the hood, take a look at I understand that there are many other good solutions out there, I'll list them here: -- [**DXF**](https://github.com/davedoesdev/dxf) (python): module with command line to interact with the registry. While -some functionality is the same, **DXF does not allow to pull and save entire images, only blobs**. This means images +- [**DXF**](https://github.com/davedoesdev/dxf) (python): module with a command line to interact with the registry. While +some functionality is the same, **DXF does not allow pulling and saving entire images, only blobs**. This means images will not run again once pulled from the registry. -- [**docker-ls**](https://github.com/mayflower/docker-ls) (go): module with command line to manipulate docker +- [**docker-ls**](https://github.com/mayflower/docker-ls) (go): module with a command line to manipulate docker registries, focusing on listing repositories and tags. Also allows removal of tags, but **does not allow pushing and pulling**. -- [**registry-cli**](https://github.com/andrey-pohilko/registry-cli) (python): module with command line to manipulate -docker registries. Allows removal of tags by regex, with configurable filters and number of images to keep. but **does +- [**registry-cli**](https://github.com/andrey-pohilko/registry-cli) (python): module with a command line to manipulate +docker registries. Allows removal of tags by regex, with configurable filters and a number of images to keep. but **does not allow pushing and pulling**. Also, the codebase was written without type-hinting, which makes using it as an API a bit more difficult. There are also production-ready solutions: -- [**skopeo**](https://github.com/containers/skopeo) (go): very wide range of supported registries and formats. It also -implements interactions with the docker daemon, so that you can interact even with already pulled images. It can also -inspect repositories, manifests and configs. -- [**crane**](https://github.com/google/go-containerregistry/tree/main/cmd/crane) (go): also very wide of support of +- [**skopeo**](https://github.com/containers/skopeo) (go): vast range of supported registries and formats. It also +implements interactions with the docker daemon so that you can interact even with already pulled images. It can also +inspect repositories, manifests, and configs. +- [**crane**](https://github.com/google/go-containerregistry/tree/main/cmd/crane) (go): also vast of support of registry interaction. Seems to also focus on the efficiency of doing operations. -I see nothing wrong with the available solutions, but if you are looking for a code based approach, you want to use -python AND you want to use async code (like every other cool kid on the block), there are no real alternatives to +I see nothing wrong with the available solutions. However, if you are looking for a code-based approach, you want to use +python, AND you want to use async code (like every other cool kid on the block), there are no real alternatives to interact with registries. Therefore, I started this little project to fill the gap. If you know of any other alternative tools, feel free to open an issue or directly place a merge request editing this diff --git a/crpy/cmd.py b/crpy/cmd.py index 56cb616..6c18b7b 100644 --- a/crpy/cmd.py +++ b/crpy/cmd.py @@ -6,6 +6,7 @@ from rich import print from rich.table import Table +from rich.text import Text from crpy.common import HTTPConnectionError, UnauthorizedError from crpy.registry import RegistryInfo @@ -130,6 +131,23 @@ async def _auth(args): print(table) +async def _resolve(args): + ri = RegistryInfo.from_url(args.url[0], proxy=args.proxy, insecure=args.insecure) + entries = await ri.resolve(args.architecture[0] if args.architecture else None) + + table = Table(title=f"Endpoints for {args.url[0]}", title_style="bold") + table.add_column("Role", style="magenta", no_wrap=True) + table.add_column("Hostname", style="cyan") + table.add_column("IPs", style="green", overflow="fold") + table.add_column("URL", overflow="fold") + + for entry in entries: + url_text = Text(entry.url) + url_text.stylize(f"link {entry.url}") + table.add_row(entry.role, entry.hostname or "", ", ".join(entry.ips), url_text) + print(table) + + async def _version(_args) -> None: from crpy import __version__ @@ -170,7 +188,7 @@ def main(*args): "--arch", "--platform", nargs=1, - help="Architecture for the to be pulled.", + help="Architecture for the image.", default=None, ) pull.add_argument( @@ -230,7 +248,7 @@ def main(*args): "--arch", "--platform", nargs=1, - help="Architecture to retrieve the manifest for.", + help="Architecture for the image.", default=None, ) manifest.add_argument("url", nargs=1, help="Remote repository url.") @@ -272,6 +290,23 @@ def main(*args): default="index.docker.io", ) delete.set_defaults(func=_delete) + # resolve + resolve = subparsers.add_parser( + "resolve", + help="Dry-run a pull to discover every endpoint (registry, auth, CDN) and resolve their IPs. " + "Useful for configuring firewall rules, proxy allowlists, or DNS policies in restricted networks.", + ) + resolve.set_defaults(func=_resolve) + resolve.add_argument("url", nargs=1, help="Remote repository url.") + resolve.add_argument( + "--architecture", + "-a", + "--arch", + "--platform", + nargs=1, + help="Architecture for the image.", + default=None, + ) # version version = subparsers.add_parser("version", help="Displays the application version.") version.set_defaults(func=_version) diff --git a/crpy/common.py b/crpy/common.py index 036f5ad..47f8c75 100644 --- a/crpy/common.py +++ b/crpy/common.py @@ -1,9 +1,11 @@ +import asyncio import enum import hashlib import io import json +import socket from dataclasses import dataclass -from typing import Optional, Union +from typing import List, Optional, Union import aiohttp @@ -13,6 +15,7 @@ class Response: status: int data: bytes headers: Optional[dict] = None + real_url: Optional[str] = None def json(self) -> dict: return json.loads(self.data) @@ -31,7 +34,9 @@ async def _request( async with aiohttp.ClientSession(trust_env=True) as session: method_fn = getattr(session, method) async with method_fn(url, headers=headers, params=params, data=data, **aiohttp_kwargs) as response: - return Response(response.status, await response.read(), dict(response.headers)) + return Response( + response.status, await response.read(), dict(response.headers), str(response.request_info.real_url) + ) except aiohttp.ClientConnectionError as e: raise HTTPConnectionError(str(e)) @@ -102,6 +107,21 @@ def platform_from_dict(platform: dict) -> str: return base_str +async def resolve_hostname(hostname: str) -> List[str]: + """ + Resolves a hostname to a sorted list of unique IP addresses. + + :param hostname: the hostname to resolve. + :return: sorted list of unique IP address strings. + """ + try: + loop = asyncio.get_running_loop() + results = await loop.getaddrinfo(hostname, None) + return sorted({r[4][0] for r in results}) + except socket.gaierror: + return [] + + # exceptions class BaseCrpyError(Exception): pass diff --git a/crpy/registry.py b/crpy/registry.py index ae818f1..03c23d3 100644 --- a/crpy/registry.py +++ b/crpy/registry.py @@ -1,3 +1,4 @@ +import asyncio import functools import io import json @@ -6,8 +7,9 @@ import sys import tarfile import tempfile -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import List, Optional, Union +from urllib.parse import urlparse from async_lru import alru_cache from rich import print as rprint @@ -20,6 +22,7 @@ _stream, compute_sha256, platform_from_dict, + resolve_hostname, ) from crpy.image import Blob, Image from crpy.storage import get_credentials, get_layer_from_cache, save_layer @@ -45,6 +48,22 @@ print = functools.partial(rprint, file=sys.stderr) +@dataclass +class FirewallEntry: + role: str + request_url: str + ips: List[str] = field(default_factory=list) + redirect_url: Optional[str] = None + + @property + def url(self) -> str: + return self.redirect_url or self.request_url + + @property + def hostname(self) -> Optional[str]: + return urlparse(self.url).hostname + + @dataclass class RegistryInfo: """ @@ -67,6 +86,8 @@ class RegistryInfo: # networking options proxy: Optional[str] = None insecure: bool = False + # Where the authentication url was resolved + auth_server_url: Optional[str] = None @property def _headers(self) -> dict: @@ -155,9 +176,11 @@ async def auth( # check if config contains username and password we can use if not b64_token and use_config: b64_token = get_credentials(self.registry) + auth_url = get_url_from_auth_header(www_auth) + self.auth_server_url = auth_url # reuse this token in consecutive requests self.token = await get_token( - get_url_from_auth_header(www_auth), + auth_url, username=username, password=password, b64_token=b64_token, @@ -596,3 +619,49 @@ async def delete_tag(self) -> Response: url = f"{self.v2_url()}/{self.repository}/manifests/{reference}" response = await self._request_with_auth(url, headers=self._headers, method="delete") return response + + async def _head_entry(self, role: str, blob_url: str) -> FirewallEntry: + response = await self._request_with_auth(blob_url, method="head", headers=self._headers) + redirect_url = response.real_url if response.real_url and response.real_url != blob_url else None + return FirewallEntry(role, blob_url, redirect_url=redirect_url) + + async def resolve(self, architecture: Union[str, "Platform", None] = None) -> List[FirewallEntry]: + """ + Performs a dry-run pull to discover every network endpoint that a real pull would contact. Executes + authentication, manifest fetch, and HEAD requests for config and layer blobs — without downloading any data. + Each endpoint is resolved to its IP addresses via DNS. + + This is useful for configuring firewall rules, proxy allowlists, or DNS policies in restricted networks, + since container pulls often hit multiple hosts (registry, auth server, CDN) that all need to be reachable. + + ```python + ri = RegistryInfo.from_url("alpine:latest") + for entry in await ri.resolve(): + print(entry.role, entry.hostname, entry.ips) + ``` + + :param architecture: optional architecture for the image. + :return: list of FirewallEntry objects with role, request URL, redirect URL, hostname and resolved IPs. + """ + manifest = await self.get_default_manifest(architecture) + + entries: List[FirewallEntry] = [ + FirewallEntry("registry", self.manifest_url()), + ] + + if self.auth_server_url: + entries.append(FirewallEntry("auth", self.auth_server_url)) + + config_digest = manifest["config"]["digest"] + entries.append(await self._head_entry("config", f"{self.blobs_url()}/{config_digest}")) + + for idx, layer in enumerate(await self.get_layers(architecture)): + entries.append(await self._head_entry(f"layer-{idx}", f"{self.blobs_url()}/{layer}")) + + unique_hostnames = {e.hostname for e in entries if e.hostname} + resolved = await asyncio.gather(*(resolve_hostname(h) for h in unique_hostnames)) + ip_map = dict(zip(unique_hostnames, resolved)) + for entry in entries: + entry.ips = ip_map.get(entry.hostname, []) + + return entries