From 4ba622713d670f8bc926525017079d4288dd8f77 Mon Sep 17 00:00:00 2001 From: Brunno Vanelli Date: Sun, 12 Apr 2026 21:26:43 +0200 Subject: [PATCH 1/2] fix: Correctly calculate redirect urls for resolve functionality. --- crpy/cmd.py | 9 +++++-- crpy/common.py | 15 ++++-------- crpy/registry.py | 62 ++++++++++++++++++++++++++++++++---------------- 3 files changed, 52 insertions(+), 34 deletions(-) diff --git a/crpy/cmd.py b/crpy/cmd.py index 6c18b7b..a66ed5f 100644 --- a/crpy/cmd.py +++ b/crpy/cmd.py @@ -8,6 +8,7 @@ from rich.table import Table from rich.text import Text + from crpy.common import HTTPConnectionError, UnauthorizedError from crpy.registry import RegistryInfo from crpy.storage import ( @@ -137,14 +138,18 @@ async def _resolve(args): table = Table(title=f"Endpoints for {args.url[0]}", title_style="bold") table.add_column("Role", style="magenta", no_wrap=True) - table.add_column("Hostname", style="cyan") table.add_column("IPs", style="green", overflow="fold") table.add_column("URL", overflow="fold") for entry in entries: url_text = Text(entry.url) url_text.stylize(f"link {entry.url}") - table.add_row(entry.role, entry.hostname or "", ", ".join(entry.ips), url_text) + table.add_row(entry.role, ", ".join(entry.ips), url_text) + if entry.redirect: + redirect_text = Text(entry.redirect.url) + redirect_text.stylize(f"link {entry.redirect.url}") + table.add_row(f"{entry.role} (redirect)", ", ".join(entry.redirect.ips), redirect_text) + print(table) diff --git a/crpy/common.py b/crpy/common.py index 47f8c75..61be711 100644 --- a/crpy/common.py +++ b/crpy/common.py @@ -3,7 +3,6 @@ import hashlib import io import json -import socket from dataclasses import dataclass from typing import List, Optional, Union @@ -15,7 +14,6 @@ class Response: status: int data: bytes headers: Optional[dict] = None - real_url: Optional[str] = None def json(self) -> dict: return json.loads(self.data) @@ -34,9 +32,7 @@ async def _request( async with aiohttp.ClientSession(trust_env=True) as session: method_fn = getattr(session, method) async with method_fn(url, headers=headers, params=params, data=data, **aiohttp_kwargs) as response: - return Response( - response.status, await response.read(), dict(response.headers), str(response.request_info.real_url) - ) + return Response(response.status, await response.read(), dict(response.headers)) except aiohttp.ClientConnectionError as e: raise HTTPConnectionError(str(e)) @@ -114,12 +110,9 @@ async def resolve_hostname(hostname: str) -> List[str]: :param hostname: the hostname to resolve. :return: sorted list of unique IP address strings. """ - try: - loop = asyncio.get_running_loop() - results = await loop.getaddrinfo(hostname, None) - return sorted({r[4][0] for r in results}) - except socket.gaierror: - return [] + loop = asyncio.get_running_loop() + results = await loop.getaddrinfo(hostname, None) + return sorted({r[4][0] for r in results}) # exceptions diff --git a/crpy/registry.py b/crpy/registry.py index 03c23d3..a353d08 100644 --- a/crpy/registry.py +++ b/crpy/registry.py @@ -51,17 +51,16 @@ @dataclass class FirewallEntry: role: str - request_url: str + url: str ips: List[str] = field(default_factory=list) - redirect_url: Optional[str] = None + redirect: Optional["FirewallEntry"] = None @property - def url(self) -> str: - return self.redirect_url or self.request_url - - @property - def hostname(self) -> Optional[str]: - return urlparse(self.url).hostname + def hostname(self) -> str: + parsed = urlparse(self.url) + if parsed.hostname is None: + raise ValueError(f"Could not parse url {self.url}") + return parsed.hostname @dataclass @@ -113,16 +112,19 @@ async def _request_with_auth( params: dict = None, data: Union[dict, bytes, None] = None, headers: dict = None, + aiohttp_kwargs: dict = None, ) -> Response: if not headers: headers = {} + if not aiohttp_kwargs: + aiohttp_kwargs = {} response = await _request( url, {**headers, **self._headers}, params=params, data=data, method=method, - aiohttp_kwargs=self._aiohttp_kwargs, + aiohttp_kwargs=self._aiohttp_kwargs | aiohttp_kwargs, ) if response.status == 401: www_auth = response.headers["WWW-Authenticate"] @@ -620,10 +622,16 @@ async def delete_tag(self) -> Response: response = await self._request_with_auth(url, headers=self._headers, method="delete") return response - async def _head_entry(self, role: str, blob_url: str) -> FirewallEntry: - response = await self._request_with_auth(blob_url, method="head", headers=self._headers) - redirect_url = response.real_url if response.real_url and response.real_url != blob_url else None - return FirewallEntry(role, blob_url, redirect_url=redirect_url) + async def _resolve_entry(self, role: str, blob_url: str) -> FirewallEntry: + # Use GET with allow_redirects=False to capture CDN redirects (e.g. cdn01.quay.io). + response = await self._request_with_auth( + blob_url, headers=self._headers, method="get", aiohttp_kwargs={"allow_redirects": False} + ) + redirect_url = ( + str(response.headers.get("Location")) if response.headers and 300 <= response.status < 400 else None + ) + redirect = FirewallEntry(role, redirect_url, redirect=None) if redirect_url else None + return FirewallEntry(role, blob_url, redirect=redirect) async def resolve(self, architecture: Union[str, "Platform", None] = None) -> List[FirewallEntry]: """ @@ -643,25 +651,37 @@ async def resolve(self, architecture: Union[str, "Platform", None] = None) -> Li :param architecture: optional architecture for the image. :return: list of FirewallEntry objects with role, request URL, redirect URL, hostname and resolved IPs. """ + # multiple queries can be done for retrieving the manifest, but they all go to the same url manifest = await self.get_default_manifest(architecture) - entries: List[FirewallEntry] = [ - FirewallEntry("registry", self.manifest_url()), - ] - + entries: List[FirewallEntry] = [] if self.auth_server_url: entries.append(FirewallEntry("auth", self.auth_server_url)) + entries.append(FirewallEntry("manifest", self.manifest_url())) + # we retrieve the config config_digest = manifest["config"]["digest"] - entries.append(await self._head_entry("config", f"{self.blobs_url()}/{config_digest}")) + entries.append(await self._resolve_entry("config", f"{self.blobs_url()}/{config_digest}")) + # then each individual layer for idx, layer in enumerate(await self.get_layers(architecture)): - entries.append(await self._head_entry(f"layer-{idx}", f"{self.blobs_url()}/{layer}")) + entries.append(await self._resolve_entry(f"layer-{idx}", f"{self.blobs_url()}/{layer}")) - unique_hostnames = {e.hostname for e in entries if e.hostname} + # unwrap all entries once in case there were any redirects + unwrapped_entries: List[FirewallEntry] = [] + for entry in entries: + unwrapped_entries.append(entry) + if entry.redirect: + unwrapped_entries.append(entry.redirect) + + # compute hostnames + unique_hostnames = set() + for e in unwrapped_entries: + unique_hostnames.add(e.hostname) resolved = await asyncio.gather(*(resolve_hostname(h) for h in unique_hostnames)) ip_map = dict(zip(unique_hostnames, resolved)) - for entry in entries: + for entry in unwrapped_entries: entry.ips = ip_map.get(entry.hostname, []) + # return the original entries object (nested) return entries From 7a3887a6d42cc64c8d11e62d710c190c478e4a6d Mon Sep 17 00:00:00 2001 From: Brunno Vanelli Date: Sun, 12 Apr 2026 21:41:41 +0200 Subject: [PATCH 2/2] fix: Support multiple ip versions and update docs. --- README.md | 62 +++++++++++++++++++++++++++++------------------- crpy/cmd.py | 8 ++++++- crpy/common.py | 7 ++++-- crpy/registry.py | 11 ++++++--- 4 files changed, 57 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 04e6bb8..f79601a 100644 --- a/README.md +++ b/README.md @@ -134,31 +134,43 @@ useful when you need to configure firewall rules, proxy allowlists, or DNS polic pulls often hit multiple hosts (registry, auth server, CDN) that all need to be reachable: ```bash -$ crpy resolve alpine:latest - Endpoints for alpine:latest -┏━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Role ┃ Hostname ┃ IPs ┃ URL ┃ -┡━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ registry │ index.docker.io │ 100.50.185.129, │ https://index.docker.io/v2/li │ -│ │ │ 174.129.222.113, 3.81.188.6, │ brary/alpine/manifests/latest │ -│ │ │ 44.208.12.140, 52.71.174.30, │ │ -│ │ │ 52.86.153.188, 54.147.201.31, │ │ -│ │ │ 54.196.196.77 │ │ -│ auth │ auth.docker.io │ 104.18.43.178, 172.64.144.78 │ https://auth.docker.io/token? │ -│ │ │ │ service=registry.docker.io&sc │ -│ │ │ │ ope=repository:library/alpine │ -│ │ │ │ :pull │ -│ config │ index.docker.io │ 100.50.185.129, │ https://index.docker.io/v2/li │ -│ │ │ 174.129.222.113, 3.81.188.6, │ brary/alpine/blobs/sha256:a40 │ -│ │ │ 44.208.12.140, 52.71.174.30, │ c03cbb81c59bfb0e0887ab0b18597 │ -│ │ │ 52.86.153.188, 54.147.201.31, │ 27075da7b9cc576a1cec2c771f38c │ -│ │ │ 54.196.196.77 │ 5fb │ -│ layer-0 │ index.docker.io │ 100.50.185.129, │ https://index.docker.io/v2/li │ -│ │ │ 174.129.222.113, 3.81.188.6, │ brary/alpine/blobs/sha256:589 │ -│ │ │ 44.208.12.140, 52.71.174.30, │ 002ba0eaed121a1dbf42f6648f29e │ -│ │ │ 52.86.153.188, 54.147.201.31, │ 5be55d5c8a6ee0f8eaa0285cc21ac │ -│ │ │ 54.196.196.77 │ 153 │ -└──────────┴─────────────────┴───────────────────────────────┴───────────────────────────────┘ +$ crpy resolve -4 alpine:latest + Endpoints for alpine:latest +┏━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Role ┃ IPs ┃ URL ┃ +┡━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ auth │ 104.18.43.178, 172.64.144.78 │ https://auth.docker.io/token?service=regi │ +│ │ │ stry.docker.io&scope=repository:library/a │ +│ │ │ lpine:pull │ +├────────────────────┼───────────────────────────────────────────┼───────────────────────────────────────────┤ +│ manifest │ 32.192.123.231, 32.195.147.39, │ https://index.docker.io/v2/library/alpine │ +│ │ 34.206.220.186, 44.194.100.18, │ /manifests/latest │ +│ │ 52.71.123.245, 54.152.111.129, │ │ +│ │ 54.210.213.255, 98.94.122.193 │ │ +├────────────────────┼───────────────────────────────────────────┼───────────────────────────────────────────┤ +│ config │ 32.192.123.231, 32.195.147.39, │ https://index.docker.io/v2/library/alpine │ +│ │ 34.206.220.186, 44.194.100.18, │ /blobs/sha256:a40c03cbb81c59bfb0e0887ab0b │ +│ │ 52.71.123.245, 54.152.111.129, │ 1859727075da7b9cc576a1cec2c771f38c5fb │ +│ │ 54.210.213.255, 98.94.122.193 │ │ +├────────────────────┼───────────────────────────────────────────┼───────────────────────────────────────────┤ +│ config (redirect) │ 172.64.66.1 │ https://docker-images-prod.6aa30f8b08e164 │ +│ │ │ 09b46e0173d6de2f56.r2.cloudflarestorage.c │ +│ │ │ om/registry-v2/docker/registry/v2/blobs/s │ +│ │ │ ha256/a4/a40c03cbb81c59bfb0e0887ab0b18597 │ +│ │ │ 27075da7b9cc576a1cec2c771f38c5fb/data?... │ +├────────────────────┼───────────────────────────────────────────┼───────────────────────────────────────────┤ +│ layer-0 │ 32.192.123.231, 32.195.147.39, │ https://index.docker.io/v2/library/alpine │ +│ │ 34.206.220.186, 44.194.100.18, │ /blobs/sha256:589002ba0eaed121a1dbf42f664 │ +│ │ 52.71.123.245, 54.152.111.129, │ 8f29e5be55d5c8a6ee0f8eaa0285cc21ac153 │ +│ │ 54.210.213.255, 98.94.122.193 │ │ +├────────────────────┼───────────────────────────────────────────┼───────────────────────────────────────────┤ +│ layer-0 (redirect) │ 172.64.66.1 │ https://docker-images-prod.6aa30f8b08e164 │ +│ │ │ 09b46e0173d6de2f56.r2.cloudflarestorage.c │ +│ │ │ om/registry-v2/docker/registry/v2/blobs/s │ +│ │ │ ha256/58/589002ba0eaed121a1dbf42f6648f29e │ +│ │ │ 5be55d5c8a6ee0f8eaa0285cc21ac153/data?... │ +└────────────────────┴───────────────────────────────────────────┴───────────────────────────────────────────┘ + ``` # Why creating this package? diff --git a/crpy/cmd.py b/crpy/cmd.py index a66ed5f..2f786cf 100644 --- a/crpy/cmd.py +++ b/crpy/cmd.py @@ -134,7 +134,8 @@ async def _auth(args): async def _resolve(args): ri = RegistryInfo.from_url(args.url[0], proxy=args.proxy, insecure=args.insecure) - entries = await ri.resolve(args.architecture[0] if args.architecture else None) + ip_version = 6 if args.ipv6 else (4 if args.ipv4 else 0) + entries = await ri.resolve(args.architecture[0] if args.architecture else None, ip_version=ip_version) table = Table(title=f"Endpoints for {args.url[0]}", title_style="bold") table.add_column("Role", style="magenta", no_wrap=True) @@ -145,10 +146,12 @@ async def _resolve(args): url_text = Text(entry.url) url_text.stylize(f"link {entry.url}") table.add_row(entry.role, ", ".join(entry.ips), url_text) + table.add_section() if entry.redirect: redirect_text = Text(entry.redirect.url) redirect_text.stylize(f"link {entry.redirect.url}") table.add_row(f"{entry.role} (redirect)", ", ".join(entry.redirect.ips), redirect_text) + table.add_section() print(table) @@ -312,6 +315,9 @@ def main(*args): help="Architecture for the image.", default=None, ) + ip_group = resolve.add_mutually_exclusive_group() + ip_group.add_argument("-4", dest="ipv4", action="store_true", default=False, help="Resolve IPv4 addresses only.") + ip_group.add_argument("-6", dest="ipv6", action="store_true", default=False, help="Resolve IPv6 addresses only.") # version version = subparsers.add_parser("version", help="Displays the application version.") version.set_defaults(func=_version) diff --git a/crpy/common.py b/crpy/common.py index 61be711..221fbef 100644 --- a/crpy/common.py +++ b/crpy/common.py @@ -3,6 +3,7 @@ import hashlib import io import json +import socket from dataclasses import dataclass from typing import List, Optional, Union @@ -103,15 +104,17 @@ def platform_from_dict(platform: dict) -> str: return base_str -async def resolve_hostname(hostname: str) -> List[str]: +async def resolve_hostname(hostname: str, family: int = socket.AF_UNSPEC) -> List[str]: """ Resolves a hostname to a sorted list of unique IP addresses. :param hostname: the hostname to resolve. + :param family: socket address family to filter results. Use ``socket.AF_INET`` for IPv4 only, + ``socket.AF_INET6`` for IPv6 only, or ``socket.AF_UNSPEC`` (default) for both. :return: sorted list of unique IP address strings. """ loop = asyncio.get_running_loop() - results = await loop.getaddrinfo(hostname, None) + results = await loop.getaddrinfo(hostname, None, family=family) return sorted({r[4][0] for r in results}) diff --git a/crpy/registry.py b/crpy/registry.py index a353d08..1fca24d 100644 --- a/crpy/registry.py +++ b/crpy/registry.py @@ -4,11 +4,12 @@ import json import pathlib import re +import socket import sys import tarfile import tempfile from dataclasses import dataclass, field -from typing import List, Optional, Union +from typing import List, Optional, Union, Literal from urllib.parse import urlparse from async_lru import alru_cache @@ -633,7 +634,9 @@ async def _resolve_entry(self, role: str, blob_url: str) -> FirewallEntry: redirect = FirewallEntry(role, redirect_url, redirect=None) if redirect_url else None return FirewallEntry(role, blob_url, redirect=redirect) - async def resolve(self, architecture: Union[str, "Platform", None] = None) -> List[FirewallEntry]: + async def resolve( + self, architecture: Union[str, "Platform", None] = None, ip_version: Literal[4, 6, 0] = 0 + ) -> List[FirewallEntry]: """ Performs a dry-run pull to discover every network endpoint that a real pull would contact. Executes authentication, manifest fetch, and HEAD requests for config and layer blobs — without downloading any data. @@ -649,6 +652,7 @@ async def resolve(self, architecture: Union[str, "Platform", None] = None) -> Li ``` :param architecture: optional architecture for the image. + :param ip_version: IP version filter. Use `4` for IPv4 only, `6` for IPv6 only, or `0` (default) for both. :return: list of FirewallEntry objects with role, request URL, redirect URL, hostname and resolved IPs. """ # multiple queries can be done for retrieving the manifest, but they all go to the same url @@ -678,7 +682,8 @@ async def resolve(self, architecture: Union[str, "Platform", None] = None) -> Li unique_hostnames = set() for e in unwrapped_entries: unique_hostnames.add(e.hostname) - resolved = await asyncio.gather(*(resolve_hostname(h) for h in unique_hostnames)) + family = {4: socket.AF_INET, 6: socket.AF_INET6}.get(ip_version, socket.AF_UNSPEC) + resolved = await asyncio.gather(*(resolve_hostname(h, family=family) for h in unique_hostnames)) ip_map = dict(zip(unique_hostnames, resolved)) for entry in unwrapped_entries: entry.ips = ip_map.get(entry.hostname, [])