diff --git a/httpxthrottlecache/httpxclientmanager.py b/httpxthrottlecache/httpxclientmanager.py index e854829..8271d2e 100644 --- a/httpxthrottlecache/httpxclientmanager.py +++ b/httpxthrottlecache/httpxclientmanager.py @@ -11,7 +11,7 @@ import httpx from httpx._types import ProxyTypes -from pyrate_limiter import Duration, Limiter +from pyrate_limiter import Limiter from .filecache.transport import CachingTransport from .ratelimiter import AsyncRateLimitingTransport, RateLimitingTransport, create_rate_limiter @@ -40,7 +40,6 @@ class HttpxThrottleCache: rate_limiter_enabled: bool = True cache_mode: Literal[False, "Disabled", "FileCache"] = "FileCache" request_per_sec_limit: int = 10 - max_delay: Duration = field(default_factory=lambda: Duration.DAY) _client: Optional[httpx.Client] = None rate_limiter: Optional[Limiter] = None @@ -62,7 +61,7 @@ def __post_init__(self): if self.rate_limiter_enabled and self.rate_limiter is None: self.rate_limiter = create_rate_limiter( - requests_per_second=self.request_per_sec_limit, max_delay=self.max_delay + requests_per_second=self.request_per_sec_limit ) if (self.cache_mode != "Disabled" or self.cache_mode is False) and not self.cache_rules: @@ -162,8 +161,9 @@ async def task(url: str, path: Optional[Path]) -> Path | bytes: def _get_httpx_transport_params(self, params: dict[str, Any]): http2 = params.get("http2", False) proxy = self.proxy + verify = params.get("verify", True) - return {"http2": http2, "proxy": proxy} + return {"http2": http2, "proxy": proxy, "verify": verify} @contextmanager def http_client(self, bypass_cache: bool = False, **kwargs: dict[str, Any]) -> Generator[httpx.Client, None, None]: @@ -191,8 +191,8 @@ def close(self): self._client.close() self._client = None - def update_rate_limiter(self, requests_per_second: int, max_delay: Duration = Duration.DAY): - self.rate_limiter = create_rate_limiter(requests_per_second=requests_per_second, max_delay=Duration.DAY) + def update_rate_limiter(self, requests_per_second: int): + self.rate_limiter = create_rate_limiter(requests_per_second=requests_per_second) self.close() diff --git a/httpxthrottlecache/ratelimiter.py b/httpxthrottlecache/ratelimiter.py index 826747a..b17855e 100644 --- a/httpxthrottlecache/ratelimiter.py +++ b/httpxthrottlecache/ratelimiter.py @@ -6,22 +6,14 @@ from typing import Any import httpx -from pyrate_limiter import Duration, InMemoryBucket, Limiter, Rate +from pyrate_limiter import Duration, Limiter, Rate logger = logging.getLogger(__name__) -def create_rate_limiter(requests_per_second: int, max_delay: Duration | int = Duration.DAY) -> Limiter: +def create_rate_limiter(requests_per_second: int) -> Limiter: rate = Rate(requests_per_second, Duration.SECOND) - rate_limits = [rate] - - base_bucket = InMemoryBucket(rate_limits) - - bucket = base_bucket - - limiter = Limiter(bucket, max_delay=max_delay, raise_when_fail=False, retry_until_max_delay=True) - - return limiter + return Limiter(rate) class RateLimitingTransport(httpx.HTTPTransport): @@ -33,9 +25,7 @@ def handle_request(self, request: httpx.Request, **kwargs: dict[str, Any]) -> ht # using a constant string for item name means that the same # rate is applied to all requests. if self.limiter: - while not self.limiter.try_acquire(__name__): - logger.debug("Lock acquisition timed out, retrying") # pragma: no cover - + self.limiter.try_acquire(__name__) logger.debug("Acquired lock") logger.info("Making HTTP Request %s", request) @@ -49,9 +39,7 @@ def __init__(self, limiter: Limiter, **kwargs: dict[str, Any]): async def handle_async_request(self, request: httpx.Request, **kwargs: dict[str, Any]) -> httpx.Response: if self.limiter: - while not await self.limiter.try_acquire_async(__name__): - logger.debug("Lock acquisition timed out, retrying") # pragma: no cover - + await self.limiter.try_acquire_async(__name__) logger.debug("Acquired lock") logger.info("Making HTTP Request %s", request) diff --git a/pyproject.toml b/pyproject.toml index c0ce254..2951789 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ "aiofiles>=24.1.0", "filelock>=3.18.0", "httpx>=0.28.1", - "pyrate-limiter>=3.9.0", + "pyrate-limiter>=4.0.0", ] classifiers = [ "Programming Language :: Python :: 3.10", diff --git a/tests/test_misc2.py b/tests/test_misc2.py index 1c3d97e..1fa975c 100644 --- a/tests/test_misc2.py +++ b/tests/test_misc2.py @@ -17,6 +17,27 @@ def test_user_agent_factor(tmp_path): htc._populate_user_agent(params) assert params["headers"]["User-Agent"] == "foo" + +def test_verify_passed_to_transport(tmp_path): + """Issue #37: verify parameter should be forwarded to transport""" + htc = HttpxThrottleCache( + cache_dir=tmp_path, + cache_mode="Disabled", + httpx_params={"verify": False, "http2": False}, + rate_limiter_enabled=False, + ) + transport_params = htc._get_httpx_transport_params(htc.httpx_params) + assert transport_params["verify"] is False + + htc_default = HttpxThrottleCache( + cache_dir=tmp_path, + cache_mode="Disabled", + rate_limiter_enabled=False, + ) + transport_params_default = htc_default._get_httpx_transport_params(htc_default.httpx_params) + assert transport_params_default["verify"] is True + + def test_no_ratelimit(manager_cache): htc = HttpxThrottleCache(cache_mode=manager_cache.cache_mode, cache_dir=manager_cache.cache_dir, user_agent_factory=lambda: "foo", rate_limiter_enabled=False) url = "https://example.com/file.bin" diff --git a/uv.lock b/uv.lock index fec197a..0016dba 100644 --- a/uv.lock +++ b/uv.lock @@ -385,7 +385,7 @@ requires-dist = [ { name = "aiofiles", specifier = ">=24.1.0" }, { name = "filelock", specifier = ">=3.18.0" }, { name = "httpx", specifier = ">=0.28.1" }, - { name = "pyrate-limiter", specifier = ">=3.9.0" }, + { name = "pyrate-limiter", specifier = ">=4.0.0" }, ] [package.metadata.requires-dev] @@ -1296,11 +1296,11 @@ wheels = [ [[package]] name = "pyrate-limiter" -version = "3.9.0" +version = "4.0.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ec/da/f682c5c5f9f0a5414363eb4397e6b07d84a02cde69c4ceadcbf32c85537c/pyrate_limiter-3.9.0.tar.gz", hash = "sha256:6b882e2c77cda07a241d3730975daea4258344b39c878f1dd8849df73f70b0ce", size = 289308, upload-time = "2025-07-30T14:36:58.659Z" } +sdist = { url = "https://files.pythonhosted.org/packages/03/98/2b3dc1ba6bdf2efaeaa3e102124cbd2636a4ccec241ffeb8a1de207f5cd4/pyrate_limiter-4.0.2.tar.gz", hash = "sha256:b678841e2215f114ef6f98c7093755ca3b466de83cb5a881231fd6e321fa14b5", size = 301304, upload-time = "2026-01-23T09:37:33.612Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/04/af/d8bf0959ece9bc4679bd203908c31019556a421d76d8143b0c6871c7f614/pyrate_limiter-3.9.0-py3-none-any.whl", hash = "sha256:77357840c8cf97a36d67005d4e090787043f54000c12c2b414ff65657653e378", size = 33628, upload-time = "2025-07-30T14:36:57.71Z" }, + { url = "https://files.pythonhosted.org/packages/13/b9/80ffe3f2c34d3247186d74b1d08c1fed1e3ad4127ff6a8a5501b7bf16a97/pyrate_limiter-4.0.2-py3-none-any.whl", hash = "sha256:35ec42b9bb9cfabcafab14d0c5c6523f48378c3da2949e534ce3cbdfea71eadd", size = 36439, upload-time = "2026-01-23T09:37:32.097Z" }, ] [[package]]