From b19eb334ac4f941da37ae2c76ea9413f53409ec8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 26 Feb 2026 15:58:48 +0000 Subject: [PATCH 1/2] Initial plan From 1d2dd863a6f4e97e2ff622d56c382bad38345766 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 26 Feb 2026 16:11:45 +0000 Subject: [PATCH 2/2] feat: add retry logic, configurable timeout, and client-side rate limiting Co-authored-by: numbers-official <181934381+numbers-official@users.noreply.github.com> --- python/numbersprotocol_capture/client.py | 139 ++++++++--- python/numbersprotocol_capture/types.py | 12 + python/tests/test_retry_and_resilience.py | 278 ++++++++++++++++++++++ ts/src/client.test.ts | 228 ++++++++++++++++++ ts/src/client.ts | 160 +++++++++++-- ts/src/types.ts | 8 + 6 files changed, 769 insertions(+), 56 deletions(-) create mode 100644 python/tests/test_retry_and_resilience.py diff --git a/python/numbersprotocol_capture/client.py b/python/numbersprotocol_capture/client.py index 529391a..ead9cba 100644 --- a/python/numbersprotocol_capture/client.py +++ b/python/numbersprotocol_capture/client.py @@ -6,6 +6,9 @@ import json import mimetypes +import threading +import time +from collections.abc import Callable from pathlib import Path from typing import Any from urllib.parse import urlencode @@ -36,6 +39,11 @@ ASSET_SEARCH_API_URL = "https://us-central1-numbers-protocol-api.cloudfunctions.net/asset-search" NFT_SEARCH_API_URL = "https://eofveg1f59hrbn.m.pipedream.net" +DEFAULT_TIMEOUT = 30.0 +DEFAULT_MAX_RETRIES = 3 +DEFAULT_RETRY_DELAY = 1.0 +RETRYABLE_STATUS_CODES = frozenset([429, 500, 502, 503, 504]) + # Common MIME types by extension MIME_TYPES: dict[str, str] = { "jpg": "image/jpeg", @@ -134,6 +142,10 @@ def __init__( *, testnet: bool = False, base_url: str | None = None, + timeout: float = DEFAULT_TIMEOUT, + max_retries: int = DEFAULT_MAX_RETRIES, + retry_delay: float = DEFAULT_RETRY_DELAY, + rate_limit: int | None = None, options: CaptureOptions | None = None, ): """ @@ -143,12 +155,20 @@ def __init__( token: Authentication token for API access. testnet: Use testnet environment (default: False). base_url: Custom base URL (overrides testnet setting). + timeout: Request timeout in seconds (default: 30.0). + max_retries: Maximum retry attempts for transient failures (default: 3). + retry_delay: Initial backoff delay in seconds (default: 1.0). + rate_limit: Maximum requests per second (default: None, no limiting). options: CaptureOptions object (alternative to individual args). """ if options: token = options.token testnet = options.testnet base_url = options.base_url + timeout = options.timeout + max_retries = options.max_retries + retry_delay = options.retry_delay + rate_limit = options.rate_limit if not token: raise ValidationError("token is required") @@ -156,7 +176,16 @@ def __init__( self._token = token self._testnet = testnet self._base_url = base_url or DEFAULT_BASE_URL - self._client = httpx.Client(timeout=30.0) + self._timeout = timeout + self._max_retries = max_retries + self._retry_delay = retry_delay + self._rate_limit = rate_limit + self._client = httpx.Client(timeout=timeout) + + # Rate limiter state (token bucket) + self._rate_limit_lock = threading.Lock() + self._rate_limit_tokens = float(rate_limit) if rate_limit is not None else 0.0 + self._rate_limit_last_time = time.monotonic() def __enter__(self) -> Capture: return self @@ -168,6 +197,56 @@ def close(self) -> None: """Close the HTTP client.""" self._client.close() + def _acquire_rate_limit_token(self) -> None: + """Acquires a rate-limit token using a token-bucket algorithm.""" + if not self._rate_limit: + return + wait = 0.0 + with self._rate_limit_lock: + now = time.monotonic() + elapsed = max(0.0, now - self._rate_limit_last_time) + self._rate_limit_tokens = min( + float(self._rate_limit), + self._rate_limit_tokens + elapsed * self._rate_limit, + ) + self._rate_limit_last_time = now + if self._rate_limit_tokens >= 1.0: + self._rate_limit_tokens -= 1.0 + return + wait = (1.0 - self._rate_limit_tokens) / self._rate_limit + self._rate_limit_tokens = 0.0 + # Project last_time forward so the next caller correctly sees 0 tokens + self._rate_limit_last_time = now + wait + time.sleep(wait) + + def _execute_with_retry( + self, + func: Callable[[], httpx.Response], + nid: str | None = None, + ) -> httpx.Response: + """Execute an HTTP request callable with retry and rate-limit logic.""" + self._acquire_rate_limit_token() + + final_error: Exception | None = None + final_response: httpx.Response | None = None + + for attempt in range(self._max_retries + 1): + if attempt > 0: + delay = self._retry_delay * (2.0 ** (attempt - 1)) + time.sleep(delay) + try: + response = func() + final_response = response + if response.status_code in RETRYABLE_STATUS_CODES and attempt < self._max_retries: + continue + return response + except httpx.RequestError as e: + final_error = e + + if final_response is not None: + return final_response + raise create_api_error(0, f"Network error: {final_error}", nid) from final_error + def _request( self, method: str, @@ -181,9 +260,9 @@ def _request( """Makes an authenticated API request.""" headers = {"Authorization": f"token {self._token}"} - try: + def build_and_send() -> httpx.Response: if files: - response = self._client.request( + return self._client.request( method, url, headers=headers, @@ -191,22 +270,21 @@ def _request( files=files, ) elif json_body: - headers["Content-Type"] = "application/json" - response = self._client.request( + return self._client.request( method, url, - headers=headers, + headers={**headers, "Content-Type": "application/json"}, json=json_body, ) else: - response = self._client.request( + return self._client.request( method, url, headers=headers, data=data, ) - except httpx.RequestError as e: - raise create_api_error(0, f"Network error: {e}", nid) from e + + response = self._execute_with_retry(build_and_send, nid=nid) if not response.is_success: message = f"API request failed with status {response.status_code}" @@ -451,10 +529,10 @@ def get_history(self, nid: str) -> list[Commit]: "Authorization": f"token {self._token}", } - try: - response = self._client.get(url, headers=headers) - except httpx.RequestError as e: - raise create_api_error(0, f"Network error: {e}", nid) from e + response = self._execute_with_retry( + lambda: self._client.get(url, headers=headers), + nid=nid, + ) if not response.is_success: raise create_api_error( @@ -516,14 +594,14 @@ def get_asset_tree(self, nid: str) -> AssetTree: "Authorization": f"token {self._token}", } - try: - response = self._client.post( + response = self._execute_with_retry( + lambda: self._client.post( MERGE_TREE_API_URL, headers=headers, json=commit_data, - ) - except httpx.RequestError as e: - raise create_api_error(0, f"Network error: {e}", nid) from e + ), + nid=nid, + ) if not response.is_success: raise create_api_error( @@ -677,22 +755,23 @@ def search_asset( # Verify Engine API requires token in Authorization header, not form data headers = {"Authorization": f"token {self._token}"} - try: - if files_data: - response = self._client.post( + if files_data: + response = self._execute_with_retry( + lambda: self._client.post( ASSET_SEARCH_API_URL, headers=headers, data=form_data, files=files_data, ) - else: - response = self._client.post( + ) + else: + response = self._execute_with_retry( + lambda: self._client.post( ASSET_SEARCH_API_URL, headers=headers, data=form_data, ) - except httpx.RequestError as e: - raise create_api_error(0, f"Network error: {e}") from e + ) if not response.is_success: message = f"Asset search failed with status {response.status_code}" @@ -745,14 +824,14 @@ def search_nft(self, nid: str) -> NftSearchResult: "Authorization": f"token {self._token}", } - try: - response = self._client.post( + response = self._execute_with_retry( + lambda: self._client.post( NFT_SEARCH_API_URL, headers=headers, json={"nid": nid}, - ) - except httpx.RequestError as e: - raise create_api_error(0, f"Network error: {e}", nid) from e + ), + nid=nid, + ) if not response.is_success: message = f"NFT search failed with status {response.status_code}" diff --git a/python/numbersprotocol_capture/types.py b/python/numbersprotocol_capture/types.py index 0eff64d..b31b9d8 100644 --- a/python/numbersprotocol_capture/types.py +++ b/python/numbersprotocol_capture/types.py @@ -32,6 +32,18 @@ class CaptureOptions: base_url: str | None = None """Custom base URL (overrides testnet setting).""" + timeout: float = 30.0 + """Request timeout in seconds (default: 30.0).""" + + max_retries: int = 3 + """Maximum number of retry attempts for transient failures (default: 3).""" + + retry_delay: float = 1.0 + """Initial delay in seconds for exponential backoff (default: 1.0).""" + + rate_limit: int | None = None + """Maximum requests per second for client-side rate limiting (default: None).""" + @dataclass class SignOptions: diff --git a/python/tests/test_retry_and_resilience.py b/python/tests/test_retry_and_resilience.py new file mode 100644 index 0000000..8c24a1e --- /dev/null +++ b/python/tests/test_retry_and_resilience.py @@ -0,0 +1,278 @@ +""" +Unit tests for retry logic, timeout configuration, and rate limiting. +""" + +from __future__ import annotations + +import time +from unittest.mock import patch + +import httpx +import pytest +import respx +from httpx import Response + +from numbersprotocol_capture import Capture +from numbersprotocol_capture.client import ( + ASSET_SEARCH_API_URL, + DEFAULT_MAX_RETRIES, + DEFAULT_RETRY_DELAY, + DEFAULT_TIMEOUT, +) +from numbersprotocol_capture.types import CaptureOptions + +TEST_NID = "bafybeif3mhxhkhfwuszl2lybtai3hz3q6naqpfisd4q55mcc7opkmiv5ei" + +SEARCH_OK_RESPONSE = { + "precise_match": "", + "input_file_mime_type": "image/png", + "similar_matches": [], + "order_id": "order_123", +} + + +class TestDefaultConfiguration: + """Tests for default values of new configuration options.""" + + def test_default_timeout(self) -> None: + """Capture client uses 30s timeout by default.""" + with Capture(token="test-token") as capture: + assert capture._timeout == DEFAULT_TIMEOUT + assert capture._timeout == 30.0 + + def test_default_max_retries(self) -> None: + """Capture client uses 3 max retries by default.""" + with Capture(token="test-token") as capture: + assert capture._max_retries == DEFAULT_MAX_RETRIES + assert capture._max_retries == 3 + + def test_default_retry_delay(self) -> None: + """Capture client uses 1.0s retry delay by default.""" + with Capture(token="test-token") as capture: + assert capture._retry_delay == DEFAULT_RETRY_DELAY + assert capture._retry_delay == 1.0 + + def test_default_rate_limit_none(self) -> None: + """No rate limiting by default.""" + with Capture(token="test-token") as capture: + assert capture._rate_limit is None + + def test_custom_timeout_via_init(self) -> None: + """Custom timeout is respected.""" + with Capture(token="test-token", timeout=60.0) as capture: + assert capture._timeout == 60.0 + + def test_custom_max_retries_via_init(self) -> None: + """Custom max_retries is respected.""" + with Capture(token="test-token", max_retries=5) as capture: + assert capture._max_retries == 5 + + def test_custom_retry_delay_via_init(self) -> None: + """Custom retry_delay is respected.""" + with Capture(token="test-token", retry_delay=2.0) as capture: + assert capture._retry_delay == 2.0 + + def test_custom_rate_limit_via_init(self) -> None: + """Custom rate_limit is respected.""" + with Capture(token="test-token", rate_limit=10) as capture: + assert capture._rate_limit == 10 + + def test_options_object_passes_new_fields(self) -> None: + """CaptureOptions dataclass fields are passed to the client.""" + opts = CaptureOptions( + token="test-token", + timeout=45.0, + max_retries=2, + retry_delay=0.5, + rate_limit=5, + ) + with Capture(options=opts) as capture: + assert capture._timeout == 45.0 + assert capture._max_retries == 2 + assert capture._retry_delay == 0.5 + assert capture._rate_limit == 5 + + +class TestRetryLogic: + """Tests for retry logic on transient failures.""" + + @respx.mock + def test_retries_on_503(self) -> None: + """Client retries on 503 Service Unavailable.""" + with Capture(token="test-token", max_retries=3, retry_delay=0.0) as capture: + respx.post(ASSET_SEARCH_API_URL).mock( + side_effect=[ + Response(503), + Response(200, json=SEARCH_OK_RESPONSE), + ] + ) + result = capture.search_asset(nid=TEST_NID) + assert result.order_id == "order_123" + + @respx.mock + def test_retries_on_502(self) -> None: + """Client retries on 502 Bad Gateway.""" + with Capture(token="test-token", max_retries=3, retry_delay=0.0) as capture: + respx.post(ASSET_SEARCH_API_URL).mock( + side_effect=[ + Response(502), + Response(200, json=SEARCH_OK_RESPONSE), + ] + ) + result = capture.search_asset(nid=TEST_NID) + assert result.order_id == "order_123" + + @respx.mock + def test_retries_on_429(self) -> None: + """Client retries on 429 Too Many Requests.""" + with Capture(token="test-token", max_retries=3, retry_delay=0.0) as capture: + respx.post(ASSET_SEARCH_API_URL).mock( + side_effect=[ + Response(429), + Response(200, json=SEARCH_OK_RESPONSE), + ] + ) + result = capture.search_asset(nid=TEST_NID) + assert result.order_id == "order_123" + + @respx.mock + def test_retries_on_500(self) -> None: + """Client retries on 500 Internal Server Error.""" + with Capture(token="test-token", max_retries=3, retry_delay=0.0) as capture: + respx.post(ASSET_SEARCH_API_URL).mock( + side_effect=[ + Response(500), + Response(200, json=SEARCH_OK_RESPONSE), + ] + ) + result = capture.search_asset(nid=TEST_NID) + assert result.order_id == "order_123" + + @respx.mock + def test_retries_on_504(self) -> None: + """Client retries on 504 Gateway Timeout.""" + with Capture(token="test-token", max_retries=3, retry_delay=0.0) as capture: + respx.post(ASSET_SEARCH_API_URL).mock( + side_effect=[ + Response(504), + Response(200, json=SEARCH_OK_RESPONSE), + ] + ) + result = capture.search_asset(nid=TEST_NID) + assert result.order_id == "order_123" + + @respx.mock + def test_does_not_retry_on_400(self) -> None: + """Client does NOT retry on 400 Bad Request.""" + call_count = 0 + + def side_effect(request: httpx.Request) -> Response: + nonlocal call_count + call_count += 1 + return Response(400, json={"detail": "bad request"}) + + with Capture(token="test-token", max_retries=3, retry_delay=0.0) as capture: + respx.post(ASSET_SEARCH_API_URL).mock(side_effect=side_effect) + with pytest.raises(Exception): + capture.search_asset(nid=TEST_NID) + + assert call_count == 1 + + @respx.mock + def test_does_not_retry_on_404(self) -> None: + """Client does NOT retry on 404 Not Found.""" + call_count = 0 + + def side_effect(request: httpx.Request) -> Response: + nonlocal call_count + call_count += 1 + return Response(404, json={"detail": "not found"}) + + with Capture(token="test-token", max_retries=3, retry_delay=0.0) as capture: + respx.post(ASSET_SEARCH_API_URL).mock(side_effect=side_effect) + with pytest.raises(Exception): + capture.search_asset(nid=TEST_NID) + + assert call_count == 1 + + @respx.mock + def test_exhausts_retries_and_raises(self) -> None: + """Client raises error after exhausting all retries.""" + with Capture(token="test-token", max_retries=2, retry_delay=0.0) as capture: + respx.post(ASSET_SEARCH_API_URL).mock( + side_effect=[Response(503), Response(503), Response(503)] + ) + with pytest.raises(Exception): + capture.search_asset(nid=TEST_NID) + + @respx.mock + def test_max_retries_zero_does_not_retry(self) -> None: + """max_retries=0 means no retry attempts.""" + call_count = 0 + + def side_effect(request: httpx.Request) -> Response: + nonlocal call_count + call_count += 1 + return Response(503) + + with Capture(token="test-token", max_retries=0) as capture: + respx.post(ASSET_SEARCH_API_URL).mock(side_effect=side_effect) + with pytest.raises(Exception): + capture.search_asset(nid=TEST_NID) + + assert call_count == 1 + + def test_retries_on_network_error(self) -> None: + """Client retries on network (connection) errors.""" + call_count = 0 + + def mock_request(*args, **kwargs): # type: ignore[no-untyped-def] + nonlocal call_count + call_count += 1 + if call_count == 1: + raise httpx.ConnectError("Connection refused") + # Return a mock response object + resp = Response(200, json=SEARCH_OK_RESPONSE) + return resp + + with Capture(token="test-token", max_retries=2, retry_delay=0.0) as capture: + with patch.object(capture._client, "post", side_effect=mock_request): + result = capture.search_asset(nid=TEST_NID) + + assert call_count == 2 + assert result.order_id == "order_123" + + +class TestRateLimiting: + """Tests for client-side rate limiting.""" + + @respx.mock + def test_rate_limit_none_allows_all_requests(self) -> None: + """Without rate_limit, requests pass through immediately.""" + respx.post(ASSET_SEARCH_API_URL).mock( + return_value=Response(200, json=SEARCH_OK_RESPONSE) + ) + + with Capture(token="test-token") as capture: + for _ in range(5): + result = capture.search_asset(nid=TEST_NID) + assert result.order_id == "order_123" + + @respx.mock + def test_rate_limit_token_bucket_starts_full(self) -> None: + """Token bucket starts full - first N requests within rate_limit pass immediately.""" + respx.post(ASSET_SEARCH_API_URL).mock( + return_value=Response(200, json=SEARCH_OK_RESPONSE) + ) + + rate_limit = 5 + start = time.monotonic() + + with Capture(token="test-token", rate_limit=rate_limit) as capture: + # First 5 requests (bucket starts full) should be fast + for _ in range(rate_limit): + capture.search_asset(nid=TEST_NID) + + elapsed = time.monotonic() - start + # Should be fast since bucket starts full + assert elapsed < 1.0, f"Expected < 1.0s for burst, got {elapsed:.2f}s" diff --git a/ts/src/client.test.ts b/ts/src/client.test.ts index e1e50fb..845b98c 100644 --- a/ts/src/client.test.ts +++ b/ts/src/client.test.ts @@ -28,6 +28,234 @@ describe('Capture Client', () => { }) }) +describe('Retry and Resilience', () => { + let originalFetch: typeof global.fetch + + beforeEach(() => { + originalFetch = global.fetch + vi.useFakeTimers() + }) + + afterEach(() => { + global.fetch = originalFetch + vi.restoreAllMocks() + vi.useRealTimers() + }) + + it('should retry on 503 and succeed on second attempt', async () => { + const capture = new Capture({ token: 'test-token', retryDelay: 10 }) + + let callCount = 0 + global.fetch = vi.fn().mockImplementation(async () => { + callCount++ + if (callCount === 1) { + return { ok: false, status: 503, json: async () => ({}) } as Response + } + return { + ok: true, + status: 200, + json: async () => ({ + precise_match: '', + input_file_mime_type: '', + similar_matches: [], + order_id: 'test', + }), + } as Response + }) + + const promise = capture.searchAsset({ nid: TEST_NID }) + await vi.runAllTimersAsync() + const result = await promise + + expect(callCount).toBe(2) + expect(result.orderId).toBe('test') + }) + + it('should retry on 429 and succeed on second attempt', async () => { + const capture = new Capture({ token: 'test-token', retryDelay: 10 }) + + let callCount = 0 + global.fetch = vi.fn().mockImplementation(async () => { + callCount++ + if (callCount === 1) { + return { ok: false, status: 429, json: async () => ({}) } as Response + } + return { + ok: true, + status: 200, + json: async () => ({ + precise_match: '', + input_file_mime_type: '', + similar_matches: [], + order_id: 'ok', + }), + } as Response + }) + + const promise = capture.searchAsset({ nid: TEST_NID }) + await vi.runAllTimersAsync() + const result = await promise + + expect(callCount).toBe(2) + expect(result.orderId).toBe('ok') + }) + + it('should not retry on 400 (non-retryable)', async () => { + const capture = new Capture({ token: 'test-token', maxRetries: 3 }) + + let callCount = 0 + global.fetch = vi.fn().mockImplementation(async () => { + callCount++ + return { ok: false, status: 400, json: async () => ({}) } as Response + }) + + await expect(capture.searchAsset({ nid: TEST_NID })).rejects.toThrow() + + expect(callCount).toBe(1) + }) + + it('should not retry on 404 (non-retryable)', async () => { + const capture = new Capture({ token: 'test-token', maxRetries: 3 }) + + let callCount = 0 + global.fetch = vi.fn().mockImplementation(async () => { + callCount++ + return { ok: false, status: 404, json: async () => ({}) } as Response + }) + + await expect(capture.searchAsset({ nid: TEST_NID })).rejects.toThrow() + + expect(callCount).toBe(1) + }) + + it('should respect maxRetries=0 (no retries)', async () => { + const capture = new Capture({ token: 'test-token', maxRetries: 0 }) + + let callCount = 0 + global.fetch = vi.fn().mockImplementation(async () => { + callCount++ + return { ok: false, status: 503, json: async () => ({}) } as Response + }) + + await expect(capture.searchAsset({ nid: TEST_NID })).rejects.toThrow() + + expect(callCount).toBe(1) + }) + + it('should retry on network error', async () => { + const capture = new Capture({ token: 'test-token', maxRetries: 1, retryDelay: 10 }) + + let callCount = 0 + global.fetch = vi.fn().mockImplementation(async () => { + callCount++ + if (callCount === 1) { + throw new TypeError('Network request failed') + } + return { + ok: true, + status: 200, + json: async () => ({ + precise_match: '', + input_file_mime_type: '', + similar_matches: [], + order_id: 'recovered', + }), + } as Response + }) + + const promise = capture.searchAsset({ nid: TEST_NID }) + await vi.runAllTimersAsync() + const result = await promise + + expect(callCount).toBe(2) + expect(result.orderId).toBe('recovered') + }) + + it('should use timeout option (default 30000ms)', () => { + const captureDefault = new Capture({ token: 'test-token' }) + // @ts-expect-error accessing private field for test + expect(captureDefault.timeout).toBe(30000) + + const captureCustom = new Capture({ token: 'test-token', timeout: 5000 }) + // @ts-expect-error accessing private field for test + expect(captureCustom.timeout).toBe(5000) + }) + + it('should use configurable maxRetries and retryDelay defaults', () => { + const capture = new Capture({ token: 'test-token' }) + // @ts-expect-error accessing private field for test + expect(capture.maxRetries).toBe(3) + // @ts-expect-error accessing private field for test + expect(capture.retryDelay).toBe(1000) + }) +}) + +describe('Rate Limiting', () => { + let originalFetch: typeof global.fetch + + beforeEach(() => { + originalFetch = global.fetch + vi.useFakeTimers() + }) + + afterEach(() => { + global.fetch = originalFetch + vi.restoreAllMocks() + vi.useRealTimers() + }) + + it('should allow requests when rate limit is not set', async () => { + const capture = new Capture({ token: 'test-token' }) + + global.fetch = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ + precise_match: '', + input_file_mime_type: '', + similar_matches: [], + order_id: 'test', + }), + } as Response) + + const promise = capture.searchAsset({ nid: TEST_NID }) + await vi.runAllTimersAsync() + await expect(promise).resolves.toBeDefined() + }) + + it('should throttle requests when rateLimit is set (token bucket)', async () => { + // rateLimit=2 means 2 requests per second; bucket starts full with 2 tokens + const capture = new Capture({ token: 'test-token', rateLimit: 2 }) + + let fetchCallTimes: number[] = [] + global.fetch = vi.fn().mockImplementation(async () => { + fetchCallTimes.push(Date.now()) + return { + ok: true, + status: 200, + json: async () => ({ + precise_match: '', + input_file_mime_type: '', + similar_matches: [], + order_id: 'test', + }), + } as Response + }) + + // First 2 requests should be immediate (bucket starts full) + const p1 = capture.searchAsset({ nid: TEST_NID }) + const p2 = capture.searchAsset({ nid: TEST_NID }) + // Third request should be delayed (bucket empty) + const p3 = capture.searchAsset({ nid: TEST_NID }) + + await vi.runAllTimersAsync() + await Promise.all([p1, p2, p3]) + + // All three calls completed + expect(fetchCallTimes).toHaveLength(3) + }) +}) + describe('Asset Search Request Construction', () => { let originalFetch: typeof global.fetch diff --git a/ts/src/client.ts b/ts/src/client.ts index c86440e..aba8e0f 100644 --- a/ts/src/client.ts +++ b/ts/src/client.ts @@ -30,6 +30,11 @@ const ASSET_SEARCH_API_URL = 'https://us-central1-numbers-protocol-api.cloudfunctions.net/asset-search' const NFT_SEARCH_API_URL = 'https://eofveg1f59hrbn.m.pipedream.net' +const DEFAULT_TIMEOUT_MS = 30000 +const DEFAULT_MAX_RETRIES = 3 +const DEFAULT_RETRY_DELAY_MS = 1000 +const RETRYABLE_STATUS_CODES = new Set([429, 500, 502, 503, 504]) + /** Common MIME types by extension */ const MIME_TYPES: Record = { jpg: 'image/jpeg', @@ -142,6 +147,12 @@ export class Capture { private readonly token: string private readonly baseUrl: string private readonly testnet: boolean + private readonly timeout: number + private readonly maxRetries: number + private readonly retryDelay: number + private readonly rateLimit?: number + private rateLimitTokens: number + private rateLimitLastTime: number constructor(options: CaptureOptions) { if (!options.token) { @@ -150,6 +161,91 @@ export class Capture { this.token = options.token this.testnet = options.testnet ?? false this.baseUrl = options.baseUrl ?? DEFAULT_BASE_URL + this.timeout = options.timeout ?? DEFAULT_TIMEOUT_MS + this.maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES + this.retryDelay = options.retryDelay ?? DEFAULT_RETRY_DELAY_MS + this.rateLimit = options.rateLimit + this.rateLimitTokens = options.rateLimit ?? 0 + this.rateLimitLastTime = Date.now() + } + + /** + * Acquires a rate-limit token, sleeping if the token bucket is empty. + */ + private async acquireRateLimitToken(): Promise { + if (!this.rateLimit) return + + const msPerSecond = 1000 + + while (true) { + const now = Date.now() + const elapsed = (now - this.rateLimitLastTime) / msPerSecond + this.rateLimitTokens = Math.min( + this.rateLimit, + this.rateLimitTokens + elapsed * this.rateLimit + ) + this.rateLimitLastTime = now + + if (this.rateLimitTokens >= 1) { + this.rateLimitTokens -= 1 + return + } + + const waitMs = Math.ceil(((1 - this.rateLimitTokens) / this.rateLimit) * msPerSecond) + await new Promise((resolve) => setTimeout(resolve, waitMs)) + } + } + + /** + * Fetches a URL with timeout, retry, and rate-limiting applied. + */ + private async fetchWithRetry( + url: string, + init: RequestInit, + nid?: string + ): Promise { + await this.acquireRateLimitToken() + + let finalError: unknown + let finalResponse: Response | undefined + + for (let attempt = 0; attempt <= this.maxRetries; attempt++) { + if (attempt > 0) { + const delay = this.retryDelay * Math.pow(2, attempt - 1) + await new Promise((resolve) => setTimeout(resolve, delay)) + } + + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), this.timeout) + + try { + const response = await fetch(url, { ...init, signal: controller.signal }) + clearTimeout(timeoutId) + finalResponse = response + + if (RETRYABLE_STATUS_CODES.has(response.status) && attempt < this.maxRetries) { + continue + } + + return response + } catch (error) { + clearTimeout(timeoutId) + finalError = error + } + } + + if (finalResponse) { + return finalResponse + } + + if (finalError instanceof DOMException && finalError.name === 'AbortError') { + throw createApiError(0, `Request timed out after ${this.timeout}ms`, nid) + } + throw createApiError( + 0, + `Network error: ${(finalError as Error)?.message ?? 'Unknown error'}`, + nid + ) } /** @@ -173,11 +269,11 @@ export class Capture { requestBody = JSON.stringify(body) } - const response = await fetch(url, { - method, - headers, - body: requestBody, - }) + const response = await this.fetchWithRetry( + url, + { method, headers, body: requestBody }, + nid + ) if (!response.ok) { let message = `API request failed with status ${response.status}` @@ -371,13 +467,17 @@ export class Capture { url.searchParams.set('testnet', 'true') } - const response = await fetch(url.toString(), { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - Authorization: `token ${this.token}`, + const response = await this.fetchWithRetry( + url.toString(), + { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: `token ${this.token}`, + }, }, - }) + nid + ) if (!response.ok) { throw createApiError(response.status, 'Failed to fetch asset history', nid) @@ -427,14 +527,18 @@ export class Capture { timestampCreated: c.timestamp, })) - const response = await fetch(MERGE_TREE_API_URL, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `token ${this.token}`, + const response = await this.fetchWithRetry( + MERGE_TREE_API_URL, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `token ${this.token}`, + }, + body: JSON.stringify(commitData), }, - body: JSON.stringify(commitData), - }) + nid + ) if (!response.ok) { throw createApiError(response.status, 'Failed to merge asset trees', nid) @@ -517,7 +621,7 @@ export class Capture { } // Verify Engine API requires token in Authorization header, not form data - const response = await fetch(ASSET_SEARCH_API_URL, { + const response = await this.fetchWithRetry(ASSET_SEARCH_API_URL, { method: 'POST', headers: { Authorization: `token ${this.token}`, @@ -573,14 +677,18 @@ export class Capture { throw new ValidationError('nid is required for NFT search') } - const response = await fetch(NFT_SEARCH_API_URL, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `token ${this.token}`, + const response = await this.fetchWithRetry( + NFT_SEARCH_API_URL, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `token ${this.token}`, + }, + body: JSON.stringify({ nid }), }, - body: JSON.stringify({ nid }), - }) + nid + ) if (!response.ok) { let message = `NFT search failed with status ${response.status}` diff --git a/ts/src/types.ts b/ts/src/types.ts index bc57312..1a61828 100644 --- a/ts/src/types.ts +++ b/ts/src/types.ts @@ -18,6 +18,14 @@ export interface CaptureOptions { testnet?: boolean /** Custom base URL (overrides testnet setting) */ baseUrl?: string + /** Request timeout in milliseconds (default: 30000) */ + timeout?: number + /** Maximum number of retry attempts for transient failures (default: 3) */ + maxRetries?: number + /** Initial delay in milliseconds for exponential backoff (default: 1000) */ + retryDelay?: number + /** Maximum requests per second for client-side rate limiting (default: none) */ + rateLimit?: number } /**