From 99b10ef862fb26a2c91ceeb5a1ff391d1e375f6a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Feb 2026 20:14:48 +0000 Subject: [PATCH 1/2] Initial plan From 50f58fd05c10eab59ba64fba029491e5bdb7c280 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Feb 2026 20:26:30 +0000 Subject: [PATCH 2/2] fix: remove token leakage to Pipedream NFT endpoint and add response size limits - Remove Authorization header from search_nft/searchNft calls to the third-party Pipedream endpoint (https://eofveg1f59hrbn.m.pipedream.net) - Add MAX_RESPONSE_SIZE = 10 MB constant in both Python and TypeScript SDKs - Python: add _read_response_body() streaming helper; refactor _request, search_asset, and search_nft to use httpx streaming with size checks - TypeScript: add readJsonWithSizeLimit() streaming helper using ReadableStream; apply to request, getHistory, getAssetTree, searchAsset, and searchNft - Add tests for token non-leakage and response size enforcement Co-authored-by: numbers-official <181934381+numbers-official@users.noreply.github.com> --- python/numbersprotocol_capture/client.py | 163 +++++++++++------------ python/tests/test_client.py | 63 +++++++++ ts/src/client.test.ts | 103 ++++++++++++++ ts/src/client.ts | 86 +++++++++--- 4 files changed, 315 insertions(+), 100 deletions(-) diff --git a/python/numbersprotocol_capture/client.py b/python/numbersprotocol_capture/client.py index 529391a..34f3359 100644 --- a/python/numbersprotocol_capture/client.py +++ b/python/numbersprotocol_capture/client.py @@ -13,7 +13,7 @@ import httpx from .crypto import create_integrity_proof, sha256, sign_integrity_proof -from .errors import CaptureError, ValidationError, create_api_error +from .errors import CaptureError, NetworkError, ValidationError, create_api_error from .types import ( Asset, AssetSearchOptions, @@ -36,6 +36,9 @@ ASSET_SEARCH_API_URL = "https://us-central1-numbers-protocol-api.cloudfunctions.net/asset-search" NFT_SEARCH_API_URL = "https://eofveg1f59hrbn.m.pipedream.net" +# Maximum allowed response body size (10 MB) +MAX_RESPONSE_SIZE = 10 * 1024 * 1024 + # Common MIME types by extension MIME_TYPES: dict[str, str] = { "jpg": "image/jpeg", @@ -168,6 +171,20 @@ def close(self) -> None: """Close the HTTP client.""" self._client.close() + def _read_response_body(self, response: httpx.Response) -> bytes: + """Reads response body with a maximum size limit.""" + content_length = response.headers.get("content-length") + if content_length and int(content_length) > MAX_RESPONSE_SIZE: + raise NetworkError("Response body too large") + chunks: list[bytes] = [] + total_size = 0 + for chunk in response.iter_bytes(): + total_size += len(chunk) + if total_size > MAX_RESPONSE_SIZE: + raise NetworkError("Response body too large") + chunks.append(chunk) + return b"".join(chunks) + def _request( self, method: str, @@ -179,47 +196,37 @@ def _request( nid: str | None = None, ) -> dict[str, Any]: """Makes an authenticated API request.""" - headers = {"Authorization": f"token {self._token}"} + headers: dict[str, str] = {"Authorization": f"token {self._token}"} + if json_body: + headers["Content-Type"] = "application/json" + + kwargs: dict[str, Any] = {"headers": headers} + if files: + kwargs["data"] = data + kwargs["files"] = files + elif json_body: + kwargs["json"] = json_body + elif data: + kwargs["data"] = data try: - if files: - response = self._client.request( - method, - url, - headers=headers, - data=data, - files=files, - ) - elif json_body: - headers["Content-Type"] = "application/json" - response = self._client.request( - method, - url, - headers=headers, - json=json_body, - ) - else: - response = self._client.request( - method, - url, - headers=headers, - data=data, - ) + with self._client.stream(method, url, **kwargs) as response: + body = self._read_response_body(response) + + if not response.is_success: + message = f"API request failed with status {response.status_code}" + try: + error_data = json.loads(body) + message = error_data.get("detail") or error_data.get("message") or message + except Exception: + pass + raise create_api_error(response.status_code, message, nid) + + result: dict[str, Any] = json.loads(body) + return result except httpx.RequestError as e: raise create_api_error(0, f"Network error: {e}", nid) from e - if not response.is_success: - message = f"API request failed with status {response.status_code}" - try: - error_data = response.json() - message = error_data.get("detail") or error_data.get("message") or message - except Exception: - pass - raise create_api_error(response.status_code, message, nid) - - result: dict[str, Any] = response.json() - return result - def register( self, file: FileInput, @@ -678,37 +685,30 @@ def search_asset( headers = {"Authorization": f"token {self._token}"} try: + request_kwargs: dict[str, Any] = { + "headers": headers, + "data": form_data, + } if files_data: - response = self._client.post( - ASSET_SEARCH_API_URL, - headers=headers, - data=form_data, - files=files_data, - ) - else: - response = self._client.post( - ASSET_SEARCH_API_URL, - headers=headers, - data=form_data, - ) + request_kwargs["files"] = files_data + with self._client.stream("POST", ASSET_SEARCH_API_URL, **request_kwargs) as response: + body = self._read_response_body(response) + if not response.is_success: + message = f"Asset search failed with status {response.status_code}" + try: + error_data = json.loads(body) + message = ( + error_data.get("message") + or error_data.get("error") + or message + ) + except Exception: + pass + raise create_api_error(response.status_code, message) + data = json.loads(body) except httpx.RequestError as e: raise create_api_error(0, f"Network error: {e}") from e - if not response.is_success: - message = f"Asset search failed with status {response.status_code}" - try: - error_data = response.json() - message = ( - error_data.get("message") - or error_data.get("error") - or message - ) - except Exception: - pass - raise create_api_error(response.status_code, message) - - data = response.json() - # Map response to our type similar_matches = [ SimilarMatch(nid=m["nid"], distance=m["distance"]) @@ -742,33 +742,32 @@ def search_nft(self, nid: str) -> NftSearchResult: headers = { "Content-Type": "application/json", - "Authorization": f"token {self._token}", } try: - response = self._client.post( + with self._client.stream( + "POST", NFT_SEARCH_API_URL, headers=headers, json={"nid": nid}, - ) + ) as response: + body = self._read_response_body(response) + if not response.is_success: + message = f"NFT search failed with status {response.status_code}" + try: + error_data = json.loads(body) + message = ( + error_data.get("message") + or error_data.get("error") + or message + ) + except Exception: + pass + raise create_api_error(response.status_code, message, nid) + data = json.loads(body) except httpx.RequestError as e: raise create_api_error(0, f"Network error: {e}", nid) from e - if not response.is_success: - message = f"NFT search failed with status {response.status_code}" - try: - error_data = response.json() - message = ( - error_data.get("message") - or error_data.get("error") - or message - ) - except Exception: - pass - raise create_api_error(response.status_code, message, nid) - - data = response.json() - # Map response to our type records = [ NftRecord( diff --git a/python/tests/test_client.py b/python/tests/test_client.py index 8a081c0..4bcb0d8 100644 --- a/python/tests/test_client.py +++ b/python/tests/test_client.py @@ -1,6 +1,8 @@ """Tests for the Capture client.""" import pytest +import respx +from httpx import Response as HttpxResponse from numbersprotocol_capture import Capture, ValidationError @@ -76,3 +78,64 @@ def test_update_empty_nid_raises_error(self) -> None: with Capture(token="test-token") as capture: with pytest.raises(ValidationError, match="nid is required"): capture.update("", caption="test") + + +class TestNftSearchSecurity: + """Tests verifying NFT search does not leak the auth token to third-party endpoint.""" + + @respx.mock + def test_search_nft_does_not_send_authorization_header(self) -> None: + """Verify token is NOT sent in Authorization header to the Pipedream endpoint.""" + nft_search_url = "https://eofveg1f59hrbn.m.pipedream.net" + test_token = "my-secret-token" + + route = respx.post(nft_search_url).mock( + return_value=HttpxResponse(200, json={"records": [], "order_id": "order_1"}) + ) + + with Capture(token=test_token) as capture: + capture.search_nft("bafybeitest") + + request = route.calls[0].request + assert "Authorization" not in request.headers + + +class TestResponseSizeLimit: + """Tests for HTTP response body size limits.""" + + @respx.mock + def test_search_nft_raises_on_oversized_response(self) -> None: + """Verify NetworkError is raised when NFT search response exceeds size limit.""" + from numbersprotocol_capture.errors import NetworkError + from numbersprotocol_capture.client import MAX_RESPONSE_SIZE + + nft_search_url = "https://eofveg1f59hrbn.m.pipedream.net" + oversized_body = b"x" * (MAX_RESPONSE_SIZE + 1) + + respx.post(nft_search_url).mock( + return_value=HttpxResponse(200, content=oversized_body) + ) + + with Capture(token="test-token") as capture: + with pytest.raises(NetworkError, match="Response body too large"): + capture.search_nft("bafybeitest") + + @respx.mock + def test_search_nft_raises_on_large_content_length_header(self) -> None: + """Verify NetworkError is raised when Content-Length header exceeds size limit.""" + from numbersprotocol_capture.errors import NetworkError + from numbersprotocol_capture.client import MAX_RESPONSE_SIZE + + nft_search_url = "https://eofveg1f59hrbn.m.pipedream.net" + + respx.post(nft_search_url).mock( + return_value=HttpxResponse( + 200, + content=b'{"records":[]}', + headers={"content-length": str(MAX_RESPONSE_SIZE + 1)}, + ) + ) + + with Capture(token="test-token") as capture: + with pytest.raises(NetworkError, match="Response body too large"): + capture.search_nft("bafybeitest") diff --git a/ts/src/client.test.ts b/ts/src/client.test.ts index e1e50fb..dd5133e 100644 --- a/ts/src/client.test.ts +++ b/ts/src/client.test.ts @@ -337,3 +337,106 @@ describe('Asset Search Validation', () => { ).rejects.toThrow('sampleCount must be a positive integer') }) }) + +describe('NFT Search Security', () => { + const NFT_SEARCH_API_URL = 'https://eofveg1f59hrbn.m.pipedream.net' + let originalFetch: typeof global.fetch + + beforeEach(() => { + originalFetch = global.fetch + }) + + afterEach(() => { + global.fetch = originalFetch + vi.restoreAllMocks() + }) + + it('should NOT send Authorization header to the Pipedream endpoint', async () => { + const testToken = 'my-secret-token' + const capture = new Capture({ token: testToken }) + + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + headers: { get: () => null } as unknown as Headers, + body: null, + json: async () => ({ records: [], order_id: 'order_1' }), + } as Response) + + global.fetch = mockFetch + + await capture.searchNft('bafybeitest') + + const [url, options] = mockFetch.mock.calls[0] + expect(url).toBe(NFT_SEARCH_API_URL) + + const headers = options?.headers as Record + expect(headers?.Authorization).toBeUndefined() + }) +}) + +describe('Response Size Limit', () => { + let originalFetch: typeof global.fetch + + beforeEach(() => { + originalFetch = global.fetch + }) + + afterEach(() => { + global.fetch = originalFetch + vi.restoreAllMocks() + }) + + it('should throw NetworkError when Content-Length header exceeds limit', async () => { + const capture = new Capture({ token: 'test-token' }) + const MAX_RESPONSE_SIZE = 10 * 1024 * 1024 + + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + headers: { + get: (name: string) => + name === 'content-length' ? String(MAX_RESPONSE_SIZE + 1) : null, + } as unknown as Headers, + body: null, + json: async () => ({ + precise_match: '', + input_file_mime_type: '', + similar_matches: [], + order_id: '', + }), + } as Response) + + global.fetch = mockFetch + + await expect(capture.searchAsset({ nid: TEST_NID })).rejects.toThrow( + 'Response body too large' + ) + }) + + it('should throw NetworkError when streaming body exceeds limit', async () => { + const capture = new Capture({ token: 'test-token' }) + const MAX_RESPONSE_SIZE = 10 * 1024 * 1024 + const oversizedChunk = new Uint8Array(MAX_RESPONSE_SIZE + 1) + + let readerReleased = false + const mockReader = { + read: vi + .fn() + .mockResolvedValueOnce({ done: false, value: oversizedChunk }) + .mockResolvedValue({ done: true, value: undefined }), + releaseLock: () => { readerReleased = true }, + } + + const mockFetch = vi.fn().mockResolvedValue({ + ok: true, + headers: { get: () => null } as unknown as Headers, + body: { getReader: () => mockReader } as unknown as ReadableStream, + } as Response) + + global.fetch = mockFetch + + await expect(capture.searchAsset({ nid: TEST_NID })).rejects.toThrow( + 'Response body too large' + ) + expect(readerReleased).toBe(true) + }) +}) diff --git a/ts/src/client.ts b/ts/src/client.ts index c86440e..56b4ec3 100644 --- a/ts/src/client.ts +++ b/ts/src/client.ts @@ -18,6 +18,7 @@ import { ValidationError, createApiError, CaptureError, + NetworkError, } from './errors.js' import { sha256, createIntegrityProof, signIntegrityProof } from './crypto.js' @@ -30,6 +31,9 @@ const ASSET_SEARCH_API_URL = 'https://us-central1-numbers-protocol-api.cloudfunctions.net/asset-search' const NFT_SEARCH_API_URL = 'https://eofveg1f59hrbn.m.pipedream.net' +/** Maximum allowed response body size (10 MB) */ +const MAX_RESPONSE_SIZE = 10 * 1024 * 1024 + /** Common MIME types by extension */ const MIME_TYPES: Record = { jpg: 'image/jpeg', @@ -48,6 +52,48 @@ const MIME_TYPES: Record = { txt: 'text/plain', } +/** + * Reads response body as JSON while enforcing a maximum size limit. + * Throws NetworkError if the response body exceeds MAX_RESPONSE_SIZE. + */ +async function readJsonWithSizeLimit(response: Response): Promise { + const contentLength = response.headers?.get('content-length') + if (contentLength !== null && parseInt(contentLength, 10) > MAX_RESPONSE_SIZE) { + throw new NetworkError('Response body too large') + } + + if (!response.body) { + return response.json() + } + + const reader = response.body.getReader() + const chunks: Uint8Array[] = [] + let totalSize = 0 + + try { + while (true) { + const { done, value } = await reader.read() + if (done) break + totalSize += value.byteLength + if (totalSize > MAX_RESPONSE_SIZE) { + throw new NetworkError('Response body too large') + } + chunks.push(value) + } + } finally { + reader.releaseLock() + } + + const allChunks = new Uint8Array(totalSize) + let offset = 0 + for (const chunk of chunks) { + allChunks.set(chunk, offset) + offset += chunk.byteLength + } + + return JSON.parse(new TextDecoder().decode(allChunks)) +} + /** * Detects MIME type from filename extension. */ @@ -182,15 +228,15 @@ export class Capture { if (!response.ok) { let message = `API request failed with status ${response.status}` try { - const errorData = await response.json() - message = errorData.detail || errorData.message || message + const errorData = await readJsonWithSizeLimit(response) as Record + message = (errorData?.detail as string) || (errorData?.message as string) || message } catch { // Use default message } throw createApiError(response.status, message, nid) } - return response.json() as Promise + return readJsonWithSizeLimit(response) as Promise } /** @@ -383,7 +429,7 @@ export class Capture { throw createApiError(response.status, 'Failed to fetch asset history', nid) } - const data = (await response.json()) as HistoryApiResponse + const data = (await readJsonWithSizeLimit(response)) as HistoryApiResponse return data.commits.map((c) => ({ assetTreeCid: c.assetTreeCid, @@ -440,7 +486,7 @@ export class Capture { throw createApiError(response.status, 'Failed to merge asset trees', nid) } - const data = await response.json() + const data = await readJsonWithSizeLimit(response) as Record // The API returns { mergedAssetTree: {...}, assetTrees: [...] } // We return the merged tree @@ -528,18 +574,18 @@ export class Capture { if (!response.ok) { let message = `Asset search failed with status ${response.status}` try { - const errorData = await response.json() - message = errorData.message || errorData.error || message + const errorData = await readJsonWithSizeLimit(response) as Record + message = (errorData?.message as string) || (errorData?.error as string) || message } catch { // Use default message } throw createApiError(response.status, message) } - const data = await response.json() + const data = await readJsonWithSizeLimit(response) as Record // Map response to our type - const similarMatches: SimilarMatch[] = (data.similar_matches || []).map( + const similarMatches: SimilarMatch[] = ((data.similar_matches as Array<{ nid: string; distance: number }>) || []).map( (m: { nid: string; distance: number }) => ({ nid: m.nid, distance: m.distance, @@ -547,10 +593,10 @@ export class Capture { ) return { - preciseMatch: data.precise_match || '', - inputFileMimeType: data.input_file_mime_type || '', + preciseMatch: (data.precise_match as string) || '', + inputFileMimeType: (data.input_file_mime_type as string) || '', similarMatches, - orderId: data.order_id || '', + orderId: (data.order_id as string) || '', } } @@ -577,7 +623,6 @@ export class Capture { method: 'POST', headers: { 'Content-Type': 'application/json', - Authorization: `token ${this.token}`, }, body: JSON.stringify({ nid }), }) @@ -585,18 +630,23 @@ export class Capture { if (!response.ok) { let message = `NFT search failed with status ${response.status}` try { - const errorData = await response.json() - message = errorData.message || errorData.error || message + const errorData = await readJsonWithSizeLimit(response) as Record + message = (errorData?.message as string) || (errorData?.error as string) || message } catch { // Use default message } throw createApiError(response.status, message, nid) } - const data = await response.json() + const data = await readJsonWithSizeLimit(response) as Record // Map response to our type - const records: NftRecord[] = (data.records || []).map( + const records: NftRecord[] = ((data.records as Array<{ + token_id: string + contract: string + network: string + owner?: string + }>) || []).map( (r: { token_id: string contract: string @@ -612,7 +662,7 @@ export class Capture { return { records, - orderId: data.order_id || '', + orderId: (data.order_id as string) || '', } } }