diff --git a/pyproject.toml b/pyproject.toml index 1d50b0b61..a5121ade2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-langchain" -version = "0.8.12" +version = "0.8.13" description = "Python SDK that enables developers to build and deploy LangGraph agents to the UiPath Cloud Platform" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/src/uipath_langchain/agent/exceptions/exceptions.py b/src/uipath_langchain/agent/exceptions/exceptions.py index 29406b1e9..86c9561c0 100644 --- a/src/uipath_langchain/agent/exceptions/exceptions.py +++ b/src/uipath_langchain/agent/exceptions/exceptions.py @@ -45,6 +45,8 @@ class AgentRuntimeErrorCode(str, Enum): # State STATE_ERROR = "STATE_ERROR" + FILE_ERROR = "FILE_ERROR" + LLM_INVALID_RESPONSE = "LLM_INVALID_RESPONSE" TOOL_INVALID_WRAPPER_STATE = "TOOL_INVALID_WRAPPER_STATE" diff --git a/src/uipath_langchain/agent/multimodal/invoke.py b/src/uipath_langchain/agent/multimodal/invoke.py index 7555eee57..9f9688ba2 100644 --- a/src/uipath_langchain/agent/multimodal/invoke.py +++ b/src/uipath_langchain/agent/multimodal/invoke.py @@ -1,6 +1,6 @@ """LLM invocation with multimodal file attachments.""" -import asyncio +import logging from typing import Any from langchain_core.language_models import BaseChatModel @@ -12,25 +12,39 @@ ) from langchain_core.messages.content import create_file_block, create_image_block -from .types import FileInfo +from .types import MAX_FILE_SIZE_BYTES, FileInfo from .utils import download_file_base64, is_image, is_pdf, sanitize_filename +logger = logging.getLogger("uipath") + async def build_file_content_block( file_info: FileInfo, + *, + max_size: int = MAX_FILE_SIZE_BYTES, ) -> DataContentBlock: """Build a LangChain content block for a file attachment. + Downloads the file with size enforcement and creates the content block. + Size validation happens during download (via Content-Length check and + streaming guard) to avoid loading oversized files into memory. + Args: file_info: File URL, name, and MIME type. + max_size: Maximum allowed raw file size in bytes. LLM providers + enforce payload limits; base64 encoding adds ~30% overhead. Returns: A DataContentBlock for the file (image or PDF). Raises: - ValueError: If the MIME type is not supported. + ValueError: If the MIME type is not supported or the file exceeds + the size limit for LLM payloads. """ - base64_file = await download_file_base64(file_info.url) + try: + base64_file = await download_file_base64(file_info.url, max_size=max_size) + except ValueError as exc: + raise ValueError(f"File '{file_info.name}': {exc}") from exc if is_image(file_info.mime_type): return create_image_block(base64=base64_file, mime_type=file_info.mime_type) @@ -47,6 +61,9 @@ async def build_file_content_block( async def build_file_content_blocks(files: list[FileInfo]) -> list[DataContentBlock]: """Build content blocks from file attachments. + Files are processed sequentially to avoid loading multiple large files + into memory simultaneously. + Args: files: List of file information to convert to content blocks @@ -56,9 +73,10 @@ async def build_file_content_blocks(files: list[FileInfo]) -> list[DataContentBl if not files: return [] - file_content_blocks: list[DataContentBlock] = await asyncio.gather( - *[build_file_content_block(file) for file in files] - ) + file_content_blocks: list[DataContentBlock] = [] + for file in files: + block = await build_file_content_block(file) + file_content_blocks.append(block) return file_content_blocks @@ -100,6 +118,9 @@ async def llm_call_with_files( all_messages = list(messages) + [file_message] response = await model.ainvoke(all_messages) + + del all_messages, file_message, content_blocks + if not isinstance(response, AIMessage): raise TypeError(f"LLM returned {type(response).__name__} instead of AIMessage") return response diff --git a/src/uipath_langchain/agent/multimodal/types.py b/src/uipath_langchain/agent/multimodal/types.py index 42ace7d40..bd0869ca5 100644 --- a/src/uipath_langchain/agent/multimodal/types.py +++ b/src/uipath_langchain/agent/multimodal/types.py @@ -2,6 +2,15 @@ from dataclasses import dataclass +MAX_FILE_SIZE_BYTES: int = 30 * 1024 * 1024 # 30MB + +IMAGE_MIME_TYPES: set[str] = { + "image/png", + "image/jpeg", + "image/gif", + "image/webp", +} + @dataclass class FileInfo: @@ -10,11 +19,3 @@ class FileInfo: url: str name: str mime_type: str - - -IMAGE_MIME_TYPES: set[str] = { - "image/png", - "image/jpeg", - "image/gif", - "image/webp", -} diff --git a/src/uipath_langchain/agent/multimodal/utils.py b/src/uipath_langchain/agent/multimodal/utils.py index 3dbeebe8f..19696098e 100644 --- a/src/uipath_langchain/agent/multimodal/utils.py +++ b/src/uipath_langchain/agent/multimodal/utils.py @@ -2,6 +2,7 @@ import base64 import re +from collections.abc import AsyncIterator import httpx from uipath._utils._ssl_context import get_httpx_client_kwargs @@ -36,10 +37,99 @@ def is_image(mime_type: str) -> bool: return mime_type.lower() in IMAGE_MIME_TYPES -async def download_file_base64(url: str) -> str: - """Download a file from a URL and return its content as a base64 string.""" +def _format_mb(size_bytes: int, decimals: int = 1) -> str: + """Format a byte count as MB. + + Args: + size_bytes: Size in bytes. + decimals: Number of decimal places (0 for rounded integer). + """ + return f"{size_bytes / (1024 * 1024):.{decimals}f} MB" + + +async def encode_streamed_base64( + chunks: AsyncIterator[bytes], + *, + max_size: int = 0, +) -> str: + """Incrementally base64-encode an async stream of byte chunks. + + Encodes chunks as they arrive so the raw file bytes are never assembled + into a single contiguous buffer. base64 processes 3-byte groups, so a + remainder of 0-2 bytes is buffered between chunks. + + Args: + chunks: Async iterator yielding raw byte chunks. + max_size: Maximum allowed total size in bytes. 0 means unlimited. + + Returns: + The full base64-encoded string. + + Raises: + ValueError: If the total size exceeds max_size. + """ + encoded_buf = bytearray() + remainder = b"" + total = 0 + + async for chunk in chunks: + total += len(chunk) + if max_size > 0 and total > max_size: + raise ValueError( + f"File exceeds the {_format_mb(max_size, decimals=0)}" + f" limit for LLM payloads" + f" (downloaded {_format_mb(total)} so far)" + ) + + data = remainder + chunk + usable = len(data) - (len(data) % 3) + if usable > 0: + encoded_buf += base64.b64encode(data[:usable]) + remainder = data[usable:] + else: + remainder = data + + if remainder: + encoded_buf += base64.b64encode(remainder) + + result = encoded_buf.decode("ascii") + del encoded_buf + return result + + +async def download_file_base64(url: str, *, max_size: int = 0) -> str: + """Download a file from a URL and return its content as a base64 string. + + Args: + url: The URL to download from. + max_size: Maximum allowed file size in bytes. 0 means unlimited. + + Raises: + ValueError: If the file exceeds max_size. + httpx.HTTPStatusError: If the HTTP request fails. + """ async with httpx.AsyncClient(**get_httpx_client_kwargs()) as client: - response = await client.get(url) - response.raise_for_status() - file_content = response.content - return base64.b64encode(file_content).decode("utf-8") + async with client.stream("GET", url) as response: + response.raise_for_status() + + # Fast reject via Content-Length before reading the body + if max_size > 0: + content_length = response.headers.get("content-length") + if content_length: + try: + content_length_value = int(content_length) + except ValueError: + content_length_value = None + if ( + content_length_value is not None + and content_length_value > max_size + ): + raise ValueError( + f"File is {_format_mb(content_length_value)}" + f" which exceeds the {_format_mb(max_size, decimals=0)}" + f" limit for Agent LLM payloads" + ) + + return await encode_streamed_base64( + response.aiter_bytes(), max_size=max_size + ) diff --git a/src/uipath_langchain/agent/tools/internal_tools/analyze_files_tool.py b/src/uipath_langchain/agent/tools/internal_tools/analyze_files_tool.py index 094b65d64..6010311f5 100644 --- a/src/uipath_langchain/agent/tools/internal_tools/analyze_files_tool.py +++ b/src/uipath_langchain/agent/tools/internal_tools/analyze_files_tool.py @@ -1,4 +1,3 @@ -import asyncio import uuid from typing import Any, cast @@ -19,7 +18,12 @@ ) from uipath.eval.mocks import mockable from uipath.platform import UiPath +from uipath.runtime.errors import UiPathErrorCategory +from uipath_langchain.agent.exceptions import ( + AgentRuntimeError, + AgentRuntimeErrorCode, +) from uipath_langchain.agent.multimodal import FileInfo, build_file_content_block from uipath_langchain.agent.react.jsonschema_pydantic_converter import create_model from uipath_langchain.agent.react.types import AgentGraphState @@ -77,8 +81,16 @@ async def tool_fn(**kwargs: Any): if not files: return {"analysisResult": "No attachments provided to analyze."} - human_message = HumanMessage(content=analysis_task) - human_message_with_files = await add_files_to_message(human_message, files) + try: + human_message = HumanMessage(content=analysis_task) + human_message_with_files = await add_files_to_message(human_message, files) + except ValueError as exc: + raise AgentRuntimeError( + code=AgentRuntimeErrorCode.FILE_ERROR, + title="File attachment too large", + detail=str(exc), + category=UiPathErrorCategory.USER, + ) from exc messages: list[AnyMessage] = [ SystemMessage(content=ANALYZE_FILES_SYSTEM_MESSAGE), @@ -87,6 +99,8 @@ async def tool_fn(**kwargs: Any): config = var_child_runnable_config.get(None) result = await non_streaming_llm.ainvoke(messages, config=config) + del messages, human_message_with_files, files + analysis_result = extract_text_content(result) return {"analysisResult": analysis_result} @@ -172,9 +186,10 @@ async def add_files_to_message( if not files: return message - file_content_blocks: list[DataContentBlock] = await asyncio.gather( - *[build_file_content_block(file) for file in files] - ) + file_content_blocks: list[DataContentBlock] = [] + for file in files: + block = await build_file_content_block(file) + file_content_blocks.append(block) return append_content_blocks_to_message( message, cast(list[ContentBlock], file_content_blocks) ) diff --git a/tests/agent/multimodal/__init__.py b/tests/agent/multimodal/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/agent/multimodal/test_utils.py b/tests/agent/multimodal/test_utils.py new file mode 100644 index 000000000..14db39cc4 --- /dev/null +++ b/tests/agent/multimodal/test_utils.py @@ -0,0 +1,222 @@ +"""Tests for multimodal — file download, size limits, and content block creation.""" + +import base64 + +import httpx +import pytest +from pytest_httpx import HTTPXMock + +from uipath_langchain.agent.multimodal.invoke import build_file_content_block +from uipath_langchain.agent.multimodal.types import MAX_FILE_SIZE_BYTES, FileInfo +from uipath_langchain.agent.multimodal.utils import ( + download_file_base64, + encode_streamed_base64, +) + +FILE_URL = "https://blob.storage.example.com/file.pdf" + + +class _ChunkedStream(httpx.AsyncByteStream): + def __init__(self, chunks: list[bytes]) -> None: + self._chunks = chunks + + async def __aiter__(self): + for chunk in self._chunks: + yield chunk + + +async def _async_iter(chunks: list[bytes]): + """Helper: wrap a list of byte chunks as an async iterator.""" + for chunk in chunks: + yield chunk + + +class TestEncodeStreamedBase64: + """Tests for encode_streamed_base64 — incremental encoding with size limit.""" + + async def test_encodes_single_chunk(self) -> None: + content = b"hello world" + result = await encode_streamed_base64(_async_iter([content])) + assert result == base64.b64encode(content).decode("ascii") + + async def test_encodes_multiple_chunks(self) -> None: + chunks = [b"hello ", b"world"] + result = await encode_streamed_base64(_async_iter(chunks)) + assert result == base64.b64encode(b"hello world").decode("ascii") + + async def test_encodes_empty_stream(self) -> None: + result = await encode_streamed_base64(_async_iter([])) + assert result == "" + + async def test_encodes_single_byte_chunks(self) -> None: + """Handles worst-case chunking where every chunk is 1 byte.""" + data = b"abcdefgh" + chunks = [bytes([b]) for b in data] + result = await encode_streamed_base64(_async_iter(chunks)) + assert result == base64.b64encode(data).decode("ascii") + + async def test_rejects_when_exceeds_max_size(self) -> None: + chunks = [b"x" * 60, b"x" * 60] + with pytest.raises(ValueError, match="exceeds"): + await encode_streamed_base64(_async_iter(chunks), max_size=100) + + async def test_allows_exactly_at_limit(self) -> None: + content = b"x" * 100 + result = await encode_streamed_base64(_async_iter([content]), max_size=100) + assert result == base64.b64encode(content).decode("ascii") + + async def test_unlimited_when_max_size_zero(self) -> None: + content = b"x" * 10_000 + result = await encode_streamed_base64(_async_iter([content]), max_size=0) + assert result == base64.b64encode(content).decode("ascii") + + async def test_error_message_formats_as_mb(self) -> None: + """Error message shows MB, not raw bytes.""" + limit = 10 * 1024 * 1024 # 10 MB + chunks = [b"x" * (limit + 1)] + with pytest.raises(ValueError, match=r"10 MB.*limit"): + await encode_streamed_base64(_async_iter(chunks), max_size=limit) + + +class TestDownloadFileBase64: + """Tests for download_file_base64 — streaming download with optional size limit.""" + + async def test_downloads_and_encodes(self, httpx_mock: HTTPXMock) -> None: + content = b"hello world" + httpx_mock.add_response(url=FILE_URL, content=content) + + result = await download_file_base64(FILE_URL) + + assert result == base64.b64encode(content).decode("utf-8") + + async def test_http_error_propagates(self, httpx_mock: HTTPXMock) -> None: + httpx_mock.add_response(url=FILE_URL, status_code=404) + + with pytest.raises(httpx.HTTPStatusError): + await download_file_base64(FILE_URL) + + async def test_rejects_via_content_length(self, httpx_mock: HTTPXMock) -> None: + """Content-Length header triggers fast rejection before download.""" + oversized = b"x" * 200 + httpx_mock.add_response(url=FILE_URL, content=oversized) + + with pytest.raises(ValueError, match="exceeds"): + await download_file_base64(FILE_URL, max_size=100) + + async def test_rejects_during_streaming(self, httpx_mock: HTTPXMock) -> None: + """Streaming guard catches oversized files without Content-Length.""" + stream = _ChunkedStream([b"x" * 60, b"x" * 60]) + httpx_mock.add_response( + url=FILE_URL, + stream=stream, + headers={"transfer-encoding": "chunked"}, + ) + + with pytest.raises(ValueError, match="exceeds"): + await download_file_base64(FILE_URL, max_size=100) + + async def test_invalid_content_length_falls_back_to_streaming( + self, httpx_mock: HTTPXMock + ) -> None: + """Malformed Content-Length should not crash download logic.""" + content = b"x" * 80 + httpx_mock.add_response( + url=FILE_URL, + content=content, + headers={"content-length": "invalid"}, + ) + + result = await download_file_base64(FILE_URL, max_size=100) + + assert result == base64.b64encode(content).decode("utf-8") + + async def test_unlimited_when_max_size_zero(self, httpx_mock: HTTPXMock) -> None: + """Default max_size=0 allows any file size.""" + content = b"x" * 10_000 + httpx_mock.add_response(url=FILE_URL, content=content) + + result = await download_file_base64(FILE_URL, max_size=0) + + assert result == base64.b64encode(content).decode("utf-8") + + async def test_file_exactly_at_limit_succeeds(self, httpx_mock: HTTPXMock) -> None: + content = b"x" * 100 + httpx_mock.add_response(url=FILE_URL, content=content) + + result = await download_file_base64(FILE_URL, max_size=100) + + assert result == base64.b64encode(content).decode("utf-8") + + +class TestBuildFileContentBlock: + """Tests for build_file_content_block — size limit enforced during download.""" + + async def test_small_image_succeeds(self, httpx_mock: HTTPXMock) -> None: + content = b"tiny image bytes" + httpx_mock.add_response(url=FILE_URL, content=content) + file_info = FileInfo(url=FILE_URL, name="photo.png", mime_type="image/png") + + block = await build_file_content_block(file_info) + + assert block["type"] == "image" + + async def test_small_pdf_succeeds(self, httpx_mock: HTTPXMock) -> None: + content = b"tiny pdf bytes" + httpx_mock.add_response(url=FILE_URL, content=content) + file_info = FileInfo(url=FILE_URL, name="doc.pdf", mime_type="application/pdf") + + block = await build_file_content_block(file_info) + + assert block["type"] == "file" + + async def test_rejects_file_exceeding_default_limit( + self, httpx_mock: HTTPXMock + ) -> None: + """Files larger than MAX_FILE_SIZE_BYTES are rejected.""" + oversized_content = b"x" * (MAX_FILE_SIZE_BYTES + 1) + httpx_mock.add_response(url=FILE_URL, content=oversized_content) + file_info = FileInfo(url=FILE_URL, name="huge.pdf", mime_type="application/pdf") + + with pytest.raises(ValueError, match="exceeds"): + await build_file_content_block(file_info) + + async def test_rejects_file_exceeding_custom_limit( + self, httpx_mock: HTTPXMock + ) -> None: + """The max_size parameter is respected.""" + content = b"x" * 100 + httpx_mock.add_response(url=FILE_URL, content=content) + file_info = FileInfo(url=FILE_URL, name="big.png", mime_type="image/png") + + with pytest.raises(ValueError, match="exceeds"): + await build_file_content_block(file_info, max_size=10) + + async def test_file_within_custom_limit_succeeds( + self, httpx_mock: HTTPXMock + ) -> None: + content = b"abc" + httpx_mock.add_response(url=FILE_URL, content=content) + file_info = FileInfo(url=FILE_URL, name="small.png", mime_type="image/png") + + block = await build_file_content_block(file_info, max_size=1000) + + assert block["type"] == "image" + + async def test_unsupported_mime_type_raises(self, httpx_mock: HTTPXMock) -> None: + content = b"some data" + httpx_mock.add_response(url=FILE_URL, content=content) + file_info = FileInfo(url=FILE_URL, name="data.csv", mime_type="text/csv") + + with pytest.raises(ValueError, match="Unsupported"): + await build_file_content_block(file_info) + + async def test_error_includes_filename(self, httpx_mock: HTTPXMock) -> None: + """ValueError from download includes the filename for debuggability.""" + content = b"x" * 200 + httpx_mock.add_response(url=FILE_URL, content=content) + file_info = FileInfo( + url=FILE_URL, name="report.pdf", mime_type="application/pdf" + ) + + with pytest.raises(ValueError, match="report.pdf"): + await build_file_content_block(file_info, max_size=100) diff --git a/uv.lock b/uv.lock index 99792480e..a9b2dfe3f 100644 --- a/uv.lock +++ b/uv.lock @@ -3324,7 +3324,7 @@ wheels = [ [[package]] name = "uipath-langchain" -version = "0.8.12" +version = "0.8.13" source = { editable = "." } dependencies = [ { name = "httpx" },