From df3f98b6db15a2d7877e61066b11e5354ce71b51 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Tue, 17 Feb 2026 13:45:17 -0600 Subject: [PATCH 01/81] add sowmya's changes --- deploy/compose/.env | 20 +- deploy/compose/vectordb.yaml | 23 + docs/migration-oracle-26ai.md | 330 +++++++ src/nvidia_rag/utils/vdb/oracle/__init__.py | 20 + .../utils/vdb/oracle/oracle_queries.py | 345 +++++++ src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py | 882 ++++++++++++++++++ 6 files changed, 1619 insertions(+), 1 deletion(-) create mode 100644 docs/migration-oracle-26ai.md create mode 100644 src/nvidia_rag/utils/vdb/oracle/__init__.py create mode 100644 src/nvidia_rag/utils/vdb/oracle/oracle_queries.py create mode 100644 src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py diff --git a/deploy/compose/.env b/deploy/compose/.env index 9f6ccf796..5e44036af 100644 --- a/deploy/compose/.env +++ b/deploy/compose/.env @@ -87,4 +87,22 @@ export OCR_MS_GPU_ID=0 # Paths # ========================== -export PROMPT_CONFIG_FILE=${PWD}/src/nvidia_rag/rag_server/prompt.yaml \ No newline at end of file +export PROMPT_CONFIG_FILE=${PWD}/src/nvidia_rag/rag_server/prompt.yaml + +# ========================== +# Oracle 26ai Vector Database (Default) +# ========================== +# Set APP_VECTORSTORE_NAME=oracle to use Oracle 26ai (default) +# Set APP_VECTORSTORE_NAME=milvus to use Milvus instead + +export APP_VECTORSTORE_NAME=oracle +export ORACLE_USER=rag_user +export ORACLE_PASSWORD= +export ORACLE_DSN=localhost:1521/FREEPDB1 + +# Oracle Vector Index Configuration +export ORACLE_VECTOR_INDEX_TYPE=IVF +export ORACLE_DISTANCE_METRIC=COSINE + +# For hybrid search (vector + text), set: +# export APP_VECTORSTORE_SEARCH_TYPE=hybrid \ No newline at end of file diff --git a/deploy/compose/vectordb.yaml b/deploy/compose/vectordb.yaml index 651f85ddc..c113ebece 100644 --- a/deploy/compose/vectordb.yaml +++ b/deploy/compose/vectordb.yaml @@ -109,6 +109,29 @@ services: retries: 10 profiles: ["elasticsearch"] + # Oracle 26ai Vector Database (CPU-based) + # Uses Oracle Free tier container for development/testing + # For production, connect to your Oracle 26ai instance using ORACLE_DSN + oracle-26ai: + container_name: oracle-26ai + image: container-registry.oracle.com/database/free:latest + environment: + - ORACLE_PWD=${ORACLE_PASSWORD:-oracle123} + - ORACLE_CHARACTERSET=AL32UTF8 + ports: + - "1521:1521" + - "5500:5500" + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/oracle:/opt/oracle/oradata + healthcheck: + test: ["CMD", "bash", "-c", "echo 'SELECT 1 FROM DUAL;' | sqlplus -s system/${ORACLE_PASSWORD:-oracle123}@localhost:1521/FREEPDB1"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 300s + shm_size: 1g + profiles: ["oracle"] + networks: default: name: nvidia-rag \ No newline at end of file diff --git a/docs/migration-oracle-26ai.md b/docs/migration-oracle-26ai.md new file mode 100644 index 000000000..f7c3615e2 --- /dev/null +++ b/docs/migration-oracle-26ai.md @@ -0,0 +1,330 @@ + + +# Migration Guide: Milvus to Oracle AI Database 26ai + +This guide covers migrating the NVIDIA RAG Blueprint from Milvus to Oracle AI Database 26ai as the vector database. Oracle 26ai is now the **default** vector store for this blueprint. + +## Overview + +Oracle AI Database 26ai is Oracle's next-generation AI-native database that integrates vector search capabilities directly into the database engine. Key benefits include: + +- **Native VECTOR data type** - Store and query vectors alongside relational data +- **IVF and HNSW indexes** - CPU-optimized vector indexes for fast similarity search +- **Hybrid search** - Combine vector similarity with Oracle Text full-text search +- **No separate vector DB** - Unified database for all data types +- **Enterprise features** - Built-in security, scalability, and high availability + +## Prerequisites + +Before migrating, ensure you have: + +1. **Oracle 26ai database instance** - Either: + - Oracle Cloud Autonomous Database with AI Vector Search + - On-premises Oracle 26ai installation + - Docker container for development (Oracle Free tier) + +2. **Database user with privileges**: + - CREATE TABLE + - CREATE INDEX + - UNLIMITED TABLESPACE (or appropriate quota) + +3. **Network connectivity** from RAG servers to Oracle database + +4. **Python environment** with Oracle dependencies + +## Step 1: Install Oracle Dependencies + +Install the Oracle optional dependencies: + +```bash +# Using pip +pip install nvidia_rag[oracle] + +# Using uv +uv sync --extra oracle +``` + +This installs: +- `oracledb>=2.0.0` - Oracle Database Python driver (thin client, no Oracle Client needed) +- `langchain-community>=0.4` - LangChain integration with OracleVS + +## Step 2: Configure Environment Variables + +Set the following environment variables for Oracle connection: + +```bash +# Vector store selection (oracle is now default) +export APP_VECTORSTORE_NAME=oracle + +# Oracle connection credentials +export ORACLE_USER=rag_user +export ORACLE_PASSWORD=your_secure_password +export ORACLE_DSN=hostname:1521/service_name + +# Optional: Vector index configuration +export ORACLE_VECTOR_INDEX_TYPE=IVF # IVF (default) or HNSW +export ORACLE_DISTANCE_METRIC=COSINE # COSINE (default), L2, DOT, MANHATTAN + +# Optional: Enable hybrid search (vector + text) +export APP_VECTORSTORE_SEARCH_TYPE=hybrid +``` + +### Connection String Formats + +Oracle DSN can be specified in multiple formats: + +```bash +# Easy Connect format +export ORACLE_DSN=hostname:1521/service_name + +# Easy Connect Plus with options +export ORACLE_DSN=hostname:1521/service_name?connect_timeout=30 + +# TNS alias (requires tnsnames.ora) +export ORACLE_DSN=mydb_alias +``` + +## Step 3: Prepare Oracle Database + +Connect to your Oracle database as a DBA and create the RAG user: + +```sql +-- Create user +CREATE USER rag_user IDENTIFIED BY your_secure_password; + +-- Grant privileges +GRANT CONNECT, RESOURCE TO rag_user; +GRANT CREATE TABLE TO rag_user; +GRANT CREATE INDEX TO rag_user; +GRANT UNLIMITED TABLESPACE TO rag_user; + +-- For hybrid search (Oracle Text) +GRANT CTXAPP TO rag_user; +``` + +### Verify Vector Search Support + +Ensure your Oracle 26ai instance has vector search enabled: + +```sql +-- Check database version (should be 26.x) +SELECT * FROM V$VERSION; + +-- Verify VECTOR data type is available +SELECT VECTOR('[1.0, 2.0, 3.0]', 3, FLOAT32) FROM DUAL; +``` + +## Step 4: Deploy Services + +### Option A: Using Docker Compose (Development) + +Start the Oracle container for development: + +```bash +# Set Oracle password +export ORACLE_PASSWORD=oracle123 + +# Start Oracle 26ai container +docker compose -f deploy/compose/vectordb.yaml --profile oracle up -d + +# Wait for Oracle to be ready (first startup takes ~5 minutes) +docker logs -f oracle-26ai +``` + +Then start the RAG services: + +```bash +# Source environment +source deploy/compose/.env + +# Start RAG and Ingestor servers +docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build +docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build +``` + +### Option B: External Oracle Database (Production) + +For production, connect to your existing Oracle 26ai instance: + +```bash +# Set connection to your Oracle instance +export ORACLE_USER=rag_user +export ORACLE_PASSWORD=your_secure_password +export ORACLE_DSN=your-oracle-host:1521/your_service + +# Source other environment variables +source deploy/compose/.env + +# Start RAG services +docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build +docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build +``` + +## Step 5: Re-ingest Documents + +**Important**: Documents must be re-ingested when switching vector databases. Data does not automatically migrate from Milvus to Oracle. + +1. Access the RAG UI at `http://localhost:8090` +2. Create a new collection +3. Upload your documents +4. Verify ingestion via health check: + +```bash +curl -X GET 'http://localhost:8082/v1/health?check_dependencies=true' -H 'accept: application/json' +``` + +## Configuration Reference + +### Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `APP_VECTORSTORE_NAME` | Vector store type | `oracle` | +| `ORACLE_USER` | Database username | - | +| `ORACLE_PASSWORD` | Database password | - | +| `ORACLE_DSN` | Connection DSN | - | +| `ORACLE_VECTOR_INDEX_TYPE` | Index type: `IVF` or `HNSW` | `IVF` | +| `ORACLE_DISTANCE_METRIC` | Distance: `COSINE`, `L2`, `DOT`, `MANHATTAN` | `COSINE` | +| `APP_VECTORSTORE_SEARCH_TYPE` | Search type: `dense` or `hybrid` | `dense` | + +### Vector Index Types + +**IVF (Inverted File Index)** - Default, recommended for CPU deployment: +- Good balance of speed and accuracy +- Lower memory usage +- Best for large-scale deployments + +**HNSW (Hierarchical Navigable Small World)**: +- Higher accuracy +- Faster query time +- Higher memory usage +- Best for smaller datasets with high accuracy requirements + +### Distance Metrics + +| Metric | Use Case | +|--------|----------| +| `COSINE` | Default, best for normalized embeddings (most NLP models) | +| `L2` | Euclidean distance, good for image embeddings | +| `DOT` | Dot product, fast but requires normalized vectors | +| `MANHATTAN` | L1 distance, robust to outliers | + +## Hybrid Search Configuration + +Hybrid search combines vector similarity search with Oracle Text full-text search for improved relevance: + +```bash +export APP_VECTORSTORE_SEARCH_TYPE=hybrid +``` + +This creates both: +- Vector index (IVF/HNSW) on the `vector` column +- Oracle Text index (CONTEXT) on the `text` column + +Queries will combine semantic similarity with keyword matching. + +## Rollback to Milvus + +To switch back to Milvus: + +```bash +# Change vector store +export APP_VECTORSTORE_NAME=milvus + +# Start Milvus containers +docker compose -f deploy/compose/vectordb.yaml up -d + +# Restart RAG services +docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d +docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d +``` + +Note: You will need to re-ingest documents into Milvus. + +## Troubleshooting + +### Connection Errors + +**ORA-01017: invalid username/password; logon denied** +- Verify `ORACLE_USER` and `ORACLE_PASSWORD` are correct +- Check if the user exists and is not locked + +**ORA-12541: TNS:no listener** +- Verify `ORACLE_DSN` format and hostname +- Check network connectivity: `telnet hostname 1521` +- Ensure Oracle listener is running + +**ORA-12514: TNS:listener does not currently know of service requested** +- Verify the service name in `ORACLE_DSN` +- List available services: `lsnrctl services` + +### Vector Operations + +**ORA-51801: VECTOR data type not supported** +- Ensure you're using Oracle 26ai (not an older version) +- Check `COMPATIBLE` parameter is set to 23.4.0 or higher + +**Vector dimension mismatch** +- Ensure your embedding model dimension matches the collection +- Default dimension is 2048 (for NeMo Retriever embeddings) +- Check with: `SELECT VECTOR_DIMS(vector) FROM your_table WHERE ROWNUM = 1` + +### Performance Issues + +**Slow ingestion** +- Increase batch size in `oracle_vdb.py` +- Consider disabling indexes during bulk load, then rebuild +- Use parallel DML: `ALTER SESSION ENABLE PARALLEL DML` + +**Slow queries** +- Ensure vector index exists: `SELECT * FROM USER_INDEXES WHERE INDEX_TYPE = 'VECTOR'` +- Rebuild index if needed: `ALTER INDEX idx_name REBUILD` +- Check index statistics: `DBMS_STATS.GATHER_INDEX_STATS` + +## Architecture + +``` +┌─────────────────────────────────────────────────────────┐ +│ NVIDIA RAG Blueprint │ +├─────────────────────────────────────────────────────────┤ +│ RAG Server / Ingestor Server │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ │ +│ │ OracleVDB │ ◄── VDBRagIngest base class │ +│ │ (oracle_vdb) │ │ +│ └────────┬────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────┐ ┌─────────────────┐ │ +│ │ oracledb │ │ LangChain │ │ +│ │ (thin client) │ │ OracleVS │ │ +│ └────────┬────────┘ └────────┬────────┘ │ +│ │ │ │ +│ └──────────┬───────────┘ │ +│ ▼ │ +│ ┌─────────────────────┐ │ +│ │ Oracle 26ai DB │ │ +│ │ - VECTOR columns │ │ +│ │ - IVF indexes │ │ +│ │ - Oracle Text │ │ +│ └─────────────────────┘ │ +└─────────────────────────────────────────────────────────┘ +``` + +## Related Topics + +- [NVIDIA RAG Blueprint Documentation](readme.md) +- [Change the Vector Database](change-vectordb.md) +- [Best Practices for Common Settings](accuracy_perf.md) +- [Troubleshoot](troubleshooting.md) + +## External Resources + +- [Oracle AI Vector Search Documentation](https://docs.oracle.com/en/database/oracle/oracle-database/26/vecse/) +- [Oracle Database 26ai Release Notes](https://docs.oracle.com/en/database/oracle/oracle-database/26/) +- [LangChain OracleVS Integration](https://python.langchain.com/docs/integrations/vectorstores/oracle) +- [oracledb Python Driver](https://python-oracledb.readthedocs.io/) \ No newline at end of file diff --git a/src/nvidia_rag/utils/vdb/oracle/__init__.py b/src/nvidia_rag/utils/vdb/oracle/__init__.py new file mode 100644 index 000000000..ac9d03237 --- /dev/null +++ b/src/nvidia_rag/utils/vdb/oracle/__init__.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Oracle 26ai Vector Database module for NVIDIA RAG Blueprint.""" + +from nvidia_rag.utils.vdb.oracle.oracle_vdb import OracleVDB + +__all__ = ["OracleVDB"] \ No newline at end of file diff --git a/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py b/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py new file mode 100644 index 000000000..490228da7 --- /dev/null +++ b/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py @@ -0,0 +1,345 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Oracle 26ai SQL query utilities for vector database operations. +Provides DDL and query functions for document and metadata management. + +Functions: +1. create_vector_table_ddl: Generate DDL for vector collection tables +2. create_vector_index_ddl: Generate DDL for IVF vector index +3. create_metadata_schema_table_ddl: DDL for metadata schema storage +4. create_document_info_table_ddl: DDL for document info storage +5. get_unique_sources_query: Retrieve all unique document sources +6. get_delete_docs_query: Delete documents by source value +7. get_similarity_search_query: Vector similarity search query +8. get_hybrid_search_query: Combined vector + text search query +""" + +from typing import Literal + +DistanceMetric = Literal["COSINE", "L2", "DOT", "MANHATTAN"] +IndexType = Literal["IVF", "HNSW"] + + +def create_vector_table_ddl( + table_name: str, + dimension: int = 2048, +) -> str: + """ + Generate DDL for creating a vector collection table. + + Args: + table_name: Name of the table to create + dimension: Vector dimension size + + Returns: + DDL statement string + """ + return f""" + CREATE TABLE {table_name} ( + id RAW(16) DEFAULT SYS_GUID() PRIMARY KEY, + text CLOB, + vector VECTOR({dimension}, FLOAT32), + source VARCHAR2(4000), + content_metadata CLOB CHECK (content_metadata IS JSON), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """ + + +def create_vector_index_ddl( + table_name: str, + index_type: IndexType = "IVF", + distance_metric: DistanceMetric = "COSINE", + ivf_neighbor_partitions: int = 100, + hnsw_m: int = 16, + hnsw_ef_construction: int = 200, +) -> str: + """ + Generate DDL for creating a vector index (IVF or HNSW). + + Args: + table_name: Name of the table + index_type: IVF or HNSW + distance_metric: COSINE, L2, DOT, or MANHATTAN + ivf_neighbor_partitions: Number of partitions for IVF index + hnsw_m: Max connections per node for HNSW + hnsw_ef_construction: Size of dynamic candidate list for HNSW + + Returns: + DDL statement string + """ + index_name = f"{table_name}_vec_idx" + + if index_type == "IVF": + return f""" + CREATE VECTOR INDEX {index_name} ON {table_name}(vector) + ORGANIZATION NEIGHBOR PARTITIONS + WITH DISTANCE {distance_metric} + WITH TARGET ACCURACY 95 + PARAMETERS ( + type IVF, + neighbor_partitions {ivf_neighbor_partitions} + ) + """ + else: # HNSW + return f""" + CREATE VECTOR INDEX {index_name} ON {table_name}(vector) + ORGANIZATION INMEMORY NEIGHBOR GRAPH + WITH DISTANCE {distance_metric} + WITH TARGET ACCURACY 95 + PARAMETERS ( + type HNSW, + m {hnsw_m}, + efConstruction {hnsw_ef_construction} + ) + """ + + +def create_text_index_ddl(table_name: str) -> str: + """ + Generate DDL for creating Oracle Text index for hybrid search. + + Args: + table_name: Name of the table + + Returns: + DDL statement string + """ + index_name = f"{table_name}_text_idx" + return f""" + CREATE INDEX {index_name} ON {table_name}(text) + INDEXTYPE IS CTXSYS.CONTEXT + PARAMETERS ('SYNC (ON COMMIT)') + """ + + +def create_metadata_schema_table_ddl() -> str: + """Generate DDL for metadata schema storage table.""" + return """ + CREATE TABLE metadata_schema ( + id RAW(16) DEFAULT SYS_GUID() PRIMARY KEY, + collection_name VARCHAR2(255) NOT NULL UNIQUE, + metadata_schema CLOB CHECK (metadata_schema IS JSON), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """ + + +def create_document_info_table_ddl() -> str: + """Generate DDL for document info storage table.""" + return """ + CREATE TABLE document_info ( + id RAW(16) DEFAULT SYS_GUID() PRIMARY KEY, + collection_name VARCHAR2(255) NOT NULL, + info_type VARCHAR2(50) NOT NULL, + document_name VARCHAR2(4000) NOT NULL, + info_value CLOB CHECK (info_value IS JSON), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT doc_info_unique UNIQUE (collection_name, info_type, document_name) + ) + """ + + +def get_unique_sources_query(table_name: str) -> str: + """ + Generate query to retrieve all unique document sources. + + Args: + table_name: Name of the collection table + + Returns: + SQL query string + """ + return f""" + SELECT DISTINCT source, + MAX(content_metadata) KEEP (DENSE_RANK FIRST ORDER BY created_at DESC) as content_metadata + FROM {table_name} + WHERE source IS NOT NULL + GROUP BY source + ORDER BY source + """ + + +def get_delete_docs_query(table_name: str) -> str: + """ + Generate parameterized delete query for documents by source. + + Args: + table_name: Name of the collection table + + Returns: + SQL query string with :source_value bind variable + """ + return f""" + DELETE FROM {table_name} + WHERE source = :source_value + """ + + +def get_delete_metadata_schema_query() -> str: + """Generate parameterized delete query for metadata schema.""" + return """ + DELETE FROM metadata_schema + WHERE collection_name = :collection_name + """ + + +def get_metadata_schema_query() -> str: + """Generate parameterized query to retrieve metadata schema.""" + return """ + SELECT metadata_schema + FROM metadata_schema + WHERE collection_name = :collection_name + """ + + +def get_delete_document_info_query() -> str: + """Generate parameterized delete query for document info.""" + return """ + DELETE FROM document_info + WHERE collection_name = :collection_name + AND document_name = :document_name + AND info_type = :info_type + """ + + +def get_delete_document_info_by_collection_query() -> str: + """Generate parameterized delete query for all document info in a collection.""" + return """ + DELETE FROM document_info + WHERE collection_name = :collection_name + """ + + +def get_document_info_query() -> str: + """Generate parameterized query to retrieve document info.""" + return """ + SELECT info_value + FROM document_info + WHERE collection_name = :collection_name + AND document_name = :document_name + AND info_type = :info_type + """ + + +def get_collection_document_info_query() -> str: + """Generate parameterized query to retrieve collection-level document info.""" + return """ + SELECT info_value + FROM document_info + WHERE collection_name = :collection_name + AND info_type = :info_type + """ + + +def get_similarity_search_query( + table_name: str, + distance_metric: DistanceMetric = "COSINE", +) -> str: + """ + Generate vector similarity search query. + + Args: + table_name: Name of the collection table + distance_metric: Distance function to use + + Returns: + SQL query string with :query_vector and :top_k bind variables + """ + return f""" + SELECT id, text, source, content_metadata, + VECTOR_DISTANCE(vector, :query_vector, {distance_metric}) as distance + FROM {table_name} + ORDER BY distance + FETCH FIRST :top_k ROWS ONLY + """ + + +def get_hybrid_search_query( + table_name: str, + distance_metric: DistanceMetric = "COSINE", + vector_weight: float = 0.7, + text_weight: float = 0.3, +) -> str: + """ + Generate hybrid search query combining vector similarity and text search. + + Args: + table_name: Name of the collection table + distance_metric: Distance function for vector search + vector_weight: Weight for vector similarity score + text_weight: Weight for text search score + + Returns: + SQL query string with :query_vector, :query_text, and :top_k bind variables + """ + return f""" + WITH vector_results AS ( + SELECT id, text, source, content_metadata, + VECTOR_DISTANCE(vector, :query_vector, {distance_metric}) as vec_distance, + ROW_NUMBER() OVER (ORDER BY VECTOR_DISTANCE(vector, :query_vector, {distance_metric})) as vec_rank + FROM {table_name} + FETCH FIRST :top_k * 2 ROWS ONLY + ), + text_results AS ( + SELECT id, SCORE(1) as text_score, + ROW_NUMBER() OVER (ORDER BY SCORE(1) DESC) as text_rank + FROM {table_name} + WHERE CONTAINS(text, :query_text, 1) > 0 + FETCH FIRST :top_k * 2 ROWS ONLY + ) + SELECT v.id, v.text, v.source, v.content_metadata, + ({vector_weight} * (1 / (1 + v.vec_distance)) + + {text_weight} * COALESCE(t.text_score / 100, 0)) as combined_score + FROM vector_results v + LEFT JOIN text_results t ON v.id = t.id + ORDER BY combined_score DESC + FETCH FIRST :top_k ROWS ONLY + """ + + +def get_count_query(table_name: str) -> str: + """Generate query to count documents in a collection.""" + return f"SELECT COUNT(*) as cnt FROM {table_name}" + + +def check_table_exists_query() -> str: + """Generate query to check if a table exists.""" + return """ + SELECT COUNT(*) as cnt + FROM user_tables + WHERE table_name = UPPER(:table_name) + """ + + +def drop_table_ddl(table_name: str) -> str: + """Generate DDL to drop a table.""" + return f"DROP TABLE {table_name} CASCADE CONSTRAINTS PURGE" + + +def get_all_collections_query() -> str: + """Generate query to list all collection tables.""" + return """ + SELECT table_name + FROM user_tables + WHERE table_name NOT IN ('METADATA_SCHEMA', 'DOCUMENT_INFO') + AND table_name NOT LIKE 'SYS%' + AND table_name NOT LIKE 'DR$%' + ORDER BY table_name + """ \ No newline at end of file diff --git a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py new file mode 100644 index 000000000..f68cb1fa6 --- /dev/null +++ b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py @@ -0,0 +1,882 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Oracle 26ai Vector Database implementation for NVIDIA RAG Blueprint. + +This module provides the OracleVDB class which implements vector database +operations using Oracle AI Database 26ai's native vector search capabilities. +Supports CPU-based IVF indexes with optional hybrid search using Oracle Text. + +Key Features: +- Native VECTOR data type support +- IVF (Inverted File Index) for CPU-optimized search +- Hybrid search combining vector similarity with Oracle Text +- LangChain OracleVS integration for retrieval +""" + +import json +import logging +import os +import time +from concurrent.futures import Future +from typing import Any + +import oracledb +from langchain_community.vectorstores.oraclevs import OracleVS +from langchain_community.vectorstores.utils import DistanceStrategy +from langchain_core.documents import Document +from langchain_core.runnables import RunnableAssign, RunnableLambda +from opentelemetry import context as otel_context + +from nvidia_rag.rag_server.response_generator import APIError, ErrorCodeMapping +from nvidia_rag.utils.common import ( + get_current_timestamp, + perform_document_info_aggregation, +) +from nvidia_rag.utils.configuration import NvidiaRAGConfig, SearchType +from nvidia_rag.utils.health_models import ServiceStatus +from nvidia_rag.utils.vdb import ( + DEFAULT_DOCUMENT_INFO_COLLECTION, + DEFAULT_METADATA_SCHEMA_COLLECTION, + SYSTEM_COLLECTIONS, +) +from nvidia_rag.utils.vdb.oracle.oracle_queries import ( + check_table_exists_query, + create_document_info_table_ddl, + create_metadata_schema_table_ddl, + create_text_index_ddl, + create_vector_index_ddl, + create_vector_table_ddl, + drop_table_ddl, + get_all_collections_query, + get_collection_document_info_query, + get_count_query, + get_delete_document_info_by_collection_query, + get_delete_document_info_query, + get_delete_docs_query, + get_delete_metadata_schema_query, + get_document_info_query, + get_hybrid_search_query, + get_metadata_schema_query, + get_similarity_search_query, + get_unique_sources_query, +) +from nvidia_rag.utils.vdb.vdb_ingest_base import VDBRagIngest + +logger = logging.getLogger(__name__) + + +class OracleVDB(VDBRagIngest): + """ + Oracle 26ai Vector Database implementation. + + Provides vector storage and retrieval using Oracle AI Database 26ai's + native vector capabilities with IVF indexes optimized for CPU-based deployment. + """ + + def __init__( + self, + collection_name: str, + oracle_user: str | None = None, + oracle_password: str | None = None, + oracle_dsn: str | None = None, + embedding_model: Any | None = None, + config: NvidiaRAGConfig | None = None, + meta_dataframe: Any | None = None, + meta_source_field: str | None = None, + meta_fields: list[str] | None = None, + csv_file_path: str | None = None, + index_type: str = "IVF", + distance_metric: str = "COSINE", + hybrid: bool = False, + ): + """ + Initialize Oracle VDB connection. + + Args: + collection_name: Name of the vector collection/table + oracle_user: Database username (or set ORACLE_USER env var) + oracle_password: Database password (or set ORACLE_PASSWORD env var) + oracle_dsn: Connection DSN (or set ORACLE_DSN env var) + embedding_model: Embedding model instance for retrieval + config: NvidiaRAGConfig instance + meta_dataframe: Metadata dataframe for custom metadata + meta_source_field: Source field name in metadata + meta_fields: List of metadata field names + csv_file_path: Path to CSV file for metadata + index_type: Vector index type (IVF or HNSW) + distance_metric: Distance metric (COSINE, L2, DOT) + hybrid: Enable hybrid search with Oracle Text + """ + self.config = config or NvidiaRAGConfig() + self._collection_name = collection_name.upper() if collection_name else "" + self._embedding_model = embedding_model + + # Connection parameters from args or environment + self._oracle_user = oracle_user or os.getenv("ORACLE_USER") + self._oracle_password = oracle_password or os.getenv("ORACLE_PASSWORD") + self._oracle_dsn = oracle_dsn or os.getenv("ORACLE_DSN") + + if not all([self._oracle_user, self._oracle_password, self._oracle_dsn]): + raise ValueError( + "Oracle connection requires ORACLE_USER, ORACLE_PASSWORD, and ORACLE_DSN. " + "Set via parameters or environment variables." + ) + + # Vector index configuration + self._index_type = os.getenv("ORACLE_VECTOR_INDEX_TYPE", index_type).upper() + self._distance_metric = os.getenv("ORACLE_DISTANCE_METRIC", distance_metric).upper() + self._hybrid = hybrid or (self.config.vector_store.search_type == SearchType.HYBRID) + + # Metadata fields for NV-Ingest + self.meta_dataframe = meta_dataframe + self.meta_source_field = meta_source_field + self.meta_fields = meta_fields + self.csv_file_path = csv_file_path + + # System collection initialization flags + self._metadata_schema_initialized = False + self._document_info_initialized = False + + # Initialize connection pool + try: + self._pool = oracledb.create_pool( + user=self._oracle_user, + password=self._oracle_password, + dsn=self._oracle_dsn, + min=2, + max=10, + increment=1, + ) + # Test connection + with self._pool.acquire() as conn: + with conn.cursor() as cursor: + cursor.execute("SELECT 1 FROM DUAL") + logger.info(f"Connected to Oracle at {self._oracle_dsn}") + except oracledb.Error as e: + logger.exception("Failed to connect to Oracle at %s: %s", self._oracle_dsn, e) + raise APIError( + f"Oracle database is unavailable at {self._oracle_dsn}. " + "Please verify Oracle is running and credentials are correct.", + ErrorCodeMapping.SERVICE_UNAVAILABLE, + ) from e + + @property + def collection_name(self) -> str: + """Get the collection name.""" + return self._collection_name + + @collection_name.setter + def collection_name(self, value: str) -> None: + """Set the collection name.""" + self._collection_name = value.upper() if value else "" + + def _get_connection(self): + """Acquire a connection from the pool.""" + return self._pool.acquire() + + def _table_exists(self, table_name: str) -> bool: + """Check if a table exists.""" + with self._get_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(check_table_exists_query(), {"table_name": table_name}) + result = cursor.fetchone() + return result[0] > 0 + + # ------------------------------------------------------------------------- + # NV-Ingest VDB Operations + def _check_index_exists(self, index_name: str) -> bool: + """Check if the collection table exists.""" + return self._table_exists(index_name) + + def create_index(self): + """Create the vector table and index.""" + logger.info(f"Creating Oracle collection if not exists: {self._collection_name}") + self.create_collection( + self._collection_name, + dimension=self.config.embeddings.dimensions, + ) + + def write_to_index(self, records: list, **kwargs) -> None: + """ + Write records to the Oracle vector table. + + Requires nv_ingest_client for record cleanup. + """ + try: + from nv_ingest_client.util.milvus import cleanup_records, pandas_file_reader + except ImportError as e: + raise ImportError( + "nv_ingest_client is required for write_to_index operation. " + "Install with: pip install nvidia-rag[ingest]" + ) from e + + # Load metadata if needed + meta_dataframe = self.meta_dataframe + if meta_dataframe is None and self.csv_file_path is not None: + meta_dataframe = pandas_file_reader(self.csv_file_path) + + # Clean records + cleaned_records = cleanup_records( + records=records, + meta_dataframe=meta_dataframe, + meta_source_field=self.meta_source_field, + meta_fields=self.meta_fields, + ) + + total_records = len(cleaned_records) + batch_size = 100 + uploaded_count = 0 + + logger.info(f"Starting Oracle ingestion for {total_records} records...") + + with self._get_connection() as conn: + with conn.cursor() as cursor: + insert_sql = f""" + INSERT INTO {self._collection_name} (text, vector, source, content_metadata) + VALUES (:text, :vector, :source, :content_metadata) + """ + + for i in range(0, total_records, batch_size): + batch = cleaned_records[i:i + batch_size] + batch_data = [] + + for record in batch: + vector = record.get("vector", []) + # Convert vector to Oracle VECTOR format + vector_str = "[" + ",".join(str(v) for v in vector) + "]" + + batch_data.append({ + "text": record.get("text", ""), + "vector": vector_str, + "source": record.get("source", ""), + "content_metadata": json.dumps(record.get("content_metadata", {})), + }) + + cursor.executemany(insert_sql, batch_data) + conn.commit() + + uploaded_count += len(batch) + if uploaded_count % (5 * batch_size) == 0 or uploaded_count == total_records: + logger.info(f"Ingested {uploaded_count}/{total_records} records into {self._collection_name}") + + logger.info(f"Oracle ingestion completed. Total: {uploaded_count} records") + + def retrieval(self, queries: list, **kwargs) -> list[dict[str, Any]]: + """Retrieve documents based on queries.""" + raise NotImplementedError("Use retrieval_langchain for Oracle retrieval") + + def reindex(self, records: list, **kwargs) -> None: + """Reindex documents.""" + raise NotImplementedError("Reindex not implemented for Oracle") + + def run(self, records: list) -> None: + """Run ingestion pipeline.""" + self.create_index() + self.write_to_index(records) + + def run_async(self, records: list | Future) -> list: + """Run async ingestion.""" + logger.info(f"Creating index - {self._collection_name}") + self.create_index() + + if isinstance(records, Future): + records = records.result() + + logger.info(f"Writing to index - {self._collection_name}") + self.write_to_index(records) + return records + + # ------------------------------------------------------------------------- + # VDBRag Collection Management + async def check_health(self) -> dict[str, Any]: + """Check Oracle database health.""" + status = { + "service": "Oracle 26ai", + "url": self._oracle_dsn, + "status": ServiceStatus.UNKNOWN.value, + "error": None, + } + + try: + start_time = time.time() + with self._get_connection() as conn: + with conn.cursor() as cursor: + cursor.execute("SELECT 1 FROM DUAL") + cursor.execute("SELECT COUNT(*) FROM user_tables") + table_count = cursor.fetchone()[0] + + status["status"] = ServiceStatus.HEALTHY.value + status["latency_ms"] = round((time.time() - start_time) * 1000, 2) + status["tables"] = table_count + except Exception as e: + status["status"] = ServiceStatus.ERROR.value + status["error"] = str(e) + + return status + + def create_collection( + self, + collection_name: str, + dimension: int = 2048, + collection_type: str = "text", + ) -> None: + """Create a new vector collection table.""" + table_name = collection_name.upper() + + if self._table_exists(table_name): + logger.info(f"Collection {table_name} already exists") + return + + with self._get_connection() as conn: + with conn.cursor() as cursor: + # Create table + cursor.execute(create_vector_table_ddl(table_name, dimension)) + logger.info(f"Created table {table_name}") + + # Create vector index + try: + cursor.execute(create_vector_index_ddl( + table_name, + index_type=self._index_type, + distance_metric=self._distance_metric, + )) + logger.info(f"Created {self._index_type} vector index on {table_name}") + except oracledb.Error as e: + logger.warning(f"Could not create vector index: {e}") + + # Create text index for hybrid search + if self._hybrid: + try: + cursor.execute(create_text_index_ddl(table_name)) + logger.info(f"Created text index on {table_name}") + except oracledb.Error as e: + logger.warning(f"Could not create text index: {e}") + + conn.commit() + + def check_collection_exists(self, collection_name: str) -> bool: + """Check if a collection exists.""" + return self._table_exists(collection_name.upper()) + + def get_collection(self) -> list[dict[str, Any]]: + """Get all collections with metadata.""" + self.create_metadata_schema_collection() + self.create_document_info_collection() + + collections = [] + with self._get_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(get_all_collections_query()) + tables = cursor.fetchall() + + for (table_name,) in tables: + if table_name.upper() in [s.upper() for s in SYSTEM_COLLECTIONS]: + continue + + # Get document count + cursor.execute(get_count_query(table_name)) + count = cursor.fetchone()[0] + + # Get metadata schema + metadata_schema = self.get_metadata_schema(table_name) + + # Get catalog and metrics data + catalog_data = self.get_document_info( + info_type="catalog", + collection_name=table_name, + document_name="NA", + ) + metrics_data = self.get_document_info( + info_type="collection", + collection_name=table_name, + document_name="NA", + ) + + collections.append({ + "collection_name": table_name, + "num_entities": count, + "metadata_schema": metadata_schema, + "collection_info": {**metrics_data, **catalog_data}, + }) + + return collections + + def delete_collections(self, collection_names: list[str]) -> dict[str, Any]: + """Delete collections.""" + deleted = [] + failed = [] + + for name in collection_names: + table_name = name.upper() + try: + if self._table_exists(table_name): + with self._get_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(drop_table_ddl(table_name)) + conn.commit() + deleted.append(name) + logger.info(f"Deleted collection: {name}") + + # Clean up metadata + self._delete_collection_metadata(table_name) + else: + failed.append({ + "collection_name": name, + "error_message": f"Collection {name} not found.", + }) + except Exception as e: + failed.append({ + "collection_name": name, + "error_message": str(e), + }) + logger.exception(f"Failed to delete collection {name}") + + return { + "message": "Collection deletion completed.", + "successful": deleted, + "failed": failed, + "total_success": len(deleted), + "total_failed": len(failed), + } + + def _delete_collection_metadata(self, collection_name: str) -> None: + """Delete metadata and document info for a collection.""" + with self._get_connection() as conn: + with conn.cursor() as cursor: + try: + cursor.execute( + get_delete_metadata_schema_query(), + {"collection_name": collection_name}, + ) + except Exception as e: + logger.warning(f"Error deleting metadata schema: {e}") + + try: + cursor.execute( + get_delete_document_info_by_collection_query(), + {"collection_name": collection_name}, + ) + except Exception as e: + logger.warning(f"Error deleting document info: {e}") + + conn.commit() + + # ------------------------------------------------------------------------- + # Document Management + def get_documents(self, collection_name: str) -> list[dict[str, Any]]: + """Get all documents in a collection.""" + table_name = collection_name.upper() + metadata_schema = self.get_metadata_schema(table_name) + + documents = [] + with self._get_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(get_unique_sources_query(table_name)) + for row in cursor: + source_name = row[0] + content_metadata = json.loads(row[1]) if row[1] else {} + + metadata_dict = {} + for item in metadata_schema: + field_name = item.get("name") + metadata_dict[field_name] = content_metadata.get(field_name) + + doc_info = self.get_document_info( + info_type="document", + collection_name=table_name, + document_name=os.path.basename(source_name), + ) + + documents.append({ + "document_name": os.path.basename(source_name), + "metadata": metadata_dict, + "document_info": doc_info, + }) + + return documents + + def delete_documents( + self, + collection_name: str, + source_values: list[str], + result_dict: dict[str, list[str]] | None = None, + ) -> bool: + """Delete documents by source values.""" + table_name = collection_name.upper() + + if result_dict is not None: + result_dict["deleted"] = [] + result_dict["not_found"] = [] + + existing_docs = set() + if result_dict is not None: + try: + all_docs = self.get_documents(collection_name) + existing_docs = {doc.get("document_name", "") for doc in all_docs} + except Exception as e: + logger.warning(f"Failed to check existing documents: {e}") + + with self._get_connection() as conn: + with conn.cursor() as cursor: + for source_value in source_values: + doc_basename = os.path.basename(source_value) + try: + cursor.execute( + get_delete_docs_query(table_name), + {"source_value": source_value}, + ) + deleted_count = cursor.rowcount + + if result_dict is not None: + if deleted_count > 0 or doc_basename in existing_docs: + result_dict["deleted"].append(doc_basename) + else: + result_dict["not_found"].append(doc_basename) + except Exception as e: + logger.warning(f"Failed to delete {source_value}: {e}") + if result_dict is not None: + result_dict["not_found"].append(doc_basename) + + conn.commit() + + return True + + # ------------------------------------------------------------------------- + # Metadata Schema Management + def create_metadata_schema_collection(self) -> None: + """Create metadata schema table if not exists.""" + if self._metadata_schema_initialized: + return + + if not self._table_exists(DEFAULT_METADATA_SCHEMA_COLLECTION): + with self._get_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(create_metadata_schema_table_ddl()) + conn.commit() + logger.info(f"Created {DEFAULT_METADATA_SCHEMA_COLLECTION} table") + + self._metadata_schema_initialized = True + + def add_metadata_schema( + self, + collection_name: str, + metadata_schema: list[dict[str, Any]], + ) -> None: + """Add or update metadata schema for a collection.""" + table_name = collection_name.upper() + + with self._get_connection() as conn: + with conn.cursor() as cursor: + # Delete existing + cursor.execute( + get_delete_metadata_schema_query(), + {"collection_name": table_name}, + ) + + # Insert new + cursor.execute( + """ + INSERT INTO metadata_schema (collection_name, metadata_schema) + VALUES (:collection_name, :metadata_schema) + """, + { + "collection_name": table_name, + "metadata_schema": json.dumps(metadata_schema), + }, + ) + conn.commit() + + logger.info(f"Added metadata schema for {table_name}") + + def get_metadata_schema(self, collection_name: str) -> list[dict[str, Any]]: + """Get metadata schema for a collection.""" + table_name = collection_name.upper() + + with self._get_connection() as conn: + with conn.cursor() as cursor: + cursor.execute( + get_metadata_schema_query(), + {"collection_name": table_name}, + ) + row = cursor.fetchone() + if row and row[0]: + return json.loads(row[0]) + + logger.info(f"No metadata schema found for {table_name}") + return [] + + # ------------------------------------------------------------------------- + # Document Info Management + def create_document_info_collection(self) -> None: + """Create document info table if not exists.""" + if self._document_info_initialized: + return + + if not self._table_exists(DEFAULT_DOCUMENT_INFO_COLLECTION): + with self._get_connection() as conn: + with conn.cursor() as cursor: + cursor.execute(create_document_info_table_ddl()) + conn.commit() + logger.info(f"Created {DEFAULT_DOCUMENT_INFO_COLLECTION} table") + + self._document_info_initialized = True + + def add_document_info( + self, + info_type: str, + collection_name: str, + document_name: str, + info_value: dict[str, Any], + ) -> None: + """Add document info.""" + table_name = collection_name.upper() + + # Aggregate collection info + if info_type == "collection": + info_value = self._get_aggregated_document_info(table_name, info_value) + + with self._get_connection() as conn: + with conn.cursor() as cursor: + # Delete existing + cursor.execute( + get_delete_document_info_query(), + { + "collection_name": table_name, + "document_name": document_name, + "info_type": info_type, + }, + ) + + # Insert new + cursor.execute( + """ + INSERT INTO document_info (collection_name, info_type, document_name, info_value) + VALUES (:collection_name, :info_type, :document_name, :info_value) + """, + { + "collection_name": table_name, + "info_type": info_type, + "document_name": document_name, + "info_value": json.dumps(info_value), + }, + ) + conn.commit() + + logger.info(f"Added document info for {table_name}/{document_name}") + + def _get_aggregated_document_info( + self, + collection_name: str, + info_value: dict[str, Any], + ) -> dict[str, Any]: + """Get aggregated document info for a collection.""" + try: + with self._get_connection() as conn: + with conn.cursor() as cursor: + cursor.execute( + get_collection_document_info_query(), + {"collection_name": collection_name, "info_type": "collection"}, + ) + row = cursor.fetchone() + if row and row[0]: + existing = json.loads(row[0]) + return perform_document_info_aggregation(existing, info_value) + except Exception as e: + logger.warning(f"Error getting aggregated info: {e}") + + return info_value + + def get_document_info( + self, + info_type: str, + collection_name: str, + document_name: str, + ) -> dict[str, Any]: + """Get document info.""" + table_name = collection_name.upper() + + with self._get_connection() as conn: + with conn.cursor() as cursor: + cursor.execute( + get_document_info_query(), + { + "collection_name": table_name, + "document_name": document_name, + "info_type": info_type, + }, + ) + row = cursor.fetchone() + if row and row[0]: + return json.loads(row[0]) + + return {} + + def get_catalog_metadata(self, collection_name: str) -> dict[str, Any]: + """Get catalog metadata for a collection.""" + return self.get_document_info( + info_type="catalog", + collection_name=collection_name, + document_name="NA", + ) + + def update_catalog_metadata( + self, + collection_name: str, + updates: dict[str, Any], + ) -> None: + """Update catalog metadata for a collection.""" + existing = self.get_catalog_metadata(collection_name) + merged = {**existing, **updates} + merged["last_updated"] = get_current_timestamp() + + self.add_document_info( + info_type="catalog", + collection_name=collection_name, + document_name="NA", + info_value=merged, + ) + + def get_document_catalog_metadata( + self, + collection_name: str, + document_name: str, + ) -> dict[str, Any]: + """Get catalog metadata for a document.""" + doc_info = self.get_document_info( + info_type="document", + collection_name=collection_name, + document_name=document_name, + ) + return { + "description": doc_info.get("description", ""), + "tags": doc_info.get("tags", []), + } + + def update_document_catalog_metadata( + self, + collection_name: str, + document_name: str, + updates: dict[str, Any], + ) -> None: + """Update catalog metadata for a document.""" + existing = self.get_document_info( + info_type="document", + collection_name=collection_name, + document_name=document_name, + ) + + for key in ["description", "tags"]: + if key in updates: + existing[key] = updates[key] + + self.add_document_info( + info_type="document", + collection_name=collection_name, + document_name=document_name, + info_value=existing, + ) + + # ------------------------------------------------------------------------- + # Retrieval Operations + def get_langchain_vectorstore(self, collection_name: str) -> OracleVS: + """Get LangChain OracleVS vectorstore.""" + table_name = collection_name.upper() + + # Map distance metric + distance_map = { + "COSINE": DistanceStrategy.COSINE, + "L2": DistanceStrategy.EUCLIDEAN_DISTANCE, + "DOT": DistanceStrategy.DOT_PRODUCT, + } + distance_strategy = distance_map.get(self._distance_metric, DistanceStrategy.COSINE) + + # Create connection for OracleVS + conn = oracledb.connect( + user=self._oracle_user, + password=self._oracle_password, + dsn=self._oracle_dsn, + ) + + return OracleVS( + client=conn, + embedding_function=self._embedding_model, + table_name=table_name, + distance_strategy=distance_strategy, + ) + + def retrieval_langchain( + self, + query: str, + collection_name: str, + vectorstore: OracleVS | None = None, + top_k: int = 10, + filter_expr: str | list[dict[str, Any]] = "", + otel_ctx: Any | None = None, + ) -> list[Document]: + """Perform semantic search and return documents.""" + table_name = collection_name.upper() + + logger.info( + "Oracle Retrieval: Retrieving from %s, search type: %s", + table_name, + "hybrid" if self._hybrid else "vector", + ) + + if vectorstore is None: + vectorstore = self.get_langchain_vectorstore(collection_name) + + token = otel_context.attach(otel_ctx) if otel_ctx is not None else None + + try: + start_time = time.time() + + logger.info(" [Embedding] Generating query embedding...") + retriever = vectorstore.as_retriever(search_kwargs={"k": top_k}) + + retriever_lambda = RunnableLambda(lambda x: retriever.invoke(x)) + retriever_chain = {"context": retriever_lambda} | RunnableAssign( + {"context": lambda inp: inp["context"]} + ) + + logger.info(" [VDB Search] Performing vector similarity search...") + result = retriever_chain.invoke(query, config={"run_name": "retriever"}) + docs = result.get("context", []) + + latency = time.time() - start_time + logger.info(" [VDB Search] Retrieved %d documents in %.4fs", len(docs), latency) + + return self._add_collection_name_to_retrieved_docs(docs, collection_name) + + except Exception as e: + logger.exception("Error in retrieval_langchain: %s", e) + raise APIError( + f"Oracle retrieval failed: {e}", + ErrorCodeMapping.SERVICE_UNAVAILABLE, + ) from e + finally: + if token is not None: + otel_context.detach(token) + + @staticmethod + def _add_collection_name_to_retrieved_docs( + docs: list[Document], + collection_name: str, + ) -> list[Document]: + """Add collection name to document metadata.""" + for doc in docs: + doc.metadata["collection_name"] = collection_name + return docs From 9d411e45a9d509c2d10a7cf1620ee4ee2364210e Mon Sep 17 00:00:00 2001 From: Swapnil Masurekar Date: Tue, 24 Feb 2026 01:54:48 +0530 Subject: [PATCH 02/81] fix(observability): use hardcoded GenAI attribute keys instead of deprecated SpanAttributes (#377) Signed-off-by: Swapnil Masurekar --- .../langchain_callback_handler.py | 53 ++++---- .../test_langchain_callback_handler.py | 119 ++++++++++++------ 2 files changed, 111 insertions(+), 61 deletions(-) diff --git a/src/nvidia_rag/utils/observability/langchain_callback_handler.py b/src/nvidia_rag/utils/observability/langchain_callback_handler.py index dcdd7ba23..a35a71bab 100644 --- a/src/nvidia_rag/utils/observability/langchain_callback_handler.py +++ b/src/nvidia_rag/utils/observability/langchain_callback_handler.py @@ -40,6 +40,17 @@ from .otel_metrics import OtelMetrics +# Hardcoded attribute keys (replacing deprecated SpanAttributes constants) +GEN_AI_PROMPTS = "gen_ai.prompt" +GEN_AI_COMPLETIONS = "gen_ai.completion" +LLM_REQUEST_MODEL = "gen_ai.request.model" +LLM_RESPONSE_MODEL = "gen_ai.response.model" +# Missing in opentelemetry.semconv_ai SpanAttributes (use llm.* to match existing semconv) +LLM_REQUEST_MAX_TOKENS = "llm.request.max_tokens" +LLM_REQUEST_TEMPERATURE = "llm.request.temperature" +LLM_REQUEST_TOP_P = "llm.request.top_p" +LLM_SYSTEM = "llm.system" + class Config: exception_logger = None @@ -137,9 +148,9 @@ def _set_request_params(span, kwargs, span_holder: SpanHolder): else: model = "unknown" - span.set_attribute(SpanAttributes.LLM_REQUEST_MODEL, model) + span.set_attribute(LLM_REQUEST_MODEL, model) # response is not available for LLM requests (as opposed to chat) - span.set_attribute(SpanAttributes.LLM_RESPONSE_MODEL, model) + span.set_attribute(LLM_RESPONSE_MODEL, model) if "invocation_params" in kwargs: params = ( @@ -150,13 +161,11 @@ def _set_request_params(span, kwargs, span_holder: SpanHolder): _set_span_attribute( span, - SpanAttributes.LLM_REQUEST_MAX_TOKENS, + LLM_REQUEST_MAX_TOKENS, params.get("max_tokens") or params.get("max_new_tokens"), ) - _set_span_attribute( - span, SpanAttributes.LLM_REQUEST_TEMPERATURE, params.get("temperature") - ) - _set_span_attribute(span, SpanAttributes.LLM_REQUEST_TOP_P, params.get("top_p")) + _set_span_attribute(span, LLM_REQUEST_TEMPERATURE, params.get("temperature")) + _set_span_attribute(span, LLM_REQUEST_TOP_P, params.get("top_p")) def _set_llm_request( @@ -171,11 +180,11 @@ def _set_llm_request( if should_send_prompts(): for i, msg in enumerate(prompts): span.set_attribute( - f"{SpanAttributes.LLM_PROMPTS}.{i}.role", + f"{GEN_AI_PROMPTS}.{i}.role", "user", ) span.set_attribute( - f"{SpanAttributes.LLM_PROMPTS}.{i}.content", + f"{GEN_AI_PROMPTS}.{i}.content", msg, ) @@ -207,18 +216,18 @@ def _set_chat_request( for message in messages: for msg in message: span.set_attribute( - f"{SpanAttributes.LLM_PROMPTS}.{i}.role", + f"{GEN_AI_PROMPTS}.{i}.role", _message_type_to_role(msg.type), ) # if msg.content is string if isinstance(msg.content, str): span.set_attribute( - f"{SpanAttributes.LLM_PROMPTS}.{i}.content", + f"{GEN_AI_PROMPTS}.{i}.content", msg.content, ) else: span.set_attribute( - f"{SpanAttributes.LLM_PROMPTS}.{i}.content", + f"{GEN_AI_PROMPTS}.{i}.content", json.dumps(msg.content, cls=CallbackFilteredJSONEncoder), ) i += 1 @@ -252,7 +261,7 @@ def _set_chat_response(span: Span, response: LLMResult) -> None: ) total_tokens = input_tokens + output_tokens - prefix = f"{SpanAttributes.LLM_COMPLETIONS}.{i}" + prefix = f"{GEN_AI_COMPLETIONS}.{i}" if hasattr(generation, "text") and generation.text != "": span.set_attribute( f"{prefix}.content", @@ -317,11 +326,11 @@ def _set_chat_response(span: Span, response: LLMResult) -> None: if input_tokens > 0 or output_tokens > 0 or total_tokens > 0: span.set_attribute( - SpanAttributes.LLM_USAGE_PROMPT_TOKENS, + "gen_ai.usage.input_tokens", input_tokens, ) span.set_attribute( - SpanAttributes.LLM_USAGE_COMPLETION_TOKENS, + "gen_ai.usage.output_tokens", output_tokens, ) span.set_attribute( @@ -462,7 +471,7 @@ def _create_llm_span( entity_path=entity_path, metadata=metadata, ) - span.set_attribute(SpanAttributes.LLM_SYSTEM, "Langchain") + span.set_attribute(LLM_SYSTEM, "Langchain") span.set_attribute(SpanAttributes.LLM_REQUEST_TYPE, request_type.value) return span @@ -650,10 +659,10 @@ def on_llm_end( "model_name" ) or response.llm_output.get("model_id") if model_name is not None: - span.set_attribute(SpanAttributes.LLM_RESPONSE_MODEL, model_name) + span.set_attribute(LLM_RESPONSE_MODEL, model_name) if self.spans[run_id].request_model is None: - span.set_attribute(SpanAttributes.LLM_REQUEST_MODEL, model_name) + span.set_attribute(LLM_REQUEST_MODEL, model_name) token_usage = (response.llm_output or {}).get("token_usage") or ( response.llm_output or {} @@ -673,12 +682,8 @@ def on_llm_end( prompt_tokens + completion_tokens ) - _set_span_attribute( - span, SpanAttributes.LLM_USAGE_PROMPT_TOKENS, prompt_tokens - ) - _set_span_attribute( - span, SpanAttributes.LLM_USAGE_COMPLETION_TOKENS, completion_tokens - ) + _set_span_attribute(span, "gen_ai.usage.input_tokens", prompt_tokens) + _set_span_attribute(span, "gen_ai.usage.output_tokens", completion_tokens) _set_span_attribute( span, SpanAttributes.LLM_USAGE_TOTAL_TOKENS, total_tokens ) diff --git a/tests/unit/test_observability/test_langchain_callback_handler.py b/tests/unit/test_observability/test_langchain_callback_handler.py index 054f5d18d..80511caab 100644 --- a/tests/unit/test_observability/test_langchain_callback_handler.py +++ b/tests/unit/test_observability/test_langchain_callback_handler.py @@ -6,6 +6,12 @@ import pytest from langchain_core.messages import AIMessageChunk from langchain_core.outputs import Generation, LLMResult +from opentelemetry.semconv_ai import ( + LLMRequestTypeValues, + SpanAttributes, + SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY, + TraceloopSpanKindValues, +) class SpanMock: @@ -58,7 +64,9 @@ def handler(): def test_on_chat_model_start_sets_input_words_and_prompts(handler): - from nvidia_rag.utils.observability.langchain_callback_handler import SpanAttributes + from nvidia_rag.utils.observability.langchain_callback_handler import ( + GEN_AI_PROMPTS, + ) run_id = uuid4() messages = [ @@ -78,44 +86,8 @@ def test_on_chat_model_start_sets_input_words_and_prompts(handler): assert handler.total_input_words == 4 # span should be created and attributes recorded assert run_id in handler.spans - span = handler.spans[run_id].span - # Check that at least one prompt attribute key prefix was used - prompt_prefix = f"{SpanAttributes.LLM_PROMPTS}." - prompt_keys = [k for k, _ in span.attributes if k.startswith(prompt_prefix)] - assert len(prompt_keys) >= 2 - - -def test_on_llm_start_and_end_sets_token_usage_and_ends_span(handler): - run_id = uuid4() - handler.on_llm_start( - serialized={"kwargs": {"name": "llm"}}, - prompts=["What is ML?"], - run_id=run_id, - ) - # Build a minimal valid LLMResult - gen = Generation( - text="Answer", - generation_info={"finish_reason": "stop"}, - ) - - llm_result = LLMResult( - generations=[[gen]], - llm_output={ - "model_name": "test-model", - "usage": {"prompt_tokens": 5, "completion_tokens": 7, "total_tokens": 12}, - }, - ) - - handler.on_llm_end(response=llm_result, run_id=run_id) - - span = handler.spans[run_id].span - # Verify span ended - assert span.ended is True - # Verify some token usage attributes were set from llm_output usage - attr_keys = [k for k, _ in span.attributes] - assert any("usage" in k.lower() for k in attr_keys) def test_on_chain_end_updates_avg_words_per_chunk(handler): @@ -156,3 +128,76 @@ def test_on_chain_end_updates_llm_tokens(handler): # Expect update_llm_tokens called with input words from chat (3) and output words (2) assert handler.metrics.token_calls[-1] == (3, 2) + + +# SpanAttributes from opentelemetry.semconv_ai still used in langchain_callback_handler.py. +# Missing/deprecated ones (LLM_REQUEST_MODEL, LLM_RESPONSE_MODEL, LLM_REQUEST_MAX_TOKENS, +# LLM_REQUEST_TEMPERATURE, LLM_REQUEST_TOP_P, LLM_SYSTEM) are hardcoded in the handler. +SPAN_ATTRIBUTES_USED = [ + "LLM_REQUEST_FUNCTIONS", + "LLM_USAGE_TOTAL_TOKENS", + "TRACELOOP_WORKFLOW_NAME", + "TRACELOOP_ENTITY_PATH", + "TRACELOOP_SPAN_KIND", + "TRACELOOP_ENTITY_NAME", + "LLM_REQUEST_TYPE", + "TRACELOOP_ENTITY_INPUT", + "TRACELOOP_ENTITY_OUTPUT", +] + + +def test_semconv_ai_span_attributes_exist_and_not_deprecated(): + """Ensure all SpanAttributes used in langchain_callback_handler exist and are non-empty strings.""" + for attr_name in SPAN_ATTRIBUTES_USED: + assert hasattr( + SpanAttributes, attr_name + ), f"SpanAttributes.{attr_name} is missing or was removed from opentelemetry.semconv_ai" + value = getattr(SpanAttributes, attr_name) + assert isinstance( + value, str + ), f"SpanAttributes.{attr_name} should be a string, got {type(value).__name__}" + assert ( + len(value) > 0 + ), f"SpanAttributes.{attr_name} is empty (possibly deprecated or placeholder)" + + +def test_semconv_ai_llm_request_type_values_used(): + """Ensure LLMRequestTypeValues used in the handler (CHAT, COMPLETION) exist.""" + assert hasattr(LLMRequestTypeValues, "CHAT") + assert hasattr(LLMRequestTypeValues, "COMPLETION") + assert isinstance(LLMRequestTypeValues.CHAT.value, str) + assert isinstance(LLMRequestTypeValues.COMPLETION.value, str) + + +def test_semconv_ai_traceloop_span_kind_values_used(): + """Ensure TraceloopSpanKindValues used in the handler (WORKFLOW, TASK, TOOL) exist.""" + for kind in ("WORKFLOW", "TASK", "TOOL"): + assert hasattr( + TraceloopSpanKindValues, kind + ), f"TraceloopSpanKindValues.{kind} is missing" + assert isinstance(getattr(TraceloopSpanKindValues, kind).value, str) + + +def test_langchain_callback_handler_imports_and_constants(): + """Import the handler module and verify hardcoded attribute constants and semconv_ai key.""" + from nvidia_rag.utils.observability.langchain_callback_handler import ( + GEN_AI_COMPLETIONS, + GEN_AI_PROMPTS, + LLM_REQUEST_MAX_TOKENS, + LLM_REQUEST_MODEL, + LLM_REQUEST_TEMPERATURE, + LLM_REQUEST_TOP_P, + LLM_RESPONSE_MODEL, + LLM_SYSTEM, + ) + + assert GEN_AI_PROMPTS == "gen_ai.prompt" + assert GEN_AI_COMPLETIONS == "gen_ai.completion" + assert LLM_REQUEST_MODEL == "gen_ai.request.model" + assert LLM_RESPONSE_MODEL == "gen_ai.response.model" + assert LLM_REQUEST_MAX_TOKENS == "llm.request.max_tokens" + assert LLM_REQUEST_TEMPERATURE == "llm.request.temperature" + assert LLM_REQUEST_TOP_P == "llm.request.top_p" + assert LLM_SYSTEM == "llm.system" + assert SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY is not None + assert isinstance(SUPPRESS_LANGUAGE_MODEL_INSTRUMENTATION_KEY, str) From 51a82a03d113f7f6d6aa630557d22f7165591e93 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Tue, 24 Feb 2026 13:05:29 -0600 Subject: [PATCH 03/81] added changes --- .github/workflows/build-push-main.yml | 89 ++++++++++++++++++++++++++ .github/workflows/build-push-pr.yml | 91 +++++++++++++++++++++++++++ deploy/compose/vectordb.yaml | 2 +- src/nvidia_rag/utils/vdb/__init__.py | 20 +++++- 4 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build-push-main.yml create mode 100644 .github/workflows/build-push-pr.yml diff --git a/.github/workflows/build-push-main.yml b/.github/workflows/build-push-main.yml new file mode 100644 index 000000000..645cab4c6 --- /dev/null +++ b/.github/workflows/build-push-main.yml @@ -0,0 +1,89 @@ +name: Build and Push on Main + +on: + push: + branches: + - oracle/dev + +jobs: + build-and-push: + runs-on: ubuntu-latest + env: + # OCI AUTH + OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }} + OCI_CLI_TENANCY: ${{ secrets.OCI_CLI_TENANCY }} + OCI_CLI_FINGERPRINT: ${{ secrets.OCI_CLI_FINGERPRINT }} + OCI_CLI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }} + OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }} + # IMAGE VARS + RAG_IMAGE_NAME: corrino-devops-repository/nvidia-rag-retrieval-oci + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log into OCIR + uses: oracle-actions/login-ocir@v1.3.0 + id: login-ocir + with: + auth_token: ${{ secrets.OCI_AUTH_TOKEN }} + + - name: Compute short SHA + id: short-sha + run: echo "value=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + + - name: Build Docker Image + run: | + docker build -t "${{ env.RAG_IMAGE_NAME }}":latest -f ./Dockerfile . + + - name: Tag and Push Docker Image + env: + IMAGE_REMOTE: "iad.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.RAG_IMAGE_NAME }}" + run: | + docker tag "${{ env.RAG_IMAGE_NAME }}":latest "${{ env.IMAGE_REMOTE }}:latest" + docker push "${{ env.IMAGE_REMOTE }}:latest" + + build-and-push-ingest: + runs-on: ubuntu-latest + env: + # OCI AUTH + OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }} + OCI_CLI_TENANCY: ${{ secrets.OCI_CLI_TENANCY }} + OCI_CLI_FINGERPRINT: ${{ secrets.OCI_CLI_FINGERPRINT }} + OCI_CLI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }} + OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }} + # IMAGE VARS + INGEST_IMAGE_NAME: corrino-devops-repository/nvidia-rag-ingestion-oci + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log into OCIR + uses: oracle-actions/login-ocir@v1.3.0 + id: login-ocir + with: + auth_token: ${{ secrets.OCI_AUTH_TOKEN }} + + - name: Compute short SHA + id: short-sha + run: echo "value=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + + - name: Build Docker Image + run: | + docker build -t "${{ env.INGEST_IMAGE_NAME }}":latest -f ./Dockerfile . + + - name: Tag and Push Docker Image + env: + IMAGE_REMOTE: "iad.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.INGEST_IMAGE_NAME }}" + run: | + docker tag "${{ env.INGEST_IMAGE_NAME }}":latest "${{ env.IMAGE_REMOTE }}:latest" + docker push "${{ env.IMAGE_REMOTE }}:latest" \ No newline at end of file diff --git a/.github/workflows/build-push-pr.yml b/.github/workflows/build-push-pr.yml new file mode 100644 index 000000000..7e6f6c407 --- /dev/null +++ b/.github/workflows/build-push-pr.yml @@ -0,0 +1,91 @@ + +name: Build and Push on PR + +on: + pull_request_review: + types: [submitted] + +jobs: + build-and-push-rag: + runs-on: ubuntu-latest + if: github.event.review.state == 'approved' + env: + # OCI AUTH + OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }} + OCI_CLI_TENANCY: ${{ secrets.OCI_CLI_TENANCY }} + OCI_CLI_FINGERPRINT: ${{ secrets.OCI_CLI_FINGERPRINT }} + OCI_CLI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }} + OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }} + # IMAGE VARS + RAG_IMAGE_NAME: corrino-devops-repository/nvidia-rag-retrieval-oci + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log into OCIR + uses: oracle-actions/login-ocir@v1.3.0 + id: login-ocir + with: + auth_token: ${{ secrets.OCI_AUTH_TOKEN }} + + - name: Compute short SHA + id: short-sha + run: echo "value=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + + - name: Build RAG Docker Image + run: | + docker build -t "${{ env.RAG_IMAGE_NAME }}":pr-${{ steps.short-sha.outputs.value }} -f ./Dockerfile . + + - name: Tag and Push RAG Docker Image + env: + IMAGE_REMOTE: "iad.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.RAG_IMAGE_NAME }}" + run: | + docker tag "${{ env.RAG_IMAGE_NAME }}:pr-${{ steps.short-sha.outputs.value }}" "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" + docker push "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" + + build-and-push-ingestion: + runs-on: ubuntu-latest + if: github.event.review.state == 'approved' + env: + # OCI AUTH + OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }} + OCI_CLI_TENANCY: ${{ secrets.OCI_CLI_TENANCY }} + OCI_CLI_FINGERPRINT: ${{ secrets.OCI_CLI_FINGERPRINT }} + OCI_CLI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }} + OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }} + # IMAGE VARS + INGEST_IMAGE_NAME: corrino-devops-repository/nvidia-rag-ingestion-oci + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log into OCIR + uses: oracle-actions/login-ocir@v1.3.0 + id: login-ocir + with: + auth_token: ${{ secrets.OCI_AUTH_TOKEN }} + + - name: Compute short SHA + id: short-sha + run: echo "value=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + + - name: Build Ingest Docker Image + run: | + docker build -t "${{ env.INGEST_IMAGE_NAME }}":pr-${{ steps.short-sha.outputs.value }} -f ./Dockerfile . + + - name: Tag and Push Ingest Docker Image + env: + IMAGE_REMOTE: "iad.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.INGEST_IMAGE_NAME }}" + run: | + docker tag "${{ env.INGEST_IMAGE_NAME }}:pr-${{ steps.short-sha.outputs.value }}" "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" + docker push "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" diff --git a/deploy/compose/vectordb.yaml b/deploy/compose/vectordb.yaml index c113ebece..3bd798a4b 100644 --- a/deploy/compose/vectordb.yaml +++ b/deploy/compose/vectordb.yaml @@ -78,7 +78,7 @@ services: interval: 30s timeout: 20s retries: 3 - profiles: ["", "milvus", "elasticsearch", "minio"] + profiles: ["", "milvus", "elasticsearch", "minio", "oracle"] elasticsearch: container_name: elasticsearch diff --git a/src/nvidia_rag/utils/vdb/__init__.py b/src/nvidia_rag/utils/vdb/__init__.py index f8895eb28..c3a51b85e 100644 --- a/src/nvidia_rag/utils/vdb/__init__.py +++ b/src/nvidia_rag/utils/vdb/__init__.py @@ -63,7 +63,25 @@ def _get_vdb_op( ) # Get VDBRag class object based on the configuration - if config.vector_store.name == "milvus": + # Oracle 26ai is the default vector store + if config.vector_store.name == "oracle": + from nvidia_rag.utils.vdb.oracle.oracle_vdb import OracleVDB + + return OracleVDB( + collection_name=collection_name, + oracle_user=os.getenv("ORACLE_USER"), + oracle_password=os.getenv("ORACLE_PASSWORD"), + oracle_dsn=os.getenv("ORACLE_DSN"), + embedding_model=embedding_model, + config=config, + meta_dataframe=csv_file_path, + meta_source_field=meta_source_field, + meta_fields=meta_fields, + csv_file_path=csv_file_path, + hybrid=(config.vector_store.search_type == SearchType.HYBRID), + ) + + elif config.vector_store.name == "milvus": from nvidia_rag.utils.vdb.milvus.milvus_vdb import MilvusVDB return MilvusVDB( From fe72e3e55ce67bb0e78fea21812d0f76b2ee47a0 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Tue, 24 Feb 2026 13:14:31 -0600 Subject: [PATCH 04/81] rm approval for actions --- .github/workflows/build-push-pr.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-push-pr.yml b/.github/workflows/build-push-pr.yml index 7e6f6c407..0840e6e59 100644 --- a/.github/workflows/build-push-pr.yml +++ b/.github/workflows/build-push-pr.yml @@ -2,13 +2,12 @@ name: Build and Push on PR on: - pull_request_review: - types: [submitted] + pull_request: + types: [synchronize] jobs: build-and-push-rag: - runs-on: ubuntu-latest - if: github.event.review.state == 'approved' + runs-on: ubuntu-latesta env: # OCI AUTH OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }} @@ -50,7 +49,6 @@ jobs: build-and-push-ingestion: runs-on: ubuntu-latest - if: github.event.review.state == 'approved' env: # OCI AUTH OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }} From 052a5c4669c081d8d9c253b35c2bccd9394af122 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Wed, 25 Feb 2026 17:10:25 -0600 Subject: [PATCH 05/81] runs with errors --- pyproject.toml | 7 ++++ src/nvidia_rag/ingestor_server/Dockerfile | 2 +- src/nvidia_rag/utils/vdb/__init__.py | 8 ++-- src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py | 34 +++++++++++---- uv.lock | 41 ++++++++++++++++++- 5 files changed, 77 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 09b647403..3ce5fbff4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "protobuf>=6.33.5", "lark>=1.2.2", "python-dateutil>=2.9.0.post0", + "oracledb>=3.4.2", ] [project.optional-dependencies] @@ -100,10 +101,16 @@ all = [ "pyarrow>=21.0,<22.0", # Elasticsearch support "langchain-elasticsearch>=0.3", + # Oracle 26ai support + "oracledb>=2.0.0", ] elasticsearch = [ "langchain-elasticsearch>=0.3", ] +oracle = [ + "oracledb>=2.0.0", + "langchain-community>=0.4", +] [tool.uv.sources] nvidia-rag = { workspace = true } diff --git a/src/nvidia_rag/ingestor_server/Dockerfile b/src/nvidia_rag/ingestor_server/Dockerfile index b484849ca..eccfeffd6 100644 --- a/src/nvidia_rag/ingestor_server/Dockerfile +++ b/src/nvidia_rag/ingestor_server/Dockerfile @@ -71,7 +71,7 @@ WORKDIR /workspace COPY --from=builder /build/pyproject.toml /workspace/ COPY --from=builder /build/uv.lock /workspace/ -RUN uv sync --locked --no-install-project --no-dev --extra ingest --extra elasticsearch +RUN uv sync --locked --no-install-project --no-dev --extra ingest --extra oracle COPY --from=builder /build/dist/*.whl /workspace/ diff --git a/src/nvidia_rag/utils/vdb/__init__.py b/src/nvidia_rag/utils/vdb/__init__.py index c3a51b85e..e4c12f720 100644 --- a/src/nvidia_rag/utils/vdb/__init__.py +++ b/src/nvidia_rag/utils/vdb/__init__.py @@ -63,15 +63,17 @@ def _get_vdb_op( ) # Get VDBRag class object based on the configuration - # Oracle 26ai is the default vector store + # Oracle 26ai is the default vector store if config.vector_store.name == "oracle": from nvidia_rag.utils.vdb.oracle.oracle_vdb import OracleVDB - return OracleVDB( collection_name=collection_name, oracle_user=os.getenv("ORACLE_USER"), oracle_password=os.getenv("ORACLE_PASSWORD"), - oracle_dsn=os.getenv("ORACLE_DSN"), + oracle_cs=os.getenv("ORACLE_CS"), + tnsnames_loc=os.getenv("ORACLE_TNSNAMES_LOC"), + ewallet_pem_loc=os.getenv("ORACLE_EWALLET_PEM_LOC"), + ewallet_password=os.getenv("ORACLE_EWALLET_PASSWORD"), embedding_model=embedding_model, config=config, meta_dataframe=csv_file_path, diff --git a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py index f68cb1fa6..61e19a9e7 100644 --- a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py +++ b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py @@ -92,7 +92,10 @@ def __init__( collection_name: str, oracle_user: str | None = None, oracle_password: str | None = None, - oracle_dsn: str | None = None, + oracle_cs: str | None = None, + tnsnames_loc: str | None = None, + ewallet_pem_loc: str | None = None, + ewallet_password: str | None = None, embedding_model: Any | None = None, config: NvidiaRAGConfig | None = None, meta_dataframe: Any | None = None, @@ -128,11 +131,21 @@ def __init__( # Connection parameters from args or environment self._oracle_user = oracle_user or os.getenv("ORACLE_USER") self._oracle_password = oracle_password or os.getenv("ORACLE_PASSWORD") - self._oracle_dsn = oracle_dsn or os.getenv("ORACLE_DSN") - - if not all([self._oracle_user, self._oracle_password, self._oracle_dsn]): + self._oracle_cs = oracle_cs or os.getenv("ORACLE_CS") + self._tnsnames_loc = tnsnames_loc or os.getenv("ORACLE_TNSNAMES_LOC") + self._ewallet_pem_loc = ewallet_pem_loc or os.getenv("ORACLE_EWALLET_PEM_LOC") + self._ewallet_password = ewallet_password or os.getenv("ORACLE_EWALLET_PASSWORD") + + if not all([ + self._oracle_user, + self._oracle_password, + self._oracle_cs, + self._tnsnames_loc, + self._ewallet_pem_loc, + self._ewallet_password + ]): raise ValueError( - "Oracle connection requires ORACLE_USER, ORACLE_PASSWORD, and ORACLE_DSN. " + "Oracle connection requires ORACLE_USER, ORACLE_PASSWORD, ORACLE_CS, ORACLE_TNSNAMES_LOC, ORACLE_EWALLET_PEM_LOC and ORACLE_EWALLET_PASSWORD variables." "Set via parameters or environment variables." ) @@ -156,7 +169,10 @@ def __init__( self._pool = oracledb.create_pool( user=self._oracle_user, password=self._oracle_password, - dsn=self._oracle_dsn, + dsn=self._oracle_cs, + config_dir=self._tnsnames_loc, + wallet_location=self._ewallet_pem_loc, + wallet_password=self._ewallet_password, min=2, max=10, increment=1, @@ -165,11 +181,11 @@ def __init__( with self._pool.acquire() as conn: with conn.cursor() as cursor: cursor.execute("SELECT 1 FROM DUAL") - logger.info(f"Connected to Oracle at {self._oracle_dsn}") + logger.info(f"Connected to Oracle at {self._oracle_cs}") except oracledb.Error as e: - logger.exception("Failed to connect to Oracle at %s: %s", self._oracle_dsn, e) + logger.exception("Failed to connect to Oracle at %s: %s", self._oracle_cs, e) raise APIError( - f"Oracle database is unavailable at {self._oracle_dsn}. " + f"Oracle database is unavailable at {self._oracle_cs}. " "Please verify Oracle is running and credentials are correct.", ErrorCodeMapping.SERVICE_UNAVAILABLE, ) from e diff --git a/uv.lock b/uv.lock index dbbf310f2..b1af55d6b 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.11, <3.14" resolution-markers = [ "python_full_version >= '3.13'", @@ -1812,6 +1812,7 @@ dependencies = [ { name = "langchain-nvidia-ai-endpoints" }, { name = "lark" }, { name = "minio" }, + { name = "oracledb" }, { name = "pdfplumber" }, { name = "protobuf" }, { name = "pydantic" }, @@ -1840,6 +1841,7 @@ all = [ { name = "opentelemetry-instrumentation-milvus" }, { name = "opentelemetry-processor-baggage" }, { name = "opentelemetry-sdk" }, + { name = "oracledb" }, { name = "overrides" }, { name = "pyarrow" }, { name = "tqdm" }, @@ -1868,6 +1870,10 @@ ingest = [ { name = "tqdm" }, { name = "tritonclient" }, ] +oracle = [ + { name = "langchain-community" }, + { name = "oracledb" }, +] rag = [ { name = "azure-core" }, { name = "azure-storage-blob" }, @@ -1914,6 +1920,7 @@ requires-dist = [ { name = "httpx-sse", specifier = ">=0.4.3" }, { name = "langchain", specifier = ">=1.2.7" }, { name = "langchain-community", specifier = ">=0.4" }, + { name = "langchain-community", marker = "extra == 'oracle'", specifier = ">=0.4" }, { name = "langchain-core", specifier = ">=1.0.0" }, { name = "langchain-elasticsearch", marker = "extra == 'all'", specifier = ">=0.3" }, { name = "langchain-elasticsearch", marker = "extra == 'elasticsearch'", specifier = ">=0.3" }, @@ -1953,6 +1960,9 @@ requires-dist = [ { name = "opentelemetry-sdk", marker = "extra == 'ingest'", specifier = ">=1.29,<2.0" }, { name = "opentelemetry-sdk", marker = "extra == 'rag'", specifier = ">=1.29,<2.0" }, { name = "opentelemetry-sdk-extension-prometheus-multiprocess", marker = "extra == 'rag'", specifier = ">=1.0,<2.0" }, + { name = "oracledb", specifier = ">=3.4.2" }, + { name = "oracledb", marker = "extra == 'all'", specifier = ">=2.0.0" }, + { name = "oracledb", marker = "extra == 'oracle'", specifier = ">=2.0.0" }, { name = "overrides", marker = "extra == 'all'", specifier = ">=7.7,<8.0" }, { name = "overrides", marker = "extra == 'ingest'", specifier = ">=7.7,<8.0" }, { name = "pdfplumber", specifier = ">=0.11.9" }, @@ -1975,7 +1985,7 @@ requires-dist = [ { name = "tritonclient", marker = "extra == 'ingest'", specifier = "==2.57.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.32,<1.0" }, ] -provides-extras = ["rag", "ingest", "all", "elasticsearch"] +provides-extras = ["rag", "ingest", "all", "elasticsearch", "oracle"] [package.metadata.requires-dev] docs = [ @@ -2288,6 +2298,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/5c/d3f1733665f7cd582ef0842fb1d2ed0bc1fba10875160593342d22bba375/opentelemetry_util_http-0.60b1-py3-none-any.whl", hash = "sha256:66381ba28550c91bee14dcba8979ace443444af1ed609226634596b4b0faf199", size = 8947, upload-time = "2025-12-11T13:36:37.151Z" }, ] +[[package]] +name = "oracledb" +version = "3.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f7/02/70a872d1a4a739b4f7371ab8d3d5ed8c6e57e142e2503531aafcb220893c/oracledb-3.4.2.tar.gz", hash = "sha256:46e0f2278ff1fe83fbc33a3b93c72d429323ec7eed47bc9484e217776cd437e5", size = 855467, upload-time = "2026-01-28T17:25:39.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/80/be263b668ba32b258d07c85f7bfb6967a9677e016c299207b28734f04c4b/oracledb-3.4.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b8e4b8a852251cef09038b75f30fce1227010835f4e19cfbd436027acba2697c", size = 4228552, upload-time = "2026-01-28T17:25:54.844Z" }, + { url = "https://files.pythonhosted.org/packages/91/bc/e832a649529da7c60409a81be41f3213b4c7ffda4fe424222b2145e8d43c/oracledb-3.4.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1617a1db020346883455af005efbefd51be2c4d797e43b1b38455a19f8526b48", size = 2421924, upload-time = "2026-01-28T17:25:56.984Z" }, + { url = "https://files.pythonhosted.org/packages/86/21/d867c37e493a63b5521bd248110ad5b97b18253d64a30703e3e8f3d9631e/oracledb-3.4.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ed78d7e7079a778062744ccf42141ce4806818c3f4dd6463e4a7edd561c9f86", size = 2599301, upload-time = "2026-01-28T17:25:58.529Z" }, + { url = "https://files.pythonhosted.org/packages/2a/de/9b1843ea27f7791449652d7f340f042c3053336d2c11caf29e59bab86189/oracledb-3.4.2-cp311-cp311-win32.whl", hash = "sha256:0e16fe3d057e0c41a23ad2ae95bfa002401690773376d476be608f79ac74bf05", size = 1492890, upload-time = "2026-01-28T17:26:00.662Z" }, + { url = "https://files.pythonhosted.org/packages/d6/10/cbc8afa2db0cec80530858d3e4574f9734fae8c0b7f1df261398aa026c5f/oracledb-3.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:f93cae08e8ed20f2d5b777a8602a71f9418389c661d2c937e84d94863e7e7011", size = 1843355, upload-time = "2026-01-28T17:26:02.637Z" }, + { url = "https://files.pythonhosted.org/packages/8f/81/2e6154f34b71cd93b4946c73ea13b69d54b8d45a5f6bbffe271793240d21/oracledb-3.4.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a7396664e592881225ba66385ee83ce339d864f39003d6e4ca31a894a7e7c552", size = 4220806, upload-time = "2026-01-28T17:26:04.322Z" }, + { url = "https://files.pythonhosted.org/packages/ab/a9/a1d59aaac77d8f727156ec6a3b03399917c90b7da4f02d057f92e5601f56/oracledb-3.4.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f04a2d62073407672f114d02529921de0677c6883ed7c64d8d1a3c04caa3238", size = 2233795, upload-time = "2026-01-28T17:26:05.877Z" }, + { url = "https://files.pythonhosted.org/packages/94/ec/8c4a38020cd251572bd406ddcbde98ca052ec94b5684f9aa9ef1ddfcc68c/oracledb-3.4.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d8d75e4f879b908be66cce05ba6c05791a5dbb4a15e39abc01aa25c8a2492bd9", size = 2424756, upload-time = "2026-01-28T17:26:07.35Z" }, + { url = "https://files.pythonhosted.org/packages/fa/7d/c251c2a8567151ccfcfbe3467ea9a60fb5480dc4719342e2e6b7a9679e5d/oracledb-3.4.2-cp312-cp312-win32.whl", hash = "sha256:31b7ee83c23d0439778303de8a675717f805f7e8edb5556d48c4d8343bcf14f5", size = 1453486, upload-time = "2026-01-28T17:26:08.869Z" }, + { url = "https://files.pythonhosted.org/packages/4c/78/c939f3c16fb39400c4734d5a3340db5659ba4e9dce23032d7b33ccfd3fe5/oracledb-3.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:ac25a0448fc830fb7029ad50cd136cdbfcd06975d53967e269772cc5cb8c203a", size = 1794445, upload-time = "2026-01-28T17:26:10.66Z" }, + { url = "https://files.pythonhosted.org/packages/22/68/f7126f5d911c295b57720c6b1a0609a5a2667b4546946433552a4de46333/oracledb-3.4.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:643c25d301a289a371e37fcedb59e5fa5e54fb321708e5c12821c4b55bdd8a4d", size = 4205176, upload-time = "2026-01-28T17:26:12.463Z" }, + { url = "https://files.pythonhosted.org/packages/5d/93/2fced60f92dc82e66980a8a3ba5c1ea48110bf1dd81d030edb69d88f992e/oracledb-3.4.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55397e7eb43bb7017c03a981c736c25724182f5210951181dfe3fab0e5d457fb", size = 2231298, upload-time = "2026-01-28T17:26:14.497Z" }, + { url = "https://files.pythonhosted.org/packages/75/a7/4dd286f3a6348d786fef9e6ab2e6c9b74ca9195d9a756f2a67e45743cdf0/oracledb-3.4.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b26a10f9c790bd141ffc8af68520803ed4a44a9258bf7d1eea9bfdd36bd6df7f", size = 2439430, upload-time = "2026-01-28T17:26:16.044Z" }, + { url = "https://files.pythonhosted.org/packages/19/28/94bc753e5e969c60ee5d9c914e2b4ef79999eaca8e91bcab2fbf0586b80b/oracledb-3.4.2-cp313-cp313-win32.whl", hash = "sha256:b974caec2c330c22bbe765705a5ac7d98ec3022811dec2042d561a3c65cb991b", size = 1458209, upload-time = "2026-01-28T17:26:17.652Z" }, + { url = "https://files.pythonhosted.org/packages/cb/2b/593a9b2d4c12c9de3289e67d84fe023336d99f36ba51442a5a0f5ce6acf7/oracledb-3.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:3df8eee1410d25360599968b1625b000f10c5ae0e47274031a7842a9dc418890", size = 1793558, upload-time = "2026-01-28T17:26:19.914Z" }, +] + [[package]] name = "orjson" version = "3.11.5" From 467f86a0d00f2d0219888e7ea13792b442faf05a Mon Sep 17 00:00:00 2001 From: niyatisingal Date: Thu, 26 Feb 2026 13:07:00 +0530 Subject: [PATCH 06/81] Fixing nemoguardrails config (#382) Signed-off-by: Niyati Singal --- .../nemoguardrails/config-store/nemoguard_cloud/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/compose/nemoguardrails/config-store/nemoguard_cloud/config.yml b/deploy/compose/nemoguardrails/config-store/nemoguard_cloud/config.yml index 1e014fc9f..17200db05 100644 --- a/deploy/compose/nemoguardrails/config-store/nemoguard_cloud/config.yml +++ b/deploy/compose/nemoguardrails/config-store/nemoguard_cloud/config.yml @@ -17,5 +17,7 @@ rails: - content safety check input $model=content_safety - topic safety check input $model=topic_control output: + streaming: + enabled: true flows: - content safety check output $model=content_safety \ No newline at end of file From 5e1b9ed38d8c0bbdb932b6db3be9a7272453e8e7 Mon Sep 17 00:00:00 2001 From: kumar-punit Date: Thu, 26 Feb 2026 13:45:14 +0530 Subject: [PATCH 07/81] Added MIG Slice Support for rtx6000pro (#379) * Added MIG Slice support for RTX 6000 pro Signed-off-by: Punit Kumar * Changed to default config in MIG slicing in rtx6000pro config --------- Signed-off-by: Punit Kumar Co-authored-by: niyatisingal --- .../{mig-config.yaml => mig-config-h100.yaml} | 0 .../helm/mig-slicing/mig-config-rtx6000.yaml | 26 ++++ .../{values-mig.yaml => values-mig-h100.yaml} | 0 .../helm/mig-slicing/values-mig-rtx6000.yaml | 114 ++++++++++++++++++ docs/mig-deployment.md | 40 +++--- docs/vlm.md | 2 +- 6 files changed, 160 insertions(+), 22 deletions(-) rename deploy/helm/mig-slicing/{mig-config.yaml => mig-config-h100.yaml} (100%) create mode 100644 deploy/helm/mig-slicing/mig-config-rtx6000.yaml rename deploy/helm/mig-slicing/{values-mig.yaml => values-mig-h100.yaml} (100%) create mode 100644 deploy/helm/mig-slicing/values-mig-rtx6000.yaml diff --git a/deploy/helm/mig-slicing/mig-config.yaml b/deploy/helm/mig-slicing/mig-config-h100.yaml similarity index 100% rename from deploy/helm/mig-slicing/mig-config.yaml rename to deploy/helm/mig-slicing/mig-config-h100.yaml diff --git a/deploy/helm/mig-slicing/mig-config-rtx6000.yaml b/deploy/helm/mig-slicing/mig-config-rtx6000.yaml new file mode 100644 index 000000000..14272b497 --- /dev/null +++ b/deploy/helm/mig-slicing/mig-config-rtx6000.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: custom-mig-config +data: + config.yaml: | + version: v1 + mig-configs: + all-disabled: + - devices: all + mig-enabled: false + + custom-rtx6000-4x1g24-2x1g24-1x2g48-1x4g96: + - devices: [0] + mig-enabled: true + mig-devices: + "1g.24gb": 4 + - devices: [1] + mig-enabled: true + mig-devices: + "1g.24gb": 2 + "2g.48gb": 1 + - devices: [2] + mig-enabled: true + mig-devices: + "4g.96gb": 1 diff --git a/deploy/helm/mig-slicing/values-mig.yaml b/deploy/helm/mig-slicing/values-mig-h100.yaml similarity index 100% rename from deploy/helm/mig-slicing/values-mig.yaml rename to deploy/helm/mig-slicing/values-mig-h100.yaml diff --git a/deploy/helm/mig-slicing/values-mig-rtx6000.yaml b/deploy/helm/mig-slicing/values-mig-rtx6000.yaml new file mode 100644 index 000000000..e7ae285da --- /dev/null +++ b/deploy/helm/mig-slicing/values-mig-rtx6000.yaml @@ -0,0 +1,114 @@ +# MIG-optimized resource configuration for RAG Blueprint +# This file only overrides GPU resource requirements to use MIG slices + +# NV-Ingest configuration +nv-ingest: + # Milvus - uses 1g.24gb MIG slice + milvus: + standalone: + resources: + limits: + nvidia.com/gpu: "0" + nvidia.com/mig-1g.24gb: 1 + requests: + nvidia.com/gpu: "0" + nvidia.com/mig-1g.24gb: 1 + + # NV-Ingest NIM Operator overrides + nimOperator: + # Page Elements - uses 1g.24gb + page_elements: + resources: + limits: + nvidia.com/gpu: "0" + nvidia.com/mig-1g.24gb: 1 + requests: + nvidia.com/gpu: "0" + nvidia.com/mig-1g.24gb: 1 + storage: + pvc: + storageClass: "" + + # Graphic Elements - uses 1g.24gb + graphic_elements: + resources: + limits: + nvidia.com/gpu: "0" + nvidia.com/mig-1g.24gb: 1 + requests: + nvidia.com/gpu: "0" + nvidia.com/mig-1g.24gb: 1 + storage: + pvc: + storageClass: "" + + # Table Structure - uses 1g.24gb + table_structure: + resources: + limits: + nvidia.com/gpu: "0" + nvidia.com/mig-1g.24gb: 1 + requests: + nvidia.com/gpu: "0" + nvidia.com/mig-1g.24gb: 1 + storage: + pvc: + storageClass: "" + + # OCR - uses 2g.48gb (larger slice) + nemoretriever_ocr_v1: + resources: + limits: + nvidia.com/gpu: "0" + nvidia.com/mig-2g.48gb: 1 + requests: + nvidia.com/gpu: "0" + nvidia.com/mig-2g.48gb: 1 + storage: + pvc: + storageClass: "" +# Main NIM Operator overrides for MIG +nimOperator: + # LLM - uses 4g.96gb + nim-llm: + resources: + limits: + nvidia.com/gpu: "0" + nvidia.com/mig-4g.96gb: 1 + requests: + nvidia.com/gpu: "0" + nvidia.com/mig-4g.96gb: 1 + storage: + pvc: + storageClass: "" + model: + engine: tensorrt_llm + precision: "fp8" + qosProfile: "throughput" + tensorParallelism: "1" + gpus: + - product: "rtx6000_blackwell_sv" + # Embedding - uses 1g.24gb + nvidia-nim-llama-32-nv-embedqa-1b-v2: + resources: + limits: + nvidia.com/gpu: "0" + nvidia.com/mig-1g.24gb: 1 + requests: + nvidia.com/gpu: "0" + nvidia.com/mig-1g.24gb: 1 + storage: + pvc: + storageClass: "" + # Reranking - uses 1g.24gb + nvidia-nim-llama-32-nv-rerankqa-1b-v2: + resources: + limits: + nvidia.com/gpu: "0" + nvidia.com/mig-1g.24gb: 1 + requests: + nvidia.com/gpu: "0" + nvidia.com/mig-1g.24gb: 1 + storage: + pvc: + storageClass: "" diff --git a/docs/mig-deployment.md b/docs/mig-deployment.md index bc4793ab1..f7eebdf3d 100644 --- a/docs/mig-deployment.md +++ b/docs/mig-deployment.md @@ -15,10 +15,10 @@ refer to the [MIG Supported Hardware List](https://docs.nvidia.com/datacenter/te Before you deploy, verify that you have the following: -* A Kubernetes cluster with NVIDIA H100 GPUs +* A Kubernetes cluster with NVIDIA H100 or RTX PRO 6000 GPUs :::{note} - This section showcases MIG support for `NVIDIA H100 80GB HBM3` GPU. The MIG profiles used in the `mig-config.yaml` are specific to this GPU. + This section showcases MIG support for `NVIDIA H100 80GB HBM3` GPU. The MIG profiles used in the `mig-config-h100.yaml` are specific to this GPU. Refer to the [MIG User Guide](https://docs.nvidia.com/datacenter/tesla/mig-user-guide/) for MIG profiles of other GPU types. ::: @@ -99,7 +99,7 @@ For monitoring deployment progress, refer to [Deploy on Kubernetes with Helm](./ ## Step 2: Apply the MIG configuration -Edit the MIG configuration file [`mig-config.yaml`](../deploy/helm/mig-slicing/mig-config.yaml) to adjust the slicing pattern as needed. +Edit the MIG configuration file [`mig-config-h100.yaml`](../deploy/helm/mig-slicing/mig-config-h100.yaml) to adjust the slicing pattern as needed. The following example enables a custom configuration with mixed MIG slice sizes on the same GPU. @@ -139,7 +139,7 @@ data: Apply the custom MIG configuration configMap to the node and update the ClusterPolicy, by running the following code. ```bash -kubectl apply -n nvidia-gpu-operator -f mig-slicing/mig-config.yaml +kubectl apply -n nvidia-gpu-operator -f mig-slicing/mig-config-h100.yaml kubectl patch clusterpolicies.nvidia.com/cluster-policy \ --type='json' \ -p='[{"op":"replace", "path":"/spec/migManager/config/name", "value":"custom-mig-config"}]' @@ -151,6 +151,17 @@ Label the node with MIG configuration, by running the following code. kubectl label nodes nvidia.com/mig.config=custom-7x1g10-2x1g20-1x3g40-1x7g80 --overwrite ``` +:::{important} +**For NVIDIA RTX6000 Pro Deployments:** + +Use [`mig-config-rtx6000.yaml`](../deploy/helm/mig-slicing/mig-config-rtx6000.yaml) instead: + +```bash +kubectl apply -n nvidia-gpu-operator -f mig-slicing/mig-config-rtx6000.yaml +kubectl label nodes nvidia.com/mig.config=custom-rtx6000-4x1g24-2x1g24-1x2g48-1x4g96 --overwrite +``` +::: + Verify that the MIG configuration is successfully applied, by running the following code. ```bash @@ -179,34 +190,21 @@ helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprin --password "${NGC_API_KEY}" \ --set imagePullSecret.password=$NGC_API_KEY \ --set ngcApiSecret.password=$NGC_API_KEY \ - -f mig-slicing/values-mig.yaml + -f mig-slicing/values-mig-h100.yaml ``` :::{important} **For NVIDIA RTX6000 Pro Deployments:** -If you are deploying on NVIDIA RTX6000 Pro GPUs (instead of H100 GPUs), you need to configure the NIM LLM model profile. The required configuration is already present but commented out in the [`values.yaml`](../deploy/helm/nvidia-blueprint-rag/values.yaml) file. - -Uncomment and modify the following section under `nimOperator.nim-llm.model` in [`values.yaml`](../deploy/helm/nvidia-blueprint-rag/values.yaml): -```yaml -model: - engine: tensorrt_llm - precision: "fp8" - qosProfile: "throughput" - tensorParallelism: "1" - gpus: - - product: "rtx6000_blackwell_sv" -``` +If you are deploying on NVIDIA RTX6000 Pro GPUs (instead of H100 GPUs), use [`values-mig-rtx6000.yaml`](../deploy/helm/mig-slicing/values-mig-rtx6000.yaml) and [`mig-config-rtx6000.yaml`](../deploy/helm/mig-slicing/mig-config-rtx6000.yaml) which include the RTX6000-specific MIG profiles and NIM LLM model configuration. -Then install using the modified values.yaml along with MIG values: ```sh helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.4.0.tgz \ --username '$oauthtoken' \ --password "${NGC_API_KEY}" \ --set imagePullSecret.password=$NGC_API_KEY \ --set ngcApiSecret.password=$NGC_API_KEY \ - -f values.yaml \ - -f mig-slicing/values-mig.yaml + -f mig-slicing/values-mig-rtx6000.yaml ``` ::: @@ -303,7 +301,7 @@ GPU 3: NVIDIA H100 80GB HBM3 (UUID: ...) * Ensure you have the correct MIG strategy (`mixed`) configured. * Verify that `nvidia.com/mig.config.state` is `success` before deploying. -* Customize `values-mig.yaml` to specify the correct MIG GPU resource requests for each pod. +* Customize `values-mig-h100.yaml` or `values-mig-rtx6000.yaml` to specify the correct MIG GPU resource requests for each pod. diff --git a/docs/vlm.md b/docs/vlm.md index 4a61b2f52..64c6af176 100644 --- a/docs/vlm.md +++ b/docs/vlm.md @@ -124,7 +124,7 @@ Continue with [Deploy with Docker (NVIDIA-Hosted Models)](deploy-docker-nvidia-h ## Enable VLM with Helm :::{note} -**GPU requirements for Helm**: VLM uses the same GPU normally assigned to LLM (GPU 1). With MIG slicing, assign a dedicated MIG slice to the VLM—see [mig-deployment.md](mig-deployment.md) and [values-mig.yaml](../deploy/helm/mig-slicing/values-mig.yaml). To run both VLM and LLM simultaneously, an additional GPU is required. +**GPU requirements for Helm**: VLM uses the same GPU normally assigned to LLM (GPU 1). With MIG slicing, assign a dedicated MIG slice to the VLM—see [mig-deployment.md](mig-deployment.md) and [values-mig-h100.yaml](../deploy/helm/mig-slicing/values-mig-h100.yaml) or [values-mig-rtx6000.yaml](../deploy/helm/mig-slicing/values-mig-rtx6000.yaml). To run both VLM and LLM simultaneously, an additional GPU is required. ::: 1. In [values.yaml](../deploy/helm/nvidia-blueprint-rag/values.yaml), under the `rag-server` `envVars` section, set: From 685c1d511142a5b6f0f3240ef2f7d4d8702d453a Mon Sep 17 00:00:00 2001 From: Shubhadeep Das <149712532+shubhadeepd@users.noreply.github.com> Date: Thu, 26 Feb 2026 20:25:45 +0530 Subject: [PATCH 08/81] Port release-v2.4.0 to release-v2.5.0 and update container versions (#385) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * changes to docs per bug 5767861 (#328) * Updated launchable with v2.4.0 tag (#318) * updated support matrix (#321) * Document the end‑to‑end flow from query to answer and show how to measure time spent in each stage of the RAG pipeline. (#317) * adding oberservablility * Update docs/debugging.md Co-authored-by: nkmcalli * Update docs/observability.md Co-authored-by: nkmcalli * Add query-to-answer-pipeline doc and observability/debugging updates * Trigger CI * getting build to kick in for observability file * Fix typos in query-to-answer-pipeline.md and ensure file in PR for link check * get rid of PULL_REQUEST_SUMMARY --------- Co-authored-by: nkmcalli * fixed files associated with build (#322) * Add multimodal query integration tests to CI pipeline * changes to docs per bug 5767861 * updated files per bug 5880717 (#327) * updated files per bug 5880717 * Update CONTRIBUTING.md * Update README.md * Update python-client.md * Update readme.md * Update readme.md * Update docs/deploy-helm.md Co-authored-by: nkmcalli * Update docs/deploy-helm.md Co-authored-by: nkmcalli --------- Co-authored-by: rkharwar-nv Co-authored-by: nkmcalli Co-authored-by: Pranjal Doshi Co-authored-by: nv-pranjald <150428320+nv-pranjald@users.noreply.github.com> * Fix workflow rule and doc bugs (#331) * Revert back milvus version in conf.md to v2.6.5 * Modify workflow to run on any branch * Fix workflow push rule to run on protected branches * Add files via upload (#326) Found an error in the Q&A section where images in the citation were not being printed. * Doc bug fixes (#339) * updated helm instructions (#333) * updated helm instructions * Update deploy-helm.md * fix broken image link (#334) * Add release note for Audio model deployment on Kubernetes on RTX‑6000 Pro is not supported in this release.heiss/5863956a (#335) * Add release note for Audio model deployment on Kubernetes on RTX‑6000 Pro is not supported in this release. * Add release note for Audio model deployment on Kubernetes on RTX‑6000 Pro is not supported in this release. * Fix broken image link in observability file * Fix CPU seach with GPU index doc * Fix VLLM profile instruction for nemotron-3-nano --------- Co-authored-by: Kurt Heiss * Updated troubleshoot documentation for Elasticsearch connection timeout (#341) Signed-off-by: Swapnil Masurekar * updated path to image files so that html output is rendered correctly (#363) * Updated helm instructions for mig-deployment prerequisites (#364) * Updated helm instructions for mig-deployment * Update mig-deployment.md * Doc enhancement for noteboook (#361) * Doc enhancement for noteboook * Update release notes * Update launchable.ipynb (#365) Updated branch name State name changed from "FAILURE"->"FAILED" * Fix typo in release notes --------- Co-authored-by: rkharwar-nv * fixed links in deploy-helm and mig-deploymnent (#367) * update artifacts to GA version for v2.4.0 release (#359) * updated files according to style guide (#369) * Revert deploy-helm and mig-deployment to pre-11a31a4 versions (#372) * Fix release date in changelog (#373) * Bump up version to 2.5.0 --------- Signed-off-by: Swapnil Masurekar Co-authored-by: Kurt Heiss Co-authored-by: rkharwar-nv Co-authored-by: nkmcalli Co-authored-by: Pranjal Doshi Co-authored-by: nv-pranjald <150428320+nv-pranjald@users.noreply.github.com> Co-authored-by: Swapnil Masurekar --- ci/publish_wheel.sh | 4 +- .../docker-compose-ingestor-server.yaml | 2 +- deploy/compose/docker-compose-rag-server.yaml | 4 +- deploy/helm/nvidia-blueprint-rag/Chart.yaml | 4 +- deploy/helm/nvidia-blueprint-rag/values.yaml | 6 +-- deploy/workbench/compose.yaml | 6 +-- docs/conf.py | 2 +- docs/deploy-helm.md | 8 ++-- docs/mig-deployment.md | 8 ++-- docs/notebooks.md | 48 ------------------- docs/observability.md | 17 ++++--- docs/project.json | 2 +- docs/release-notes.md | 4 +- docs/text_only_ingest.md | 2 +- docs/versions1.json | 4 +- examples/rag_react_agent/uv.lock | 2 +- notebooks/launchable.ipynb | 2 +- notebooks/rag_library_usage.ipynb | 3 +- pyproject.toml | 2 +- .../test_cases/multimodal_query.py | 2 +- uv.lock | 2 +- 21 files changed, 43 insertions(+), 91 deletions(-) diff --git a/ci/publish_wheel.sh b/ci/publish_wheel.sh index f59165fee..711abb462 100755 --- a/ci/publish_wheel.sh +++ b/ci/publish_wheel.sh @@ -25,8 +25,8 @@ if [ -n "$ARTIFACTORY_VERSION" ]; then echo "Using custom Artifactory version: $ARTIFACTORY_VERSION" ARTIFACTORY_VERSION_FINAL=$ARTIFACTORY_VERSION else - echo "Using default Artifactory version: 2.4.0.dev" - ARTIFACTORY_VERSION_FINAL="2.4.0.dev" + echo "Using default Artifactory version: 2.5.0.dev" + ARTIFACTORY_VERSION_FINAL="2.5.0.dev" fi # Build first wheel for GitLab Package Registry diff --git a/deploy/compose/docker-compose-ingestor-server.yaml b/deploy/compose/docker-compose-ingestor-server.yaml index 964a9c02c..fec1fb5aa 100644 --- a/deploy/compose/docker-compose-ingestor-server.yaml +++ b/deploy/compose/docker-compose-ingestor-server.yaml @@ -3,7 +3,7 @@ services: # Main ingestor server which is responsible for ingestion ingestor-server: container_name: ingestor-server - image: nvcr.io/nvstaging/blueprint/ingestor-server:${TAG:-2.4.0} + image: nvcr.io/nvstaging/blueprint/ingestor-server:${TAG:-2.5.0} build: # Set context to repo's root directory context: ../../ diff --git a/deploy/compose/docker-compose-rag-server.yaml b/deploy/compose/docker-compose-rag-server.yaml index b3e20808f..b0c7d25bd 100644 --- a/deploy/compose/docker-compose-rag-server.yaml +++ b/deploy/compose/docker-compose-rag-server.yaml @@ -3,7 +3,7 @@ services: # Main orchestrator server which stiches together all calls to different services to fulfill the user request rag-server: container_name: rag-server - image: nvcr.io/nvstaging/blueprint/rag-server:${TAG:-2.4.0} + image: nvcr.io/nvstaging/blueprint/rag-server:${TAG:-2.5.0} build: # Set context to repo's root directory context: ../../ @@ -211,7 +211,7 @@ services: # Sample UI container which interacts with APIs exposed by rag-server container rag-frontend: container_name: rag-frontend - image: nvcr.io/nvstaging/blueprint/rag-frontend:${TAG:-2.4.0} + image: nvcr.io/nvstaging/blueprint/rag-frontend:${TAG:-2.5.0} build: # Set context to repo's root directory context: ../../frontend diff --git a/deploy/helm/nvidia-blueprint-rag/Chart.yaml b/deploy/helm/nvidia-blueprint-rag/Chart.yaml index a8a459279..639869e53 100644 --- a/deploy/helm/nvidia-blueprint-rag/Chart.yaml +++ b/deploy/helm/nvidia-blueprint-rag/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: v2.4.0 +appVersion: v2.5.0 dependencies: - condition: nv-ingest.enabled name: nv-ingest @@ -24,4 +24,4 @@ dependencies: description: An end to end Helm chart for the NVIDIA RAG Blueprint name: nvidia-blueprint-rag type: application -version: v2.4.0 +version: v2.5.0 diff --git a/deploy/helm/nvidia-blueprint-rag/values.yaml b/deploy/helm/nvidia-blueprint-rag/values.yaml index 47ef09b68..a02fef3e4 100644 --- a/deploy/helm/nvidia-blueprint-rag/values.yaml +++ b/deploy/helm/nvidia-blueprint-rag/values.yaml @@ -57,7 +57,7 @@ apiKeysSecret: # -- RAG server container image image: repository: nvcr.io/nvstaging/blueprint/rag-server - tag: "2.4.0" + tag: "2.5.0" pullPolicy: Always # -- RAG server service configuration @@ -290,7 +290,7 @@ ingestor-server: image: repository: nvcr.io/nvstaging/blueprint/ingestor-server - tag: "2.4.0" + tag: "2.5.0" pullPolicy: Always # -- Service config for ingestor-server @@ -454,7 +454,7 @@ frontend: image: repository: nvcr.io/nvstaging/blueprint/rag-frontend pullPolicy: IfNotPresent - tag: "2.4.0" + tag: "2.5.0" imagePullSecret: name: "ngc-secret" diff --git a/deploy/workbench/compose.yaml b/deploy/workbench/compose.yaml index 04cfdd2e2..1e75d07b7 100644 --- a/deploy/workbench/compose.yaml +++ b/deploy/workbench/compose.yaml @@ -200,7 +200,7 @@ services: # Main ingestor server which is responsible for ingestion ingestor-server: container_name: ingestor-server - image: nvcr.io/nvstaging/blueprint/ingestor-server:${TAG:-2.4.0} + image: nvcr.io/nvstaging/blueprint/ingestor-server:${TAG:-2.5.0} build: # Set context to repo's root directory context: ../../ @@ -432,7 +432,7 @@ services: # Main orchestrator server which stiches together all calls to different services to fulfill the user request rag-server: container_name: rag-server - image: nvcr.io/nvstaging/blueprint/rag-server:${TAG:-2.4.0} + image: nvcr.io/nvstaging/blueprint/rag-server:${TAG:-2.5.0} build: # Set context to repo's root directory context: ../../ @@ -569,7 +569,7 @@ services: # Sample UI container which interacts with APIs exposed by rag-server container rag-frontend: container_name: rag-frontend - image: nvcr.io/nvstaging/blueprint/rag-frontend:${TAG:-2.4.0} + image: nvcr.io/nvstaging/blueprint/rag-frontend:${TAG:-2.5.0} build: # Set context to repo's root directory context: ../../frontend diff --git a/docs/conf.py b/docs/conf.py index f0ffa9e07..0bc42fd7b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -26,7 +26,7 @@ project = " NVIDIA-RAG-blueprint" copyright = "2025, NVIDIA Corporation" author = "NVIDIA Corporation" -release = "2.4.0" +release = "2.5.0" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/docs/deploy-helm.md b/docs/deploy-helm.md index bf8e792c5..f0686c6ca 100644 --- a/docs/deploy-helm.md +++ b/docs/deploy-helm.md @@ -37,7 +37,7 @@ Plan for additional space if you are enabling persistence for multiple services. 4. Verify that you have Kubernetes v1.34.2 installed and running on Ubuntu 22.04/24.04. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/setup/) and [NVIDIA Cloud Native Stack](https://github.com/NVIDIA/cloud-native-stack). -5. Verify that you have installed Helm 3. To install Helm 3 (and avoid Helm 4), follow the official Helm v3 installation instructions for your platform, for example by using the `get-helm-3` script described in the [Helm documentation](https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3). +5. Verify that you have installed Helm 3. To install Helm 3 (and avoid Helm 4), follow the official Helm v3 installation instructions for your platform, for example by using the `get-helm-3` script described in the [Helm documentation](https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3). 6. Verify that you have a default storage class available in the cluster for PVC provisioning. One option is the local path provisioner by Rancher. Refer to the [installation](https://github.com/rancher/local-path-provisioner?tab=readme-ov-file#installation) section of the README in the GitHub repository. @@ -87,7 +87,7 @@ To deploy End-to-End RAG Server and Ingestor Server, use the following procedure 2. Install the Helm chart by running the following command. ```sh - helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.4.0.tgz \ + helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.5.0.tgz \ --username '$oauthtoken' \ --password "${NGC_API_KEY}" \ --set imagePullSecret.password=$NGC_API_KEY \ @@ -112,7 +112,7 @@ To deploy End-to-End RAG Server and Ingestor Server, use the following procedure Then install using the modified values.yaml: ```sh - helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.4.0.tgz \ + helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.5.0.tgz \ --username '$oauthtoken' \ --password "${NGC_API_KEY}" \ --set imagePullSecret.password=$NGC_API_KEY \ @@ -250,7 +250,7 @@ Port-forwarding is provided as a quick method to try out the UI. However, large To change an existing deployment, after you modify the [`values.yaml`](../deploy/helm/nvidia-blueprint-rag/values.yaml) file, run the following code. ```sh -helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.4.0.tgz \ +helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.5.0.tgz \ --username '$oauthtoken' \ --password "${NGC_API_KEY}" \ --set imagePullSecret.password=$NGC_API_KEY \ diff --git a/docs/mig-deployment.md b/docs/mig-deployment.md index f7eebdf3d..878681713 100644 --- a/docs/mig-deployment.md +++ b/docs/mig-deployment.md @@ -35,9 +35,9 @@ For monitoring deployment progress, refer to [Deploy on Kubernetes with Helm](./ 3. Verify that you have the NGC CLI available on your client computer. You can download the CLI from . -4. Verify that you have Kubernetes v1.34.2 installed and running on Ubuntu 22.04/24.04. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/setup/) and [NVIDIA Cloud Native Stack 17.0](https://github.com/NVIDIA/cloud-native-stack/tree/17.0). +4. Verify that you have Kubernetes v1.34.2 installed and running on Ubuntu 22.04/24.04. For more information, see [Kubernetes documentation](https://kubernetes.io/docs/setup/) and [NVIDIA Cloud Native Stack 17.0](https://github.com/NVIDIA/cloud-native-stack/tree/25.12.0). -5. Verify that you have installed Helm 3 or later (Helm v3.20.0 recommended). For installation instructions, see [Helm Installation](https://helm.sh/docs/intro/install). +5. Verify that you have installed Helm 3. To install Helm 3 (and avoid Helm 4), follow the official Helm v3 installation instructions for your platform, for example by using the `get-helm-3` script described in the [Helm documentation](https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3). 6. Verify that you have a default storage class available in the cluster for PVC provisioning. One option is the local path provisioner by Rancher. Refer to the [installation](https://github.com/rancher/local-path-provisioner?tab=readme-ov-file#installation) section of the README in the GitHub repository. @@ -185,7 +185,7 @@ You should see output similar to the following. Run the following code to install the RAG Blueprint Helm Chart. ```bash -helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.4.0.tgz \ +helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.5.0.tgz \ --username '$oauthtoken' \ --password "${NGC_API_KEY}" \ --set imagePullSecret.password=$NGC_API_KEY \ @@ -199,7 +199,7 @@ helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprin If you are deploying on NVIDIA RTX6000 Pro GPUs (instead of H100 GPUs), use [`values-mig-rtx6000.yaml`](../deploy/helm/mig-slicing/values-mig-rtx6000.yaml) and [`mig-config-rtx6000.yaml`](../deploy/helm/mig-slicing/mig-config-rtx6000.yaml) which include the RTX6000-specific MIG profiles and NIM LLM model configuration. ```sh -helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.4.0.tgz \ +helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.5.0.tgz \ --username '$oauthtoken' \ --password "${NGC_API_KEY}" \ --set imagePullSecret.password=$NGC_API_KEY \ diff --git a/docs/notebooks.md b/docs/notebooks.md index a88952f79..7a7f50eee 100644 --- a/docs/notebooks.md +++ b/docs/notebooks.md @@ -125,54 +125,6 @@ Use the following notebook for cloud deployment scenarios. - [launchable.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/launchable.ipynb) – A deployment-ready notebook intended to run in a [Brev environment](https://console.brev.dev/environment/new). To learn more about Brev, refer to [Brev](https://docs.nvidia.com/brev/latest/about-brev.html). Follow the instructions for running Jupyter notebooks in a cloud-based environment based on the hardware requirements specified in the launchable. - -## Set Up the Notebook Environment - -To run a notebook, use the following procedure with [uv](https://docs.astral.sh/uv/) - a fast Python package manager. - -> **Note**: Python version **3.11 or higher** is required. - -1. Install uv (if not already installed): - - ```bash - curl -LsSf https://astral.sh/uv/0.8.12/install.sh | sh - ``` - -2. Create and activate a virtual environment: - - ```bash - uv venv --python=python3.12 - source .venv/bin/activate - ``` - -3. Install JupyterLab: - - ```bash - uv pip install jupyterlab - ``` - -4. Start JupyterLab: - - ```bash - jupyter lab --allow-root --ip=0.0.0.0 --NotebookApp.token='' --port=8889 --no-browser - ``` - -### Set-up Notes -- Ensure that API keys and credentials are correctly set up before you run a notebook. -- Modify endpoints or request parameters as necessary to match your specific use case. -- For the custom VDB operator notebook, ensure that Docker is available for running OpenSearch services. - - - -## Run a Notebook - -After you set up your notebook environment, to run a notebook, use the following procedure. - -1. Access JupyterLab by opening a browser and navigating to `http://:8889`. -2. Navigate to the notebook and run the cells sequentially. - - - ## Related Topics - [Get Started](deploy-docker-self-hosted.md) diff --git a/docs/observability.md b/docs/observability.md index 0c4bb2665..587c6a70e 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -45,13 +45,13 @@ Use the following procedure to enable observability with Docker. After tracing is enabled and the system is running, you can **view the traces** in **Zipkin** by opening: -

- -

+```{image} assets/zipkin_ui.png +:width: 750px +:align: center +``` Open the Zipkin UI at: **http://localhost:9411** - ## View Metrics in Grafana As part of the tracing, the RAG service also exports metrics like API request counts, LLM prompt and completion token count and words per chunk. @@ -104,11 +104,10 @@ After tracing is enabled and running, you can view inputs and outputs of differe 3. Similarly, you can view inputs and outputs for sub stages within the workflows by clicking on a substage and finding the `traceloop.entity.input` and `traceloop.entity.ouput` rows. -

- -

- - +```{image} assets/zipkin_ui_labelled.png +:width: 750px +:align: center +``` ## Enable Observability with Helm diff --git a/docs/project.json b/docs/project.json index 66344b5d0..9b67aad99 100644 --- a/docs/project.json +++ b/docs/project.json @@ -1,4 +1,4 @@ { "name": "NVIDIA-RAG-blueprint", - "version": "2.4.0" + "version": "2.5.0" } \ No newline at end of file diff --git a/docs/release-notes.md b/docs/release-notes.md index 96c48a121..c0a359206 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -8,7 +8,7 @@ This documentation contains the release notes for [NVIDIA RAG Blueprint](readme. -## Release 2.4.0 (26-02-TBD) +## Release 2.4.0 (2026-02-20) This release adds new features to the RAG pipeline for supporting agent workflows and enhances generations with VLMs augmenting multimodal input. @@ -19,7 +19,7 @@ This release contains the following key changes: - Updated NIMs and code to support [NVIDIA Ingest 26.01 release](https://docs.nvidia.com/nemo/retriever/latest/extraction/releasenotes-nv-ingest/). - Added support for non-NIM models including OpenAI, models hosted on AWS and Azure, OSS models, and others. Supported through service-specific API keys. For details, refer to [Get an API Key](api-key.md). - The RAG Blueprint now uses [nemoretriever-ocr-v1](https://build.nvidia.com/nvidia/nemoretriever-ocr-v1/modelcard) as the default OCR model. For details, refer to [NeMo Retriever OCR Configuration Guide](nemoretriever-ocr.md). -- The Vision-Language Model (VLM) inference feature now uses the model [nemotron-nano-12b-v2-vl](https://build.nvidia.com/nvidia/nemotron-nano-12b-v2-vl/modelcard). For details, refer to [VLM for Generation](vlm.md). +- Improved VLM based generation support. The Vision-Language Model (VLM) inference feature now uses the model [nemotron-nano-12b-v2-vl](https://build.nvidia.com/nvidia/nemotron-nano-12b-v2-vl/modelcard). For details, refer to [VLM for Generation](vlm.md). - User interface improvements including catalog display, image and text query, and others. For details, refer to [User Interface](user-interface.md). - Added ingestion metrics endpoint support with OpenTelemetry (OTEL) for monitoring document uploads, elements ingested, and pages processed. For details, refer to [Observability](observability.md). - Support image and text as input query. For details, refer to [Multimodal Query Support](multimodal-query.md). diff --git a/docs/text_only_ingest.md b/docs/text_only_ingest.md index 784c08978..625416090 100644 --- a/docs/text_only_ingest.md +++ b/docs/text_only_ingest.md @@ -113,7 +113,7 @@ Additionally, ensure that **table extraction**, **chart extraction**, and **imag 2. Then use the modified [`values.yaml`](../deploy/helm/nvidia-blueprint-rag/values.yaml) file in your Helm upgrade command: ```bash -helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.4.0.tgz \ +helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.5.0.tgz \ --username '$oauthtoken' \ --password "${NGC_API_KEY}" \ --values deploy/helm/nvidia-blueprint-rag/values.yaml \ diff --git a/docs/versions1.json b/docs/versions1.json index d0731c374..67cecce82 100644 --- a/docs/versions1.json +++ b/docs/versions1.json @@ -1,8 +1,8 @@ [ { "preferred": true, - "version": "2.4.0", - "url": "../2.4.0/" + "version": "2.5.0", + "url": "../2.5.0/" }, { "version": "2.3.0", diff --git a/examples/rag_react_agent/uv.lock b/examples/rag_react_agent/uv.lock index 0554ef787..244c06a23 100644 --- a/examples/rag_react_agent/uv.lock +++ b/examples/rag_react_agent/uv.lock @@ -1950,7 +1950,7 @@ wheels = [ [[package]] name = "nvidia-rag" -version = "2.4.0.dev0" +version = "2.5.0.dev0" source = { editable = "../../" } dependencies = [ { name = "anyio" }, diff --git a/notebooks/launchable.ipynb b/notebooks/launchable.ipynb index 8e5779af2..ee56c3447 100644 --- a/notebooks/launchable.ipynb +++ b/notebooks/launchable.ipynb @@ -914,7 +914,7 @@ "import subprocess\n", "\n", "REPO_URL = \"https://github.com/NVIDIA-AI-Blueprints/rag.git\"\n", - "BRANCH = \"release-v2.4.0\"\n", + "BRANCH = \"release-v2.5.0\"\n", "#BRANCH = \"develop\"\n", "# Check if we're already in the rag repo (look for deploy/compose)\n", "if os.path.exists(\"deploy/compose\"):\n", diff --git a/notebooks/rag_library_usage.ipynb b/notebooks/rag_library_usage.ipynb index 01f07fb4a..83fb0f3b0 100644 --- a/notebooks/rag_library_usage.ipynb +++ b/notebooks/rag_library_usage.ipynb @@ -75,7 +75,8 @@ "outputs": [], "source": [ "# Option A: Install from PyPI (recommended)\n", - "# Uncomment the line below to install from PyPI\n", + "# Uncomment the line below to install from PyPI.\n", + "# Note: This will require a restart of the kernel after installation if you are using this notebook in a JupyterLab session.\n", "# !uv pip install nvidia-rag[all]\n", "\n", "# Option B: Install from source in development mode (for contributors)\n", diff --git a/pyproject.toml b/pyproject.toml index 09b647403..80567fb75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nvidia_rag" -version = "2.4.0.dev" +version = "2.5.0.dev" description = "This blueprint serves as a reference solution for a foundational Retrieval Augmented Generation (RAG) pipeline." readme = "README.md" license = "Apache-2.0" diff --git a/tests/integration/test_cases/multimodal_query.py b/tests/integration/test_cases/multimodal_query.py index b888b8c7c..b38925211 100644 --- a/tests/integration/test_cases/multimodal_query.py +++ b/tests/integration/test_cases/multimodal_query.py @@ -95,7 +95,7 @@ 2. Deploy or upgrade the chart: - helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.4.0-rc1.tgz \\ + helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvidia/blueprint/charts/nvidia-blueprint-rag-v2.5.0-rc1.tgz \\ --username '$oauthtoken' \\ --password "${NGC_API_KEY}" \\ --set imagePullSecret.password=$NGC_API_KEY \\ diff --git a/uv.lock b/uv.lock index dbbf310f2..7c267bf95 100644 --- a/uv.lock +++ b/uv.lock @@ -1796,7 +1796,7 @@ wheels = [ [[package]] name = "nvidia-rag" -version = "2.4.0.dev0" +version = "2.5.0.dev0" source = { virtual = "." } dependencies = [ { name = "anyio" }, From 800d18b94a08d80511594148e887b10e00d89e1c Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Thu, 26 Feb 2026 11:21:43 -0600 Subject: [PATCH 09/81] fixed health and exluded dbtools tables --- src/nvidia_rag/utils/vdb/oracle/oracle_queries.py | 1 + src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py b/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py index 490228da7..8471a7fde 100644 --- a/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py +++ b/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py @@ -341,5 +341,6 @@ def get_all_collections_query() -> str: WHERE table_name NOT IN ('METADATA_SCHEMA', 'DOCUMENT_INFO') AND table_name NOT LIKE 'SYS%' AND table_name NOT LIKE 'DR$%' + AND table_name NOT LIKE 'DBTOOLS%' ORDER BY table_name """ \ No newline at end of file diff --git a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py index 61e19a9e7..2ace149bb 100644 --- a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py +++ b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py @@ -322,7 +322,7 @@ async def check_health(self) -> dict[str, Any]: """Check Oracle database health.""" status = { "service": "Oracle 26ai", - "url": self._oracle_dsn, + "url": self._oracle_cs, "status": ServiceStatus.UNKNOWN.value, "error": None, } @@ -824,7 +824,10 @@ def get_langchain_vectorstore(self, collection_name: str) -> OracleVS: conn = oracledb.connect( user=self._oracle_user, password=self._oracle_password, - dsn=self._oracle_dsn, + dsn=self._oracle_cs, + config_dir=self._tnsnames_loc, + wallet_location=self._ewallet_pem_loc, + wallet_password=self._ewallet_password ) return OracleVS( From 1944a7e340080097e0b0550a773e96246d205ad4 Mon Sep 17 00:00:00 2001 From: Minh Nguyen Date: Fri, 27 Feb 2026 13:16:58 +0700 Subject: [PATCH 10/81] feat: Event-Driven Document/Video Ingestion Pipeline (#351) * feat: add rag_event_ingest example - event-driven document/video ingestion pipeline - Kafka consumer that monitors MinIO object storage for new uploads - Routes documents to RAG Ingestor, videos to VSS for analysis - Docker Compose deployment for Kafka, MinIO, and consumer - Jupyter notebook for end-to-end deployment and testing - Sample test data (PDF document, MP4 video) tracked via Git LFS Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * fix: polish rag_event_ingest notebook - fix sections, descriptions, TOC Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * refactor: consolidate Setup into single cell - clone, deps, API keys Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * refactor: inline check_rag/vss/aidp_status into their usage cells Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * docs: add markdown description before every code cell Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * fix: notebook is standalone entry point, clones RAG repo to ~/rag Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * fix: set COLLECTION_NAME, load .env, simplify query_rag, add expected logs Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * fix: left-align markdown tables in notebook Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * fix: use HTML tables to force left alignment Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * fix: replace API Keys markdown table with HTML for left alignment Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * feat: add storage verification, RAG frontend hints, and configurable consumer prompts - Add verify_file_in_storage() helper to confirm files landed in MinIO - Merge storage verification into document/video ingestion checks - Add RAG Frontend UI link (port 8090) to query sections - Make Kafka consumer VSS prompts configurable via env vars in docker-compose - Install git/git-lfs in notebook setup cell - Index cells in Deploy Continuous Ingestion section Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * fix: skip RAG clone if directory already exists Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * fix: url Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * feat: add continuous ingestion notebook for video and document pipeline Add rag_event_ingest.ipynb notebook that provides an end-to-end walkthrough for: - Deploying NVIDIA RAG stack (NIMs, Milvus, Ingestor, RAG Server) - Deploying NVIDIA VSS stack (VLM, LLM, Embedding, Reranker NIMs) - Deploying continuous ingestion pipeline (Kafka, MinIO, Kafka Consumer) - Configurable video analysis prompts for the Kafka consumer - Uploading documents and videos to MinIO with storage verification - Verifying ingestion via consumer logs - Querying ingested content via RAG API or Frontend UI Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * fix: gpu Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * fix: add ensurepip, fix VSS tag to v2.4.1, use GPUs 2-3 for VSS, update hw req to 4 GPUs Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * fix: tag Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * feat: resolve comment * fix: patch VSS config to use host-mapped ports for shared RAG embedding/reranker The via-server runs on the local_deployment_single_gpu_default network, not nvidia-rag, so it cannot resolve nemoretriever-embedding-ms or nemoretriever-ranking-ms. Route through host.docker.internal with the correct host-mapped ports instead (9080 for embedding, 1976 for reranker). Co-authored-by: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor --------- Signed-off-by: Minh Nguyen Co-authored-by: anngu Co-authored-by: Cursor --- .gitattributes | 2 + examples/README.md | 12 + ...at We Learned from Seattle's 29-13 win.pdf | Bin 0 -> 132 bytes ...triots - Super Bowl LX Game Highlights.mp4 | 3 + .../deploy/docker-compose.yaml | 177 +++ .../kafka_consumer/Dockerfile | 12 + .../kafka_consumer/config/__init__.py | 201 +++ .../kafka_consumer/config/constants.py | 410 ++++++ .../kafka_consumer/config/settings.py | 199 +++ .../kafka_consumer/consumer.py | 204 +++ .../kafka_consumer/handlers/__init__.py | 6 + .../kafka_consumer/handlers/base.py | 43 + .../kafka_consumer/handlers/document.py | 89 ++ .../kafka_consumer/handlers/video.py | 118 ++ .../rag_event_ingest/kafka_consumer/main.py | 49 + .../kafka_consumer/models/__init__.py | 4 + .../kafka_consumer/models/events.py | 138 ++ .../kafka_consumer/requirements.txt | 4 + .../rag_event_ingest/kafka_consumer/router.py | 104 ++ .../kafka_consumer/services/__init__.py | 8 + .../services/document_indexer.py | 298 ++++ .../kafka_consumer/services/storage.py | 195 +++ .../kafka_consumer/services/video_analyzer.py | 326 +++++ notebooks/rag_event_ingest.ipynb | 1224 +++++++++++++++++ 24 files changed, 3826 insertions(+) create mode 100644 examples/rag_event_ingest/data/documents/Seahawks-Patriots in Super Bowl LX_ What We Learned from Seattle's 29-13 win.pdf create mode 100644 examples/rag_event_ingest/data/videos/Seattle Seahawks vs New England Patriots - Super Bowl LX Game Highlights.mp4 create mode 100644 examples/rag_event_ingest/deploy/docker-compose.yaml create mode 100644 examples/rag_event_ingest/kafka_consumer/Dockerfile create mode 100644 examples/rag_event_ingest/kafka_consumer/config/__init__.py create mode 100644 examples/rag_event_ingest/kafka_consumer/config/constants.py create mode 100644 examples/rag_event_ingest/kafka_consumer/config/settings.py create mode 100644 examples/rag_event_ingest/kafka_consumer/consumer.py create mode 100644 examples/rag_event_ingest/kafka_consumer/handlers/__init__.py create mode 100644 examples/rag_event_ingest/kafka_consumer/handlers/base.py create mode 100644 examples/rag_event_ingest/kafka_consumer/handlers/document.py create mode 100644 examples/rag_event_ingest/kafka_consumer/handlers/video.py create mode 100644 examples/rag_event_ingest/kafka_consumer/main.py create mode 100644 examples/rag_event_ingest/kafka_consumer/models/__init__.py create mode 100644 examples/rag_event_ingest/kafka_consumer/models/events.py create mode 100644 examples/rag_event_ingest/kafka_consumer/requirements.txt create mode 100644 examples/rag_event_ingest/kafka_consumer/router.py create mode 100644 examples/rag_event_ingest/kafka_consumer/services/__init__.py create mode 100644 examples/rag_event_ingest/kafka_consumer/services/document_indexer.py create mode 100644 examples/rag_event_ingest/kafka_consumer/services/storage.py create mode 100644 examples/rag_event_ingest/kafka_consumer/services/video_analyzer.py create mode 100644 notebooks/rag_event_ingest.ipynb diff --git a/.gitattributes b/.gitattributes index c8d189184..0a7e469ce 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,4 @@ data/dataset.zip filter=lfs diff=lfs merge=lfs -text data/ filter=lfs diff=lfs merge=lfs -text +examples/rag_event_ingest/data/**/*.mp4 filter=lfs diff=lfs merge=lfs -text +examples/rag_event_ingest/data/**/*.pdf filter=lfs diff=lfs merge=lfs -text diff --git a/examples/README.md b/examples/README.md index 0d56c4781..2e8d54c1c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,6 +8,7 @@ This directory contains example integrations and extensions for NVIDIA RAG. |---------|-------------|---------------| | [rag_react_agent](./rag_react_agent/) | Integration with [NeMo Agent Toolkit (NAT)](https://github.com/NVIDIA/NeMo-Agent-Toolkit) providing RAG query and search capabilities for agent workflows | [README](./rag_react_agent/README.md) | | [nvidia_rag_mcp](./nvidia_rag_mcp/) | MCP (Model Context Protocol) server and client for exposing NVIDIA RAG capabilities to MCP-compatible applications | [Documentation](../docs/mcp.md) | +| [rag_event_ingest](./rag_event_ingest/) | Automated document & video ingestion from object storage (MinIO) via Kafka, with VSS video analysis | [Notebook](../notebooks/rag_event_ingest.ipynb) | ## rag_react_agent @@ -27,3 +28,14 @@ This example provides an MCP server and client that exposes NVIDIA RAG and Inges - Manage collections and documents in the vector database See the [MCP documentation](../docs/mcp.md) for detailed setup and usage instructions. + +## rag_event_ingest + +This example deploys an event-driven ingestion pipeline that monitors MinIO object storage for new file uploads via Kafka events. Documents are indexed through the RAG Ingestor, while videos are analyzed by NVIDIA VSS (Video Search & Summarization). All content becomes queryable through the RAG Agent. + +Components: +- **kafka_consumer/** - Event-driven consumer that routes files to RAG or VSS based on file type +- **deploy/** - Docker Compose for Kafka, MinIO, and the consumer +- **data/** - Sample documents and videos for testing + +See the [notebook](../notebooks/rag_event_ingest.ipynb) for step-by-step deployment and testing. diff --git a/examples/rag_event_ingest/data/documents/Seahawks-Patriots in Super Bowl LX_ What We Learned from Seattle's 29-13 win.pdf b/examples/rag_event_ingest/data/documents/Seahawks-Patriots in Super Bowl LX_ What We Learned from Seattle's 29-13 win.pdf new file mode 100644 index 0000000000000000000000000000000000000000..3d750564d9c52cf8f6ed6fdfe29574c0d1e62c12 GIT binary patch literal 132 zcmWN`xe>!45CFiODrmss_(9$99EKU&XnaV4tGA*J_QEfX`Qd%7gLWaFV?ADtw%dN@ zt<1Ne1BvQeg3Z=f}3 O(J&f6 + /bin/sh -c " + echo 'Waiting for MinIO...'; + sleep 5; + + echo 'Setting up MinIO...'; + mc alias set minio http://minio-source-1:9000 minioadmin minioadmin; + mc mb --ignore-existing minio/aidp-bucket; + mc event add minio/aidp-bucket arn:minio:sqs::AIDP:kafka --event put,delete || true; + + echo 'MinIO setup complete!'; + echo 'Bucket: aidp-bucket on minio-source-1'; + + echo 'Keeping container alive for mc commands...'; + tail -f /dev/null + " + networks: + - nvidia-rag + + # ============================================================================= + # KAFKA CONSUMER (Event-driven Ingestion) + # ============================================================================= + kafka-consumer: + build: + context: ../kafka_consumer + dockerfile: Dockerfile + image: kafka-consumer:local + container_name: kafka-consumer + depends_on: + kafka: + condition: service_healthy + minio-source-1: + condition: service_healthy + environment: + # Kafka + - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 + - KAFKA_TOPIC=${KAFKA_TOPIC:-aidp-topic} + - CONSUMER_GROUP=${CONSUMER_GROUP:-nvingest-consumer-group} + # MinIO + - MINIO_ENDPOINT=minio-source-1:9000 + - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY:-minioadmin} + - MINIO_SECRET_KEY=${MINIO_SECRET_KEY:-minioadmin} + - MINIO_SECURE=false + - VSS_SERVER_URL=${VSS_SERVER_URL:-http://host.docker.internal:8110} + - VSS_TIMEOUT=${VSS_TIMEOUT:-1800} + - VSS_CHUNK_DURATION=${VSS_CHUNK_DURATION:-30} + - VSS_CHUNK_OVERLAP=${VSS_CHUNK_OVERLAP:-10} + - VSS_NUM_FRAMES_PER_CHUNK=${VSS_NUM_FRAMES_PER_CHUNK:-16} + - VSS_MAX_TOKENS=${VSS_MAX_TOKENS:-8192} + # RAG Ingestor + - INGESTOR_SERVER_URL=${INGESTOR_SERVER_URL:-http://ingestor-server:8082} + - COLLECTION_NAME=${COLLECTION_NAME:-aidp_bucket} + # Video Analysis Prompts (customizable via environment variables) + - "VSS_PROMPT=${VSS_PROMPT:-Analyze this sports video. TIMESTAMP FORMAT: Convert seconds to MM:SS (40s=00:40, 80s=01:20, 150s=02:30, 200s=03:20). Describe: key plays, scoring, turnovers, big gains, defensive stops, celebrations.}" + - "VSS_SYSTEM_PROMPT=${VSS_SYSTEM_PROMPT:-You are a sports broadcaster. CRITICAL: Convert all timestamps from seconds to MM:SS format. Examples: 40 seconds = 00:40, 90 seconds = 01:30, 200 seconds = 03:20, 350 seconds = 05:50. Focus on action, field position, execution, and game momentum.}" + - "VSS_CAPTION_SUMMARIZATION_PROMPT=${VSS_CAPTION_SUMMARIZATION_PROMPT:-Format: [MM:SS] Play description. CONVERT seconds to MM:SS (40s=00:40, 80s=01:20, 150s=02:30). Describe: what happened, how it was executed, the result. Be vivid like a TV broadcast.}" + - "VSS_SUMMARY_AGGREGATION_PROMPT=${VSS_SUMMARY_AGGREGATION_PROMPT:-Create game summary with MM:SS timestamps. CONVERT all times: 40s=00:40, 80s=01:20, 200s=03:20, 350s=05:50. Highlight scoring plays, turnovers, momentum shifts, spectacular plays. Write like a highlight reel narrator - vivid and exciting.}" + # Logging + - LOG_LEVEL=${LOG_LEVEL:-INFO} + restart: unless-stopped + networks: + - nvidia-rag + extra_hosts: + - "host.docker.internal:host-gateway" + +# ============================================================================= +# VOLUMES +# ============================================================================= +volumes: + kafka-data: + driver: local + minio-data: + driver: local + +# ============================================================================= +# NETWORKS +# ============================================================================= +networks: + nvidia-rag: + external: true + name: nvidia-rag diff --git a/examples/rag_event_ingest/kafka_consumer/Dockerfile b/examples/rag_event_ingest/kafka_consumer/Dockerfile new file mode 100644 index 000000000..d1ff4fc62 --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Install dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY . /app/ + +CMD ["python", "-u", "main.py"] diff --git a/examples/rag_event_ingest/kafka_consumer/config/__init__.py b/examples/rag_event_ingest/kafka_consumer/config/__init__.py new file mode 100644 index 000000000..3e38cf59c --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/config/__init__.py @@ -0,0 +1,201 @@ +# config/__init__.py +"""Configuration package for Kafka MinIO Consumer. + +Usage: + import config.settings as cfg + print(cfg.VSS_SERVER_URL) + + from config.constants import VIDEO_EXTENSIONS, DEST_VSS +""" + +# Settings (env vars) +from .settings import ( + # Kafka + KAFKA_BOOTSTRAP_SERVERS, + KAFKA_CONSUMER_GROUP, + KAFKA_TOPIC, + KAFKA_AUTO_OFFSET_RESET, + KAFKA_MAX_POLL_RECORDS, + KAFKA_MAX_POLL_INTERVAL_MS, + KAFKA_SESSION_TIMEOUT_MS, + KAFKA_HEARTBEAT_INTERVAL_MS, + # Services + INGESTOR_SERVER_URL, + VSS_SERVER_URL, + INGESTOR_TIMEOUT, + VSS_TIMEOUT, + VSS_UPLOAD_TIMEOUT, + VIDEO_INDEX_TIMEOUT, + # MinIO + MINIO_ENDPOINT, + MINIO_ACCESS_KEY, + MINIO_SECRET_KEY, + MINIO_SECURE, + MINIO_DEFAULT_COLLECTION, + MINIO_SOURCES, + # Features + ENABLE_MULTIMODAL_RAG, + ENABLE_IMAGE_PROCESSING, + ENABLE_AUDIO_PROCESSING, + VSS_EMBEDDING_ENABLED, + # Collection + EMBEDDING_DIMENSION, + CHUNK_SIZE, + CHUNK_OVERLAP, + # Logging + LOG_LEVEL, + LOG_FORMAT, + # History + HISTORY_FILE, + # VSS Prompts (configurable via env) + VSS_DEFAULT_PROMPT, + VSS_SYSTEM_PROMPT, + VSS_CAPTION_SUMMARIZATION_PROMPT, + VSS_SUMMARY_AGGREGATION_PROMPT, + # VSS Settings + VSS_CHUNK_DURATION, + VSS_CHUNK_OVERLAP, + VSS_NUM_FRAMES_PER_CHUNK, + VSS_MAX_TOKENS, + VSS_MODEL, + VSS_STREAM_ENABLED, + # API Endpoints (configurable via env) + API_INGESTOR_DOCUMENTS, + API_INGESTOR_COLLECTIONS, + API_INGESTOR_COLLECTION, + API_INGESTOR_STATUS, + API_VSS_FILES, + API_VSS_SUMMARIZE, +) + +# Constants +from .constants import ( + # File extensions + VIDEO_EXTENSIONS, + DOCUMENT_EXTENSIONS, + IMAGE_EXTENSIONS, + AUDIO_EXTENSIONS, + SKIP_EXTENSIONS, + # Content types + CONTENT_TYPE_MAP, + DEFAULT_CONTENT_TYPE, + # Routing + DEST_VSS, + DEST_RAG, + DEST_SKIP, + DEST_UNKNOWN, + # S3 Event fields + EVENT_NAME, + EVENT_RECORDS, + EVENT_S3, + EVENT_BUCKET, + EVENT_OBJECT, + EVENT_KEY, + EVENT_SIZE, + EVENT_ETAG, + EVENT_NAME_FIELD, + EVENT_FIRST_RECORD_INDEX, + EVENT_PREFIX_CREATED, + EVENT_PREFIX_REMOVED, + EVENT_TYPE_CREATE, + EVENT_TYPE_DELETE, + # Record field names (dataclass attributes) + FIELD_FILE_NAME, + FIELD_BUCKET, + FIELD_COLLECTION, + FIELD_STATUS, + FIELD_START_TIME, + FIELD_END_TIME, + FIELD_DURATION_SECONDS, + FIELD_ERROR_MESSAGE, + FIELD_TASK_ID, + # Record serialization output keys + RECORD_FILE_NAME, + RECORD_BUCKET, + RECORD_COLLECTION, + RECORD_START_TIME, + RECORD_END_TIME, + RECORD_DURATION, + RECORD_STATUS, + RECORD_ERROR, + RECORD_TASK_ID, + # Status + STATUS_PENDING, + STATUS_PROCESSING, + STATUS_FINISHED, + STATUS_FAILED, + STATUS_SKIPPED, + STATUS_DELETED, + STATUS_SUCCESS, + # Config keys (MinIO sources) + CFG_ENDPOINT, + CFG_ACCESS, + CFG_SECRET, + CFG_SECURE, + CFG_COLLECTION, + CFG_BUCKETS, + # API request fields (Ingestor) + FIELD_COLLECTION_NAME, + FIELD_BLOCKING, + FIELD_SPLIT_OPTIONS, + FIELD_CHUNK_SIZE, + FIELD_CHUNK_OVERLAP, + FIELD_GENERATE_SUMMARY, + FIELD_EMBEDDING_DIMENSION, + FIELD_TASK_ID, + # API request fields (VSS) + FIELD_ID, + FIELD_FILE, + FIELD_MODEL, + FIELD_PROMPT, + FIELD_SYSTEM_PROMPT, + FIELD_MAX_TOKENS, + FIELD_CHUNK_DURATION, + FIELD_CHUNK_OVERLAP_DURATION, + FIELD_NUM_FRAMES_PER_CHUNK, + FIELD_CAPTION_SUMMARIZATION_PROMPT, + FIELD_SUMMARY_AGGREGATION_PROMPT, + FIELD_PURPOSE, + FIELD_MEDIA_TYPE, + VALUE_VISION, + VALUE_VIDEO, + FIELD_STREAM, + RESP_DELTA, + RESP_DATA_PREFIX, + # API response fields + RESP_CONTENT, + RESP_RESPONSE, + RESP_TEXT, + RESP_CHOICES, + RESP_MESSAGE, + RESP_ERROR, + RESP_COLLECTIONS, + RESP_TASK_ID, + RESP_STATE, + RESP_RESULT, + RESP_FAILED_DOCUMENTS, + RESP_VALIDATION_ERRORS, + # Timeouts + TIMEOUT_DEFAULT, + TIMEOUT_UPLOAD, + TIMEOUT_VIDEO_UPLOAD, + TIMEOUT_TASK_CHECK, + TIMEOUT_MAX_TASK_WAIT, + VIDEO_ANALYSIS_POLL_INTERVAL, + VIDEO_ANALYSIS_MAX_WAIT, + VIDEO_DESCRIPTION_MIN_LENGTH, + # Kafka defaults + KAFKA_DEFAULT_TOPIC, + KAFKA_DEFAULT_CONSUMER_GROUP, + KAFKA_DEFAULT_AUTO_OFFSET_RESET, + KAFKA_DEFAULT_MAX_POLL_RECORDS, + KAFKA_DEFAULT_MAX_POLL_INTERVAL_MS, + KAFKA_DEFAULT_SESSION_TIMEOUT_MS, + KAFKA_DEFAULT_HEARTBEAT_INTERVAL_MS, + # Collection defaults + COLLECTION_EMBEDDING_DIMENSION, + COLLECTION_CHUNK_SIZE, + COLLECTION_CHUNK_OVERLAP, + COLLECTION_VIDEO_CHUNK_SIZE, + COLLECTION_VIDEO_CHUNK_OVERLAP, +) diff --git a/examples/rag_event_ingest/kafka_consumer/config/constants.py b/examples/rag_event_ingest/kafka_consumer/config/constants.py new file mode 100644 index 000000000..a90eee342 --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/config/constants.py @@ -0,0 +1,410 @@ +# config/constants.py +"""Static constants that don't change at runtime. + +For configurable values from environment, see settings.py +""" + +# ==================== File Extensions ==================== + +VIDEO_EXTENSIONS = frozenset({ + '.mp4', '.mkv', '.avi', '.mov', '.wmv', + '.flv', '.webm', '.m4v', '.mpeg', '.mpg' +}) + +DOCUMENT_EXTENSIONS = frozenset({ + '.pdf', '.docx', '.doc', '.txt', '.md', '.rst', + '.html', '.htm', '.pptx', '.ppt', '.xlsx', '.xls', + '.csv', '.json', '.xml' +}) + +IMAGE_EXTENSIONS = frozenset({ + '.jpg', '.jpeg', '.png', '.gif', + '.webp', '.bmp', '.tiff', '.svg' +}) + +AUDIO_EXTENSIONS = frozenset({ + '.mp3', '.wav', '.flac', '.aac', + '.ogg', '.m4a', '.wma' +}) + +SKIP_EXTENSIONS = frozenset({ + '.tmp', '.log', '.bak', '.swp', '.DS_Store', + '.gitkeep', '.gitignore' +}) + + +# ==================== Content Types ==================== + +CONTENT_TYPE_MAP = { + # Documents + '.pdf': 'application/pdf', + '.txt': 'text/plain', + '.doc': 'application/msword', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.html': 'text/html', + '.htm': 'text/html', + '.xml': 'application/xml', + '.json': 'application/json', + '.csv': 'text/csv', + '.md': 'text/markdown', + '.rst': 'text/x-rst', + '.ppt': 'application/vnd.ms-powerpoint', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + '.xls': 'application/vnd.ms-excel', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + # Images + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.png': 'image/png', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.bmp': 'image/bmp', + '.tiff': 'image/tiff', + '.svg': 'image/svg+xml', + # Videos + '.mp4': 'video/mp4', + '.mkv': 'video/x-matroska', + '.avi': 'video/x-msvideo', + '.mov': 'video/quicktime', + '.wmv': 'video/x-ms-wmv', + '.flv': 'video/x-flv', + '.webm': 'video/webm', + '.m4v': 'video/x-m4v', + '.mpeg': 'video/mpeg', + '.mpg': 'video/mpeg', + # Audio + '.mp3': 'audio/mpeg', + '.wav': 'audio/wav', + '.flac': 'audio/flac', + '.aac': 'audio/aac', + '.ogg': 'audio/ogg', + '.m4a': 'audio/mp4', + '.wma': 'audio/x-ms-wma', +} + +DEFAULT_CONTENT_TYPE = 'application/octet-stream' + + +# ==================== Routing ==================== + +# Destinations +DEST_VSS = 'vss' +DEST_RAG = 'rag' +DEST_SKIP = 'skip' +DEST_UNKNOWN = 'unknown' + +# Route result keys +KEY_DESTINATION = 'destination' +KEY_FILE_TYPE = 'file_type' +KEY_EXTENSION = 'extension' +KEY_REASON = 'reason' + +# File types +FILE_TYPE_VIDEO = 'video' +FILE_TYPE_DOCUMENT = 'document' +FILE_TYPE_IMAGE = 'image' +FILE_TYPE_AUDIO = 'audio' +FILE_TYPE_SKIP = 'skip' +FILE_TYPE_UNKNOWN = 'unknown' + +# Config keys +CFG_VIDEO_EXTENSIONS = 'video_extensions' +CFG_DOCUMENT_EXTENSIONS = 'document_extensions' +CFG_IMAGE_EXTENSIONS = 'image_extensions' +CFG_AUDIO_EXTENSIONS = 'audio_extensions' +CFG_SKIP_EXTENSIONS = 'skip_extensions' +CFG_ENABLE_IMAGE_PROCESSING = 'enable_image_processing' +CFG_ENABLE_AUDIO_PROCESSING = 'enable_audio_processing' + + +# ==================== S3 Event Fields ==================== + +# Kafka S3 event structure +EVENT_NAME = 'EventName' +EVENT_RECORDS = 'Records' +EVENT_FIRST_RECORD_INDEX = 0 # S3 events typically contain single record +EVENT_S3 = 's3' +EVENT_BUCKET = 'bucket' +EVENT_OBJECT = 'object' +EVENT_KEY = 'key' +EVENT_SIZE = 'size' +EVENT_ETAG = 'eTag' +EVENT_NAME_FIELD = 'name' + +# Event type prefixes +EVENT_PREFIX_CREATED = 's3:ObjectCreated:' +EVENT_PREFIX_REMOVED = 's3:ObjectRemoved:' + +# Event type values +EVENT_TYPE_CREATE = 'create' +EVENT_TYPE_DELETE = 'delete' + + +# ==================== Record Fields ==================== + +# IngestionRecord field names (dataclass attributes) +FIELD_FILE_NAME = 'file_name' +FIELD_BUCKET = 'bucket' +FIELD_COLLECTION = 'collection' +FIELD_STATUS = 'status' +FIELD_START_TIME = 'start_time' +FIELD_END_TIME = 'end_time' +FIELD_DURATION_SECONDS = 'duration_seconds' +FIELD_ERROR_MESSAGE = 'error_message' +FIELD_TASK_ID = 'task_id' + +# IngestionRecord serialization output keys +RECORD_FILE_NAME = FIELD_FILE_NAME +RECORD_BUCKET = FIELD_BUCKET +RECORD_COLLECTION = FIELD_COLLECTION +RECORD_START_TIME = FIELD_START_TIME +RECORD_END_TIME = FIELD_END_TIME +RECORD_DURATION = FIELD_DURATION_SECONDS +RECORD_STATUS = FIELD_STATUS +RECORD_ERROR = FIELD_ERROR_MESSAGE +RECORD_TASK_ID = FIELD_TASK_ID + + +# ==================== Task Status ==================== + +STATUS_PENDING = 'PENDING' +STATUS_PROCESSING = 'PROCESSING' +STATUS_FINISHED = 'FINISHED' +STATUS_FAILED = 'FAILED' +STATUS_SKIPPED = 'SKIPPED' +STATUS_DELETED = 'DELETED' +STATUS_SUCCESS = 'SUCCESS' + + +# ==================== Config Keys ==================== + +# MinIO/S3 source config keys +CFG_ENDPOINT = 'endpoint' +CFG_ACCESS = 'access' +CFG_SECRET = 'secret' +CFG_SECURE = 'secure' +CFG_COLLECTION = 'collection' +CFG_BUCKETS = 'buckets' + + +# ==================== API Request Fields ==================== + +# Ingestor request fields +FIELD_COLLECTION_NAME = 'collection_name' +FIELD_BLOCKING = 'blocking' +FIELD_SPLIT_OPTIONS = 'split_options' +FIELD_CHUNK_SIZE = 'chunk_size' +FIELD_CHUNK_OVERLAP = 'chunk_overlap' +FIELD_GENERATE_SUMMARY = 'generate_summary' +FIELD_EMBEDDING_DIMENSION = 'embedding_dimension' +FIELD_TASK_ID = 'task_id' + +# VSS request fields +FIELD_ID = 'id' +FIELD_FILE = 'file' +FIELD_MODEL = 'model' +FIELD_PROMPT = 'prompt' +FIELD_SYSTEM_PROMPT = 'system_prompt' +FIELD_MAX_TOKENS = 'max_tokens' +FIELD_CHUNK_DURATION = 'chunk_duration' +FIELD_CHUNK_OVERLAP_DURATION = 'chunk_overlap_duration' +FIELD_NUM_FRAMES_PER_CHUNK = 'num_frames_per_chunk' +FIELD_CAPTION_SUMMARIZATION_PROMPT = 'caption_summarization_prompt' +FIELD_SUMMARY_AGGREGATION_PROMPT = 'summary_aggregation_prompt' +FIELD_PURPOSE = 'purpose' +FIELD_MEDIA_TYPE = 'media_type' + +# VSS request values +VALUE_VISION = 'vision' +VALUE_VIDEO = 'video' + +# VSS streaming fields +FIELD_STREAM = 'stream' +RESP_DELTA = 'delta' +RESP_DATA_PREFIX = 'data: ' + + +# ==================== API Response Fields ==================== + +# Common response fields +RESP_CONTENT = 'content' +RESP_RESPONSE = 'response' +RESP_TEXT = 'text' +RESP_CHOICES = 'choices' +RESP_MESSAGE = 'message' +RESP_ERROR = 'error' + +# Ingestor response fields +RESP_COLLECTIONS = 'collections' +RESP_TASK_ID = 'task_id' +RESP_STATE = 'state' +RESP_RESULT = 'result' +RESP_FAILED_DOCUMENTS = 'failed_documents' +RESP_VALIDATION_ERRORS = 'validation_errors' + + +# ==================== Timeouts (seconds) ==================== + +TIMEOUT_DEFAULT = 30 +TIMEOUT_UPLOAD = 600 +TIMEOUT_VIDEO_UPLOAD = 600 +TIMEOUT_VIDEO_INDEX = 300 +TIMEOUT_TASK_CHECK = 30 +TIMEOUT_MAX_TASK_WAIT = 300 + +# Video analysis polling +VIDEO_ANALYSIS_POLL_INTERVAL = 10 # seconds between polls +VIDEO_ANALYSIS_MAX_WAIT = 180 # max seconds to wait for analysis +VIDEO_DESCRIPTION_MIN_LENGTH = 200 # minimum chars for valid description + + +# ==================== Kafka Defaults ==================== + +KAFKA_DEFAULT_TOPIC = 'aidp-topic' +KAFKA_DEFAULT_CONSUMER_GROUP = 'nvingest-consumer-group' +KAFKA_DEFAULT_AUTO_OFFSET_RESET = 'earliest' +KAFKA_DEFAULT_MAX_POLL_RECORDS = 1 +KAFKA_DEFAULT_MAX_POLL_INTERVAL_MS = 600000 # 10 min +KAFKA_DEFAULT_SESSION_TIMEOUT_MS = 60000 # 60s +KAFKA_DEFAULT_HEARTBEAT_INTERVAL_MS = 20000 # 20s + + +# ==================== Collection Defaults ==================== + +COLLECTION_EMBEDDING_DIMENSION = 2048 +COLLECTION_CHUNK_SIZE = 512 +COLLECTION_CHUNK_OVERLAP = 150 +COLLECTION_VIDEO_CHUNK_SIZE = 1024 +COLLECTION_VIDEO_CHUNK_OVERLAP = 100 + + +# ==================== Environment Variable Keys ==================== + +# Kafka +ENV_KAFKA_BOOTSTRAP_SERVERS = 'KAFKA_BOOTSTRAP_SERVERS' +ENV_KAFKA_TOPIC = 'KAFKA_TOPIC' +ENV_CONSUMER_GROUP = 'CONSUMER_GROUP' +ENV_KAFKA_AUTO_OFFSET_RESET = 'KAFKA_AUTO_OFFSET_RESET' +ENV_KAFKA_MAX_POLL_RECORDS = 'KAFKA_MAX_POLL_RECORDS' +ENV_KAFKA_MAX_POLL_INTERVAL_MS = 'KAFKA_MAX_POLL_INTERVAL_MS' +ENV_KAFKA_SESSION_TIMEOUT_MS = 'KAFKA_SESSION_TIMEOUT_MS' +ENV_KAFKA_HEARTBEAT_INTERVAL_MS = 'KAFKA_HEARTBEAT_INTERVAL_MS' + +# Service URLs +ENV_INGESTOR_SERVER_URL = 'INGESTOR_SERVER_URL' +ENV_VSS_SERVER_URL = 'VSS_SERVER_URL' +ENV_INGESTOR_TIMEOUT = 'INGESTOR_TIMEOUT' +ENV_VSS_TIMEOUT = 'VSS_TIMEOUT' +ENV_VSS_UPLOAD_TIMEOUT = 'VSS_UPLOAD_TIMEOUT' +ENV_VIDEO_INDEX_TIMEOUT = 'VIDEO_INDEX_TIMEOUT' + +# API Endpoints +ENV_API_INGESTOR_DOCUMENTS = 'API_INGESTOR_DOCUMENTS' +ENV_API_INGESTOR_COLLECTIONS = 'API_INGESTOR_COLLECTIONS' +ENV_API_INGESTOR_COLLECTION = 'API_INGESTOR_COLLECTION' +ENV_API_INGESTOR_STATUS = 'API_INGESTOR_STATUS' +ENV_API_VSS_FILES = 'API_VSS_FILES' +ENV_API_VSS_SUMMARIZE = 'API_VSS_SUMMARIZE' + +# MinIO +ENV_MINIO_ENDPOINT = 'MINIO_ENDPOINT' +ENV_MINIO_ACCESS_KEY = 'MINIO_ACCESS_KEY' +ENV_MINIO_SECRET_KEY = 'MINIO_SECRET_KEY' +ENV_MINIO_SECURE = 'MINIO_SECURE' +ENV_COLLECTION_NAME = 'COLLECTION_NAME' +ENV_MINIO_SOURCES = 'MINIO_SOURCES' + +# Feature Flags +ENV_ENABLE_MULTIMODAL_RAG = 'ENABLE_MULTIMODAL_RAG' +ENV_ENABLE_IMAGE_PROCESSING = 'ENABLE_IMAGE_PROCESSING' +ENV_ENABLE_AUDIO_PROCESSING = 'ENABLE_AUDIO_PROCESSING' +ENV_VSS_EMBEDDING = 'VSS_EMBEDDING' + +# Collection Settings +ENV_EMBEDDING_DIMENSION = 'EMBEDDING_DIMENSION' +ENV_CHUNK_SIZE = 'CHUNK_SIZE' +ENV_CHUNK_OVERLAP = 'CHUNK_OVERLAP' + +# Logging +ENV_LOG_LEVEL = 'LOG_LEVEL' +ENV_LOG_FORMAT = 'LOG_FORMAT' + +# History +ENV_HISTORY_FILE = 'HISTORY_FILE' + +# VSS Settings +ENV_VSS_PROMPT = 'VSS_PROMPT' +ENV_VSS_SYSTEM_PROMPT = 'VSS_SYSTEM_PROMPT' +ENV_VSS_CAPTION_SUMMARIZATION_PROMPT = 'VSS_CAPTION_SUMMARIZATION_PROMPT' +ENV_VSS_SUMMARY_AGGREGATION_PROMPT = 'VSS_SUMMARY_AGGREGATION_PROMPT' +ENV_VSS_CHUNK_DURATION = 'VSS_CHUNK_DURATION' +ENV_VSS_CHUNK_OVERLAP = 'VSS_CHUNK_OVERLAP' +ENV_VSS_NUM_FRAMES_PER_CHUNK = 'VSS_NUM_FRAMES_PER_CHUNK' +ENV_VSS_MAX_TOKENS = 'VSS_MAX_TOKENS' +ENV_VSS_MODEL = 'VSS_MODEL' +ENV_VSS_STREAM_ENABLED = 'VSS_STREAM_ENABLED' + + +# ==================== API Endpoint Defaults ==================== + +DEFAULT_API_INGESTOR_DOCUMENTS = '/v1/documents' +DEFAULT_API_INGESTOR_COLLECTIONS = '/v1/collections' +DEFAULT_API_INGESTOR_COLLECTION = '/v1/collection' +DEFAULT_API_INGESTOR_STATUS = '/v1/status' +DEFAULT_API_VSS_FILES = '/files' +DEFAULT_API_VSS_SUMMARIZE = '/summarize' + + +# ==================== Logging Defaults ==================== + +DEFAULT_LOG_LEVEL = 'INFO' +DEFAULT_LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + + +# ==================== History Defaults ==================== + +DEFAULT_HISTORY_FILE = '/tmp/ingestion_history.jsonl' + + +# ==================== MinIO Defaults ==================== + +DEFAULT_COLLECTION_NAME = 'multimodal_data' + + +# ==================== VSS Defaults ==================== + +DEFAULT_VSS_MODEL = 'Cosmos-Reason2-8B' +DEFAULT_VSS_CHUNK_DURATION = 60 # seconds per chunk +DEFAULT_VSS_CHUNK_OVERLAP = 5 # overlap between chunks +DEFAULT_VSS_NUM_FRAMES_PER_CHUNK = 8 # frames per chunk for VLM +DEFAULT_VSS_MAX_TOKENS = 1024 + +# Default prompts for general videos +DEFAULT_VSS_PROMPT = ( + "Analyze this video. TIMESTAMP FORMAT: Convert seconds to MM:SS " + "(40s=00:40, 80s=01:20, 150s=02:30, 200s=03:20). " + "Describe: key events, important moments, actions, transitions, and notable scenes. " + "Capture the flow and progression of the content." +) + +DEFAULT_VSS_SYSTEM_PROMPT = ( + "You are a video analyst providing detailed descriptions. " + "CRITICAL: Convert all timestamps from seconds to MM:SS format. " + "Examples: 40 seconds = 00:40, 90 seconds = 01:30, 200 seconds = 03:20. " + "Focus on what you clearly see: actions, scenes, transitions, and key moments. " + "Be descriptive and engaging." +) + +DEFAULT_VSS_CAPTION_SUMMARIZATION_PROMPT = ( + "Format: [MM:SS] Description. CONVERT seconds to MM:SS " + "(40s=00:40, 80s=01:20, 150s=02:30). " + "Describe: what happened, how it occurred, the result. " + "Be specific about the action and context." +) + +DEFAULT_VSS_SUMMARY_AGGREGATION_PROMPT = ( + "Create a comprehensive video summary with MM:SS timestamps. " + "CONVERT all times: 40s=00:40, 80s=01:20, 200s=03:20, 350s=05:50. " + "Highlight: key events, transitions, important moments, and notable scenes. " + "Write in an engaging narrative style." +) diff --git a/examples/rag_event_ingest/kafka_consumer/config/settings.py b/examples/rag_event_ingest/kafka_consumer/config/settings.py new file mode 100644 index 000000000..69794c2a0 --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/config/settings.py @@ -0,0 +1,199 @@ +# config/settings.py +"""Runtime settings loaded from environment variables.""" + +import os + +from .constants import ( + # Default values + KAFKA_DEFAULT_TOPIC, + KAFKA_DEFAULT_CONSUMER_GROUP, + KAFKA_DEFAULT_AUTO_OFFSET_RESET, + KAFKA_DEFAULT_MAX_POLL_RECORDS, + KAFKA_DEFAULT_MAX_POLL_INTERVAL_MS, + KAFKA_DEFAULT_SESSION_TIMEOUT_MS, + KAFKA_DEFAULT_HEARTBEAT_INTERVAL_MS, + TIMEOUT_UPLOAD, + TIMEOUT_VIDEO_UPLOAD, + TIMEOUT_VIDEO_INDEX, + COLLECTION_EMBEDDING_DIMENSION, + COLLECTION_CHUNK_SIZE, + COLLECTION_CHUNK_OVERLAP, + # API Endpoint defaults + DEFAULT_API_INGESTOR_DOCUMENTS, + DEFAULT_API_INGESTOR_COLLECTIONS, + DEFAULT_API_INGESTOR_COLLECTION, + DEFAULT_API_INGESTOR_STATUS, + DEFAULT_API_VSS_FILES, + DEFAULT_API_VSS_SUMMARIZE, + # Logging defaults + DEFAULT_LOG_LEVEL, + DEFAULT_LOG_FORMAT, + # History defaults + DEFAULT_HISTORY_FILE, + # MinIO defaults + DEFAULT_COLLECTION_NAME, + # VSS defaults + DEFAULT_VSS_MODEL, + DEFAULT_VSS_CHUNK_DURATION, + DEFAULT_VSS_CHUNK_OVERLAP, + DEFAULT_VSS_NUM_FRAMES_PER_CHUNK, + DEFAULT_VSS_MAX_TOKENS, + DEFAULT_VSS_PROMPT, + DEFAULT_VSS_SYSTEM_PROMPT, + DEFAULT_VSS_CAPTION_SUMMARIZATION_PROMPT, + DEFAULT_VSS_SUMMARY_AGGREGATION_PROMPT, + # Environment variable keys + ENV_KAFKA_BOOTSTRAP_SERVERS, + ENV_KAFKA_TOPIC, + ENV_CONSUMER_GROUP, + ENV_KAFKA_AUTO_OFFSET_RESET, + ENV_KAFKA_MAX_POLL_RECORDS, + ENV_KAFKA_MAX_POLL_INTERVAL_MS, + ENV_KAFKA_SESSION_TIMEOUT_MS, + ENV_KAFKA_HEARTBEAT_INTERVAL_MS, + ENV_INGESTOR_SERVER_URL, + ENV_VSS_SERVER_URL, + ENV_INGESTOR_TIMEOUT, + ENV_VSS_TIMEOUT, + ENV_VSS_UPLOAD_TIMEOUT, + ENV_VIDEO_INDEX_TIMEOUT, + ENV_API_INGESTOR_DOCUMENTS, + ENV_API_INGESTOR_COLLECTIONS, + ENV_API_INGESTOR_COLLECTION, + ENV_API_INGESTOR_STATUS, + ENV_API_VSS_FILES, + ENV_API_VSS_SUMMARIZE, + ENV_MINIO_ENDPOINT, + ENV_MINIO_ACCESS_KEY, + ENV_MINIO_SECRET_KEY, + ENV_MINIO_SECURE, + ENV_COLLECTION_NAME, + ENV_MINIO_SOURCES, + ENV_ENABLE_MULTIMODAL_RAG, + ENV_ENABLE_IMAGE_PROCESSING, + ENV_ENABLE_AUDIO_PROCESSING, + ENV_VSS_EMBEDDING, + ENV_EMBEDDING_DIMENSION, + ENV_CHUNK_SIZE, + ENV_CHUNK_OVERLAP, + ENV_LOG_LEVEL, + ENV_LOG_FORMAT, + ENV_HISTORY_FILE, + ENV_VSS_PROMPT, + ENV_VSS_SYSTEM_PROMPT, + ENV_VSS_CAPTION_SUMMARIZATION_PROMPT, + ENV_VSS_SUMMARY_AGGREGATION_PROMPT, + ENV_VSS_CHUNK_DURATION, + ENV_VSS_CHUNK_OVERLAP, + ENV_VSS_NUM_FRAMES_PER_CHUNK, + ENV_VSS_MAX_TOKENS, + ENV_VSS_MODEL, + ENV_VSS_STREAM_ENABLED, +) + + +# ==================== Helper Functions ==================== + +def _get_bool(key: str, default: bool = False) -> bool: + """Get boolean from environment variable.""" + return os.getenv(key, str(default)).lower() in ('true', '1', 'yes', 'on') + + +def _get_int(key: str, default: int) -> int: + """Get integer from environment variable.""" + try: + return int(os.getenv(key, str(default))) + except ValueError: + return default + + +# ==================== Kafka Settings ==================== + +KAFKA_BOOTSTRAP_SERVERS = os.getenv(ENV_KAFKA_BOOTSTRAP_SERVERS) # Required +KAFKA_CONSUMER_GROUP = os.getenv(ENV_CONSUMER_GROUP, KAFKA_DEFAULT_CONSUMER_GROUP) +KAFKA_TOPIC = os.getenv(ENV_KAFKA_TOPIC, KAFKA_DEFAULT_TOPIC) +KAFKA_AUTO_OFFSET_RESET = os.getenv(ENV_KAFKA_AUTO_OFFSET_RESET, KAFKA_DEFAULT_AUTO_OFFSET_RESET) +KAFKA_MAX_POLL_RECORDS = _get_int(ENV_KAFKA_MAX_POLL_RECORDS, KAFKA_DEFAULT_MAX_POLL_RECORDS) +KAFKA_MAX_POLL_INTERVAL_MS = _get_int(ENV_KAFKA_MAX_POLL_INTERVAL_MS, KAFKA_DEFAULT_MAX_POLL_INTERVAL_MS) +KAFKA_SESSION_TIMEOUT_MS = _get_int(ENV_KAFKA_SESSION_TIMEOUT_MS, KAFKA_DEFAULT_SESSION_TIMEOUT_MS) +KAFKA_HEARTBEAT_INTERVAL_MS = _get_int(ENV_KAFKA_HEARTBEAT_INTERVAL_MS, KAFKA_DEFAULT_HEARTBEAT_INTERVAL_MS) + + +# ==================== Service URLs ==================== + +INGESTOR_SERVER_URL = os.getenv(ENV_INGESTOR_SERVER_URL) # Required +VSS_SERVER_URL = os.getenv(ENV_VSS_SERVER_URL) # Required +INGESTOR_TIMEOUT = _get_int(ENV_INGESTOR_TIMEOUT, TIMEOUT_UPLOAD) +VSS_TIMEOUT = _get_int(ENV_VSS_TIMEOUT, TIMEOUT_VIDEO_UPLOAD) +VSS_UPLOAD_TIMEOUT = _get_int(ENV_VSS_UPLOAD_TIMEOUT, TIMEOUT_VIDEO_UPLOAD) +VIDEO_INDEX_TIMEOUT = _get_int(ENV_VIDEO_INDEX_TIMEOUT, TIMEOUT_VIDEO_INDEX) + +# API Endpoints - Ingestor Server +API_INGESTOR_DOCUMENTS = os.getenv(ENV_API_INGESTOR_DOCUMENTS, DEFAULT_API_INGESTOR_DOCUMENTS) +API_INGESTOR_COLLECTIONS = os.getenv(ENV_API_INGESTOR_COLLECTIONS, DEFAULT_API_INGESTOR_COLLECTIONS) +API_INGESTOR_COLLECTION = os.getenv(ENV_API_INGESTOR_COLLECTION, DEFAULT_API_INGESTOR_COLLECTION) +API_INGESTOR_STATUS = os.getenv(ENV_API_INGESTOR_STATUS, DEFAULT_API_INGESTOR_STATUS) + +# API Endpoints - VSS +API_VSS_FILES = os.getenv(ENV_API_VSS_FILES, DEFAULT_API_VSS_FILES) +API_VSS_SUMMARIZE = os.getenv(ENV_API_VSS_SUMMARIZE, DEFAULT_API_VSS_SUMMARIZE) + + +# ==================== MinIO Settings ==================== + +MINIO_ENDPOINT = os.getenv(ENV_MINIO_ENDPOINT) # Required +MINIO_ACCESS_KEY = os.getenv(ENV_MINIO_ACCESS_KEY) # Required +MINIO_SECRET_KEY = os.getenv(ENV_MINIO_SECRET_KEY) # Required +MINIO_SECURE = _get_bool(ENV_MINIO_SECURE, False) +# Single collection for all buckets - matches RAG server's COLLECTION_NAME +MINIO_DEFAULT_COLLECTION = os.getenv(ENV_COLLECTION_NAME, DEFAULT_COLLECTION_NAME) +MINIO_SOURCES = os.getenv(ENV_MINIO_SOURCES) # JSON config for multi-source + + +# ==================== Feature Flags ==================== + +ENABLE_MULTIMODAL_RAG = _get_bool(ENV_ENABLE_MULTIMODAL_RAG, True) +ENABLE_IMAGE_PROCESSING = _get_bool(ENV_ENABLE_IMAGE_PROCESSING, False) +ENABLE_AUDIO_PROCESSING = _get_bool(ENV_ENABLE_AUDIO_PROCESSING, False) +VSS_EMBEDDING_ENABLED = _get_bool(ENV_VSS_EMBEDDING, False) + + +# ==================== Collection Settings ==================== + +EMBEDDING_DIMENSION = _get_int(ENV_EMBEDDING_DIMENSION, COLLECTION_EMBEDDING_DIMENSION) +CHUNK_SIZE = _get_int(ENV_CHUNK_SIZE, COLLECTION_CHUNK_SIZE) +CHUNK_OVERLAP = _get_int(ENV_CHUNK_OVERLAP, COLLECTION_CHUNK_OVERLAP) + + +# ==================== Logging Settings ==================== + +LOG_LEVEL = os.getenv(ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL) +LOG_FORMAT = os.getenv(ENV_LOG_FORMAT, DEFAULT_LOG_FORMAT) + + +# ==================== History Settings ==================== + +HISTORY_FILE = os.getenv(ENV_HISTORY_FILE, DEFAULT_HISTORY_FILE) + + +# ==================== VSS Settings ==================== + +# Default prompts for general videos (can be overridden via environment variables) +# Use environment variables for specific video types (sports, talks, tutorials, etc.) +VSS_DEFAULT_PROMPT = os.getenv(ENV_VSS_PROMPT, DEFAULT_VSS_PROMPT) +VSS_SYSTEM_PROMPT = os.getenv(ENV_VSS_SYSTEM_PROMPT, DEFAULT_VSS_SYSTEM_PROMPT) +VSS_CAPTION_SUMMARIZATION_PROMPT = os.getenv( + ENV_VSS_CAPTION_SUMMARIZATION_PROMPT, DEFAULT_VSS_CAPTION_SUMMARIZATION_PROMPT +) +VSS_SUMMARY_AGGREGATION_PROMPT = os.getenv( + ENV_VSS_SUMMARY_AGGREGATION_PROMPT, DEFAULT_VSS_SUMMARY_AGGREGATION_PROMPT +) + +# VSS chunking settings for long videos +VSS_CHUNK_DURATION = _get_int(ENV_VSS_CHUNK_DURATION, DEFAULT_VSS_CHUNK_DURATION) +VSS_CHUNK_OVERLAP = _get_int(ENV_VSS_CHUNK_OVERLAP, DEFAULT_VSS_CHUNK_OVERLAP) +VSS_NUM_FRAMES_PER_CHUNK = _get_int(ENV_VSS_NUM_FRAMES_PER_CHUNK, DEFAULT_VSS_NUM_FRAMES_PER_CHUNK) +VSS_MAX_TOKENS = _get_int(ENV_VSS_MAX_TOKENS, DEFAULT_VSS_MAX_TOKENS) +VSS_MODEL = os.getenv(ENV_VSS_MODEL, DEFAULT_VSS_MODEL) +VSS_STREAM_ENABLED = _get_bool(ENV_VSS_STREAM_ENABLED, True) + diff --git a/examples/rag_event_ingest/kafka_consumer/consumer.py b/examples/rag_event_ingest/kafka_consumer/consumer.py new file mode 100644 index 000000000..b78d3c322 --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/consumer.py @@ -0,0 +1,204 @@ +# consumer.py +"""Kafka consumer for MinIO S3 events.""" + +import json +import logging +from datetime import datetime +from typing import Dict, Optional +from kafka import KafkaConsumer + +import config.settings as cfg +from pathlib import Path +from config.constants import DEST_RAG, DEST_SKIP, STATUS_FAILED, KEY_DESTINATION, KEY_FILE_TYPE, KEY_REASON +from router import FileRouter +from models.events import S3Event, HandlerResult, IngestionRecord +from handlers.base import BaseHandler +from services.storage import ObjectStorage + +logger = logging.getLogger(__name__) + + +class KafkaEventConsumer: + """Kafka consumer that routes MinIO events to handlers.""" + + def __init__( + self, + handlers: Dict[str, BaseHandler], + storage: ObjectStorage, + history_file: str = '/tmp/ingestion_history.jsonl' + ): + """Initialize Kafka consumer.""" + self.handlers = handlers + self.storage = storage + self.history_file = history_file + self.router = FileRouter() + + logger.info(f"Connecting to Kafka: {cfg.KAFKA_BOOTSTRAP_SERVERS}") + logger.info(f"Consumer group: {cfg.KAFKA_CONSUMER_GROUP}") + + self.kafka_consumer = KafkaConsumer( + cfg.KAFKA_TOPIC, + bootstrap_servers=cfg.KAFKA_BOOTSTRAP_SERVERS.split(','), + value_deserializer=lambda m: json.loads(m.decode('utf-8')), + group_id=cfg.KAFKA_CONSUMER_GROUP, + auto_offset_reset=cfg.KAFKA_AUTO_OFFSET_RESET, + enable_auto_commit=True, + max_poll_records=cfg.KAFKA_MAX_POLL_RECORDS, + max_poll_interval_ms=cfg.KAFKA_MAX_POLL_INTERVAL_MS, + session_timeout_ms=cfg.KAFKA_SESSION_TIMEOUT_MS, + heartbeat_interval_ms=cfg.KAFKA_HEARTBEAT_INTERVAL_MS + ) + + logger.info("Kafka consumer initialized") + logger.info(f"Registered handlers: {list(self.handlers.keys())}") + + def process_event(self, raw_event: dict) -> Optional[HandlerResult]: + """Process a single MinIO S3 event.""" + start_time = datetime.now() + event: Optional[S3Event] = None + result: Optional[HandlerResult] = None + + try: + logger.info(f"Received event: {json.dumps(raw_event, indent=2)}") + + event = S3Event.from_kafka_message( + raw_event, + collection_resolver=self.storage.get_collection_for_bucket + ) + + if not event: + logger.warning("Invalid event format, skipping") + return None + + logger.info(f"Processing: {event.bucket}/{event.key} ({event.size} bytes)") + + if event.event_type == 'delete': + result = self._handle_delete(event) + else: + result = self._handle_create(event) + + return result + + except (json.JSONDecodeError, KeyError, ValueError) as e: + logger.error(f"Invalid event data: {e}") + result = HandlerResult.failed_result(str(e)) + return result + + except (IOError, OSError) as e: + logger.error(f"Storage error: {e}") + result = HandlerResult.failed_result(str(e)) + return result + + finally: + if event: + self._save_record(event, result, start_time) + + def _handle_delete(self, event: S3Event) -> HandlerResult: + """Handle S3 delete event.""" + logger.info(f"🗑️ DELETE event for {event.key}") + + doc_handler = self.handlers.get(DEST_RAG) + if not doc_handler or not hasattr(doc_handler, 'indexer'): + return HandlerResult.failed_result("Delete failed - no indexer available") + + indexer = doc_handler.indexer + success = indexer.delete(event.key, event.collection) + + if self.router.is_video(event.key): + desc_filename = f"{event.key}_description.json" + desc_ok = indexer.delete(desc_filename, event.collection) + if desc_ok: + logger.info(f"✓ Deleted video description {desc_filename} from Milvus") + success = success or desc_ok + + if success: + logger.info(f"✓ Deleted {event.key} from Milvus") + return HandlerResult(success=True, status='DELETED') + + return HandlerResult.failed_result("Delete failed") + + def _handle_create(self, event: S3Event) -> HandlerResult: + """Handle S3 create event.""" + route_info = self.router.route(event.key) + destination = route_info[KEY_DESTINATION] + + logger.info(f"📁 {route_info[KEY_FILE_TYPE]} → {destination}") + + if destination == DEST_SKIP: + reason = route_info.get(KEY_REASON, 'Skipped by router') + logger.info(f"⏭️ Skipping: {reason}") + return HandlerResult.skipped_result(reason) + + handler = self.handlers.get(destination) + if not handler: + handler = self.handlers.get(DEST_RAG) + + if not handler: + return HandlerResult.failed_result(f"No handler for {destination}") + + return handler.handle(event) + + def _save_record(self, event: S3Event, result: Optional[HandlerResult], start_time: datetime): + """Save ingestion record to history file.""" + end_time = datetime.now() + duration = (end_time - start_time).total_seconds() + + record = IngestionRecord( + file_name=event.key, + bucket=event.bucket, + collection=event.collection, + status=result.status if result else STATUS_FAILED, + start_time=start_time, + end_time=end_time, + duration_seconds=duration, + error_message=result.error_message if result else None, + task_id=result.task_id if result else None + ) + + try: + with open(self.history_file, 'a') as f: + f.write(json.dumps(record.to_dict()) + '\n') + except (IOError, OSError) as e: + logger.error(f"Failed to save history: {e}") + + status_emoji = '✓' if record.status in ['SUCCESS', 'DELETED', 'SKIPPED'] else '✗' + logger.info( + f"{status_emoji} SUMMARY: {event.key} | " + f"Collection: {event.collection} | " + f"Duration: {duration:.2f}s | " + f"Status: {record.status}" + ) + + def run(self): + """Main consumer loop.""" + logger.info("Starting Kafka consumer loop...") + logger.info(f"Subscribed topics: {self.kafka_consumer.subscription()}") + logger.info("Waiting for messages...") + + try: + message_count = 0 + for message in self._poll_messages(): + message_count += 1 + logger.info( + f"[{message_count}] Message from " + f"partition {message.partition}, offset {message.offset}" + ) + self.process_event(message.value) + + except KeyboardInterrupt: + logger.info("Shutting down...") + finally: + self.kafka_consumer.close() + logger.info("Consumer closed") + + def _poll_messages(self): + """Generator that yields messages from Kafka.""" + while True: + msg_pack = self.kafka_consumer.poll(timeout_ms=5000, max_records=1) + + if not msg_pack: + logger.debug("No messages, continuing...") + continue + + for messages in msg_pack.values(): + yield from messages diff --git a/examples/rag_event_ingest/kafka_consumer/handlers/__init__.py b/examples/rag_event_ingest/kafka_consumer/handlers/__init__.py new file mode 100644 index 000000000..112493e8e --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/handlers/__init__.py @@ -0,0 +1,6 @@ +# Handlers package +from .base import BaseHandler +from .document import DocumentHandler +from .video import VideoHandler + +__all__ = ['BaseHandler', 'DocumentHandler', 'VideoHandler'] diff --git a/examples/rag_event_ingest/kafka_consumer/handlers/base.py b/examples/rag_event_ingest/kafka_consumer/handlers/base.py new file mode 100644 index 000000000..6745f2e09 --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/handlers/base.py @@ -0,0 +1,43 @@ +# handlers/base.py +"""Base handler abstract class.""" + +from abc import ABC, abstractmethod +import logging + +from models.events import S3Event, HandlerResult + +logger = logging.getLogger(__name__) + + +class BaseHandler(ABC): + """Abstract base class for file handlers.""" + + @property + @abstractmethod + def name(self) -> str: + """Handler name for logging.""" + pass + + @abstractmethod + def handle(self, event: S3Event) -> HandlerResult: + """Process an S3 event. + + Args: + event: S3 event to process + + Returns: + HandlerResult with success status and optional task_id + """ + pass + + def log_start(self, event: S3Event): + """Log handler start.""" + logger.info(f"[{self.name}] Processing {event.bucket}/{event.key}") + + def log_success(self, event: S3Event, result: HandlerResult): + """Log successful handling.""" + logger.info(f"[{self.name}] ✓ {event.key} → {result.status}") + + def log_failure(self, event: S3Event, result: HandlerResult): + """Log failed handling.""" + logger.error(f"[{self.name}] ✗ {event.key}: {result.error_message}") diff --git a/examples/rag_event_ingest/kafka_consumer/handlers/document.py b/examples/rag_event_ingest/kafka_consumer/handlers/document.py new file mode 100644 index 000000000..1df03946d --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/handlers/document.py @@ -0,0 +1,89 @@ +# handlers/document.py +"""Handler for document files (PDF, DOCX, TXT, etc.).""" + +import logging + +import requests + +from .base import BaseHandler +from models.events import S3Event, HandlerResult +from services.storage import ObjectStorage +from services.document_indexer import DocumentIndexer + +logger = logging.getLogger(__name__) + + +class DocumentHandler(BaseHandler): + """Handler for document files - sends to RAG ingestor.""" + + def __init__(self, storage: ObjectStorage, indexer: DocumentIndexer): + """Initialize document handler. + + Args: + storage: Object storage for file downloads + indexer: Document indexer for RAG pipeline + """ + self.storage = storage + self.indexer = indexer + + @property + def name(self) -> str: + return "DocumentHandler" + + def handle(self, event: S3Event) -> HandlerResult: + """Process document file. + + 1. Delete existing entries (for updates) + 2. Download from MinIO + 3. Upload to ingestor + 4. Wait for completion + + Args: + event: S3 event with document info + + Returns: + HandlerResult with task_id for status tracking + """ + self.log_start(event) + + try: + # Step 1: Delete existing entries (handles updates) + logger.info(f"🔄 Checking for existing entries of {event.key}...") + self.indexer.delete(event.key, event.collection) + + # Step 2: Download from storage + logger.info(f"📥 Downloading from storage...") + file_data = self.storage.download(event.bucket, event.key) + + # Step 3: Upload to indexer + logger.info(f"📤 Sending to indexer...") + task_id = self.indexer.upload( + file_data=file_data, + filename=event.key, + collection=event.collection + ) + + if not task_id: + result = HandlerResult.failed_result("Indexer upload failed") + self.log_failure(event, result) + return result + + # Step 4: Wait for completion + logger.info(f"⏳ Waiting for indexing (task_id: {task_id})...") + success, message = self.indexer.check_status(task_id) + + if success: + result = HandlerResult.success_result(task_id=task_id) + self.log_success(event, result) + return result + else: + result = HandlerResult.failed_result(message, task_id=task_id) + self.log_failure(event, result) + return result + + except requests.RequestException as e: + logger.error(f"Network error processing document: {e}") + return HandlerResult.failed_result(str(e)) + except (IOError, OSError) as e: + logger.error(f"Storage error processing document: {e}") + return HandlerResult.failed_result(str(e)) diff --git a/examples/rag_event_ingest/kafka_consumer/handlers/video.py b/examples/rag_event_ingest/kafka_consumer/handlers/video.py new file mode 100644 index 000000000..9e5d73a34 --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/handlers/video.py @@ -0,0 +1,118 @@ +# handlers/video.py +"""Handler for video files.""" + +import logging + +import requests + +from .base import BaseHandler +from models.events import S3Event, HandlerResult +from services.storage import ObjectStorage +from services.video_analyzer import VideoAnalyzer +from services.document_indexer import DocumentIndexer + +logger = logging.getLogger(__name__) + + +class VideoHandler(BaseHandler): + """Handler for video files - analyzes and indexes in vector store.""" + + def __init__( + self, + storage: ObjectStorage, + analyzer: VideoAnalyzer, + indexer: DocumentIndexer, + enable_multimodal_rag: bool = True + ): + """Initialize video handler. + + Args: + storage: Object storage for file downloads + analyzer: Video analyzer for VLM analysis + indexer: Document indexer for storing descriptions + enable_multimodal_rag: Whether to index video descriptions + """ + self.storage = storage + self.analyzer = analyzer + self.indexer = indexer + self.enable_multimodal_rag = enable_multimodal_rag + + @property + def name(self) -> str: + return "VideoHandler" + + def handle(self, event: S3Event) -> HandlerResult: + """Process video file. + + 1. Download from MinIO + 2. Upload to VSS + 3. (Optional) Get description and index in Milvus + + Args: + event: S3 event with video info + + Returns: + HandlerResult + """ + self.log_start(event) + + try: + # Step 1: Download from storage + logger.info(f"📥 Downloading video from storage...") + video_data = self.storage.download(event.bucket, event.key) + + # Step 2: Upload to VSS + logger.info(f"📤 Uploading to analyzer...") + success, video_name = self.analyzer.upload_video(video_data, event.key) + + if not success: + result = HandlerResult.failed_result("Video upload failed") + self.log_failure(event, result) + return result + + logger.info(f"✓ Video uploaded: {video_name}") + + # Step 3: Multi-modal RAG indexing (optional) + if self.enable_multimodal_rag and video_name: + self._index_video_description(event, video_name) + + result = HandlerResult.success_result() + self.log_success(event, result) + return result + + except requests.RequestException as e: + logger.error(f"Network error processing video: {e}") + return HandlerResult.failed_result(str(e)) + except (IOError, OSError) as e: + logger.error(f"Storage error processing video: {e}") + return HandlerResult.failed_result(str(e)) + + def _index_video_description(self, event: S3Event, video_name: str): + """Get video description from VSS and index in Milvus. + + Args: + event: S3 event + video_name: Video name (stem of filename) + """ + logger.info(f"🔄 Starting Multi-Modal RAG indexing...") + + # Get description using VSS /generate with detailed prompt + logger.info(f"📹 Generating description for video: {video_name}...") + description = self.analyzer.get_video_description(video_name) + + if not description: + logger.warning("Failed to get video description from VSS") + return + + # Index description in Milvus + success = self.indexer.index_video_description( + collection=event.collection, + video_id=video_name, + video_name=event.key, + description=description + ) + + if success: + logger.info(f"✓ Indexed video description for {event.key}") + else: + logger.warning(f"⚠ Failed to index video description") diff --git a/examples/rag_event_ingest/kafka_consumer/main.py b/examples/rag_event_ingest/kafka_consumer/main.py new file mode 100644 index 000000000..02e86c137 --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/main.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# main.py +"""Entry point for Kafka MinIO consumer.""" + +import logging + +import config.settings as cfg +from config.constants import DEST_RAG, DEST_VSS +from services import ObjectStorage, DocumentIndexer, VideoAnalyzer +from handlers import DocumentHandler, VideoHandler +from consumer import KafkaEventConsumer + +logging.basicConfig( + level=getattr(logging, cfg.LOG_LEVEL, logging.INFO), + format=cfg.LOG_FORMAT +) +logger = logging.getLogger(__name__) + + +def main(): + """Initialize and run the Kafka consumer.""" + logger.info("=" * 60) + logger.info("Starting Kafka MinIO Consumer") + logger.info("=" * 60) + + # Initialize services + logger.info("Initializing services...") + storage = ObjectStorage() + indexer = DocumentIndexer(cfg.INGESTOR_SERVER_URL) + analyzer = VideoAnalyzer(cfg.VSS_SERVER_URL) + + # Initialize handlers + logger.info("Initializing handlers...") + handlers = { + DEST_RAG: DocumentHandler(storage, indexer), + DEST_VSS: VideoHandler(storage, analyzer, indexer, enable_multimodal_rag=cfg.ENABLE_MULTIMODAL_RAG), + } + + # Initialize consumer + logger.info("Initializing Kafka consumer...") + consumer = KafkaEventConsumer(handlers=handlers, storage=storage, history_file=cfg.HISTORY_FILE) + + # Run consumer loop + logger.info("Starting consumer loop...") + consumer.run() + + +if __name__ == '__main__': + main() diff --git a/examples/rag_event_ingest/kafka_consumer/models/__init__.py b/examples/rag_event_ingest/kafka_consumer/models/__init__.py new file mode 100644 index 000000000..2abce8a0d --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/models/__init__.py @@ -0,0 +1,4 @@ +# Models package +from .events import S3Event, HandlerResult, IngestionRecord + +__all__ = ['S3Event', 'HandlerResult', 'IngestionRecord'] diff --git a/examples/rag_event_ingest/kafka_consumer/models/events.py b/examples/rag_event_ingest/kafka_consumer/models/events.py new file mode 100644 index 000000000..4baf7112f --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/models/events.py @@ -0,0 +1,138 @@ +# models/events.py +"""Data models for Kafka consumer events and results.""" + +from dataclasses import dataclass, field, fields +from datetime import datetime +from typing import Any, Callable, ClassVar, Dict, Optional +from urllib.parse import unquote_plus + +from config.constants import ( + STATUS_SUCCESS, + STATUS_FAILED, + STATUS_SKIPPED, + # S3 Event fields + EVENT_NAME, + EVENT_RECORDS, + EVENT_FIRST_RECORD_INDEX, + EVENT_S3, + EVENT_BUCKET, + EVENT_OBJECT, + EVENT_KEY, + EVENT_SIZE, + EVENT_ETAG, + EVENT_NAME_FIELD, + EVENT_PREFIX_CREATED, + EVENT_PREFIX_REMOVED, + EVENT_TYPE_CREATE, + EVENT_TYPE_DELETE, + # Record field names (for transformers) + FIELD_START_TIME, + FIELD_END_TIME, + FIELD_DURATION_SECONDS, +) + + +@dataclass +class S3Event: + """Represents a MinIO S3 event from Kafka.""" + bucket: str + key: str + size: int + etag: str + event_type: str + collection: str = '' + + @classmethod + def from_kafka_message( + cls, + event: Dict[str, Any], + collection_resolver: Callable[[str], str] + ) -> Optional['S3Event']: + """Parse S3 event from Kafka message. + + Args: + event: Raw Kafka message value + collection_resolver: Function to resolve bucket -> collection name + """ + if EVENT_NAME not in event: + return None + + event_name = event[EVENT_NAME] + + if event_name.startswith(EVENT_PREFIX_CREATED): + event_type = EVENT_TYPE_CREATE + elif event_name.startswith(EVENT_PREFIX_REMOVED): + event_type = EVENT_TYPE_DELETE + else: + return None + + records = event.get(EVENT_RECORDS, []) + if not records: + return None + + record = records[EVENT_FIRST_RECORD_INDEX] + s3_data = record[EVENT_S3] + bucket = s3_data[EVENT_BUCKET][EVENT_NAME_FIELD] + obj_data = s3_data[EVENT_OBJECT] + key = unquote_plus(obj_data[EVENT_KEY]) + size = obj_data.get(EVENT_SIZE, 0) + etag = obj_data.get(EVENT_ETAG, '') + + return cls( + bucket=bucket, + key=key, + size=size, + etag=etag, + event_type=event_type, + collection=collection_resolver(bucket) + ) + + +@dataclass +class HandlerResult: + """Result from a handler execution.""" + success: bool + status: str # SUCCESS, FAILED, SKIPPED, DELETED + error_message: Optional[str] = None + task_id: Optional[str] = None # For RAG status tracking + + @classmethod + def success_result(cls, task_id: Optional[str] = None) -> 'HandlerResult': + return cls(success=True, status=STATUS_SUCCESS, task_id=task_id) + + @classmethod + def failed_result(cls, error: str, task_id: Optional[str] = None) -> 'HandlerResult': + return cls(success=False, status=STATUS_FAILED, error_message=error, task_id=task_id) + + @classmethod + def skipped_result(cls, reason: str) -> 'HandlerResult': + return cls(success=True, status=STATUS_SKIPPED, error_message=reason) + + +@dataclass +class IngestionRecord: + """Record of an ingestion operation for history tracking.""" + file_name: str + bucket: str + collection: str + status: str + start_time: datetime + end_time: datetime = field(default_factory=datetime.now) + duration_seconds: float = 0.0 + error_message: Optional[str] = None + task_id: Optional[str] = None + + _TRANSFORMERS: ClassVar[Dict[str, Callable]] = { + FIELD_START_TIME: lambda v: v.isoformat(), + FIELD_END_TIME: lambda v: v.isoformat(), + FIELD_DURATION_SECONDS: lambda v: round(v, 2), + } + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization.""" + result = {} + for f in fields(self): + value = getattr(self, f.name) + transform = self._TRANSFORMERS.get(f.name) + result[f.name] = transform(value) if transform else value + return result diff --git a/examples/rag_event_ingest/kafka_consumer/requirements.txt b/examples/rag_event_ingest/kafka_consumer/requirements.txt new file mode 100644 index 000000000..3f3818161 --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/requirements.txt @@ -0,0 +1,4 @@ +kafka-python==2.0.2 +minio==7.2.0 +requests==2.31.0 +requests-toolbelt==1.0.0 diff --git a/examples/rag_event_ingest/kafka_consumer/router.py b/examples/rag_event_ingest/kafka_consumer/router.py new file mode 100644 index 000000000..df9432357 --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/router.py @@ -0,0 +1,104 @@ +# router.py +"""File routing module for MinIO event processing.""" + +import logging +from pathlib import Path +from typing import Dict, Any, List, Set, Union + +from config.constants import ( + VIDEO_EXTENSIONS, + DOCUMENT_EXTENSIONS, + IMAGE_EXTENSIONS, + AUDIO_EXTENSIONS, + SKIP_EXTENSIONS, + DEST_VSS, + DEST_RAG, + DEST_SKIP, + KEY_DESTINATION, + KEY_FILE_TYPE, + KEY_EXTENSION, + KEY_REASON, + FILE_TYPE_VIDEO, + FILE_TYPE_DOCUMENT, + FILE_TYPE_IMAGE, + FILE_TYPE_AUDIO, + FILE_TYPE_SKIP, + FILE_TYPE_UNKNOWN, + CFG_VIDEO_EXTENSIONS, + CFG_DOCUMENT_EXTENSIONS, + CFG_IMAGE_EXTENSIONS, + CFG_AUDIO_EXTENSIONS, + CFG_SKIP_EXTENSIONS, + CFG_ENABLE_IMAGE_PROCESSING, + CFG_ENABLE_AUDIO_PROCESSING, +) + +logger = logging.getLogger(__name__) + + +class FileRouter: + """Routes files to appropriate processing services based on file type.""" + + def __init__(self, config: Union[Dict[str, Any], Any] = None): + """Initialize router with optional config overrides.""" + if config is None: + config = {} + elif hasattr(config, '__dataclass_fields__'): + config = { + CFG_VIDEO_EXTENSIONS: config.video_extensions, + CFG_DOCUMENT_EXTENSIONS: config.document_extensions, + CFG_IMAGE_EXTENSIONS: config.image_extensions, + CFG_AUDIO_EXTENSIONS: config.audio_extensions, + CFG_SKIP_EXTENSIONS: config.skip_extensions, + CFG_ENABLE_IMAGE_PROCESSING: config.enable_image_processing, + CFG_ENABLE_AUDIO_PROCESSING: config.enable_audio_processing, + } + + self.config = config + self.video_extensions = self._to_set(config.get(CFG_VIDEO_EXTENSIONS, VIDEO_EXTENSIONS)) + self.document_extensions = self._to_set(config.get(CFG_DOCUMENT_EXTENSIONS, DOCUMENT_EXTENSIONS)) + self.image_extensions = self._to_set(config.get(CFG_IMAGE_EXTENSIONS, IMAGE_EXTENSIONS)) + self.audio_extensions = self._to_set(config.get(CFG_AUDIO_EXTENSIONS, AUDIO_EXTENSIONS)) + self.skip_extensions = self._to_set(config.get(CFG_SKIP_EXTENSIONS, SKIP_EXTENSIONS)) + self.enable_image_processing = config.get(CFG_ENABLE_IMAGE_PROCESSING, False) + self.enable_audio_processing = config.get(CFG_ENABLE_AUDIO_PROCESSING, False) + + logger.info(f"FileRouter initialized - Video: {len(self.video_extensions)} types, " + f"Documents: {len(self.document_extensions)} types") + + @staticmethod + def _to_set(value: Union[List, Set, None]) -> Set[str]: + if value is None: + return set() + return set(value) if isinstance(value, (list, tuple)) else value + + def route(self, filename: str) -> dict: + """Determine routing destination for a file.""" + ext = Path(filename).suffix.lower() + + if ext in self.skip_extensions: + return {KEY_DESTINATION: DEST_SKIP, KEY_FILE_TYPE: FILE_TYPE_SKIP, KEY_EXTENSION: ext, KEY_REASON: 'File extension in skip list'} + + if ext in self.video_extensions: + return {KEY_DESTINATION: DEST_VSS, KEY_FILE_TYPE: FILE_TYPE_VIDEO, KEY_EXTENSION: ext} + + if ext in self.document_extensions: + return {KEY_DESTINATION: DEST_RAG, KEY_FILE_TYPE: FILE_TYPE_DOCUMENT, KEY_EXTENSION: ext} + + if ext in self.image_extensions: + if self.enable_image_processing: + return {KEY_DESTINATION: DEST_RAG, KEY_FILE_TYPE: FILE_TYPE_IMAGE, KEY_EXTENSION: ext} + return {KEY_DESTINATION: DEST_SKIP, KEY_FILE_TYPE: FILE_TYPE_IMAGE, KEY_EXTENSION: ext, KEY_REASON: 'Image processing not enabled'} + + if ext in self.audio_extensions: + if self.enable_audio_processing: + return {KEY_DESTINATION: DEST_RAG, KEY_FILE_TYPE: FILE_TYPE_AUDIO, KEY_EXTENSION: ext} + return {KEY_DESTINATION: DEST_SKIP, KEY_FILE_TYPE: FILE_TYPE_AUDIO, KEY_EXTENSION: ext, KEY_REASON: 'Audio processing not enabled'} + + return {KEY_DESTINATION: DEST_RAG, KEY_FILE_TYPE: FILE_TYPE_UNKNOWN, KEY_EXTENSION: ext, KEY_REASON: 'Unknown extension, attempting RAG ingestion'} + + def is_video(self, filename: str) -> bool: + return Path(filename).suffix.lower() in self.video_extensions + + def is_document(self, filename: str) -> bool: + return Path(filename).suffix.lower() in self.document_extensions diff --git a/examples/rag_event_ingest/kafka_consumer/services/__init__.py b/examples/rag_event_ingest/kafka_consumer/services/__init__.py new file mode 100644 index 000000000..24e2e98dd --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/services/__init__.py @@ -0,0 +1,8 @@ +# services/__init__.py +"""External service clients.""" + +from .storage import ObjectStorage +from .document_indexer import DocumentIndexer +from .video_analyzer import VideoAnalyzer + +__all__ = ['ObjectStorage', 'DocumentIndexer', 'VideoAnalyzer'] diff --git a/examples/rag_event_ingest/kafka_consumer/services/document_indexer.py b/examples/rag_event_ingest/kafka_consumer/services/document_indexer.py new file mode 100644 index 000000000..55923accb --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/services/document_indexer.py @@ -0,0 +1,298 @@ +# services/document_indexer.py +"""Document indexing service for RAG pipeline.""" + +import json +import logging +import time +from pathlib import Path +from typing import Optional, Tuple +import requests + +from config import ( + API_INGESTOR_DOCUMENTS, + API_INGESTOR_COLLECTIONS, + API_INGESTOR_COLLECTION, + API_INGESTOR_STATUS, + STATUS_PENDING, + STATUS_PROCESSING, + STATUS_FINISHED, + STATUS_FAILED, + TIMEOUT_DEFAULT, + TIMEOUT_MAX_TASK_WAIT, + VIDEO_INDEX_TIMEOUT, + COLLECTION_EMBEDDING_DIMENSION, + COLLECTION_CHUNK_SIZE, + COLLECTION_CHUNK_OVERLAP, + COLLECTION_VIDEO_CHUNK_SIZE, + COLLECTION_VIDEO_CHUNK_OVERLAP, + CONTENT_TYPE_MAP, + DEFAULT_CONTENT_TYPE, + FIELD_COLLECTION_NAME, + FIELD_BLOCKING, + FIELD_SPLIT_OPTIONS, + FIELD_CHUNK_SIZE, + FIELD_CHUNK_OVERLAP, + FIELD_GENERATE_SUMMARY, + FIELD_EMBEDDING_DIMENSION, + FIELD_TASK_ID, + RESP_COLLECTIONS, + RESP_TASK_ID, + RESP_STATE, + RESP_RESULT, + RESP_FAILED_DOCUMENTS, + RESP_VALIDATION_ERRORS, + RESP_MESSAGE, + RESP_ERROR, +) + +logger = logging.getLogger(__name__) + + +class DocumentIndexer: + """Indexes documents in vector store for RAG retrieval.""" + + def __init__(self, base_url: str, timeout: int = 600): + """Initialize document indexer.""" + self.base_url = base_url.rstrip('/') + self.timeout = timeout + self._created_collections: set = set() + + logger.info(f"DocumentIndexer initialized: {self.base_url}") + + def ensure_collection_exists(self, collection_name: str) -> bool: + """Create collection if it doesn't exist.""" + if collection_name in self._created_collections: + return True + + # Check if collection exists + try: + response = requests.get( + f'{self.base_url}{API_INGESTOR_COLLECTIONS}', + timeout=TIMEOUT_DEFAULT + ) + except requests.RequestException as e: + logger.error(f"Error checking collections: {e}") + return False + + if response.status_code == 200: + collections = response.json().get(RESP_COLLECTIONS, []) + if collection_name in collections: + logger.info(f"Collection '{collection_name}' already exists") + self._created_collections.add(collection_name) + return True + + # Create collection + logger.info(f"Creating collection '{collection_name}'...") + try: + create_response = requests.post( + f'{self.base_url}{API_INGESTOR_COLLECTION}', + json={ + FIELD_COLLECTION_NAME: collection_name, + FIELD_EMBEDDING_DIMENSION: COLLECTION_EMBEDDING_DIMENSION, + 'metadata_schema': [] + }, + headers={'Content-Type': 'application/json'}, + timeout=TIMEOUT_DEFAULT + ) + except requests.RequestException as e: + logger.error(f"Error creating collection: {e}") + return False + + if create_response.status_code in [200, 201]: + logger.info(f"✓ Collection '{collection_name}' created") + self._created_collections.add(collection_name) + return True + + logger.error(f"Failed to create collection: {create_response.status_code}") + return False + + def upload( + self, + file_data: bytes, + filename: str, + collection: str, + chunk_size: int = COLLECTION_CHUNK_SIZE, + chunk_overlap: int = COLLECTION_CHUNK_OVERLAP + ) -> Optional[str]: + """Upload document to ingestor server.""" + if not self.ensure_collection_exists(collection): + logger.error("Failed to ensure collection exists") + return None + + content_type = self._get_content_type(filename) + files = {'documents': (filename, file_data, content_type)} + + data_config = { + FIELD_COLLECTION_NAME: collection, + FIELD_BLOCKING: False, + FIELD_SPLIT_OPTIONS: { + FIELD_CHUNK_SIZE: chunk_size, + FIELD_CHUNK_OVERLAP: chunk_overlap + }, + FIELD_GENERATE_SUMMARY: False + } + + logger.info(f"Uploading to collection: {collection}") + try: + response = requests.post( + f'{self.base_url}{API_INGESTOR_DOCUMENTS}', + files=files, + data={'data': json.dumps(data_config)}, + timeout=self.timeout + ) + except requests.RequestException as e: + logger.error(f"Error uploading document: {e}") + return None + + if response.status_code in [200, 201, 202]: + result = response.json() + task_id = result.get(RESP_TASK_ID) + if task_id: + logger.info(f"✓ File uploaded, task_id: {task_id}") + return task_id + logger.error("No task_id in response") + return None + + logger.error(f"Upload failed: {response.status_code} - {response.text}") + return None + + def check_status(self, task_id: str, max_wait: int = TIMEOUT_MAX_TASK_WAIT) -> Tuple[bool, str]: + """Check task status and wait for completion.""" + start_time = time.time() + + while time.time() - start_time < max_wait: + try: + response = requests.get( + f'{self.base_url}{API_INGESTOR_STATUS}', + params={FIELD_TASK_ID: task_id}, + timeout=TIMEOUT_DEFAULT + ) + except requests.RequestException as e: + return False, str(e) + + if response.status_code != 200: + return False, f"Status check failed: {response.status_code}" + + result = response.json() + state = result.get(RESP_STATE, 'UNKNOWN') + + if state == STATUS_FAILED: + return False, result.get(RESP_ERROR, 'Unknown error') + + if state == STATUS_FINISHED: + return self._parse_finished_result(result) + + if state in [STATUS_PENDING, STATUS_PROCESSING]: + elapsed = int(time.time() - start_time) + if elapsed % 5 == 0: + logger.info(f"Task {task_id}: {state} ({elapsed}s)") + + time.sleep(1) + + return False, f"Timeout after {max_wait}s" + + def _parse_finished_result(self, result: dict) -> Tuple[bool, str]: + """Parse result from a finished task.""" + task_result = result.get(RESP_RESULT, {}) + failed_docs = task_result.get(RESP_FAILED_DOCUMENTS, []) + validation_errors = task_result.get(RESP_VALIDATION_ERRORS, []) + + if failed_docs or validation_errors: + return False, f"Failed: {failed_docs}, Errors: {validation_errors}" + return True, task_result.get(RESP_MESSAGE, 'Completed') + + def delete(self, filename: str, collection: str) -> bool: + """Delete document from collection.""" + logger.info(f"Deleting '{filename}' from '{collection}'") + + try: + response = requests.delete( + f'{self.base_url}{API_INGESTOR_DOCUMENTS}', + params={FIELD_COLLECTION_NAME: collection}, + json=[filename], + headers={'Content-Type': 'application/json'}, + timeout=TIMEOUT_DEFAULT + ) + except requests.RequestException as e: + logger.error(f"Error deleting document: {e}") + return False + + if response.status_code in [200, 201, 204]: + logger.info(f"Deleted '{filename}'") + return True + + logger.error(f"Delete failed: {response.status_code}") + return False + + def index_video_description( + self, + collection: str, + video_id: str, + video_name: str, + description: str + ) -> bool: + """Index video description in Milvus.""" + try: + # Ensure collection exists first + if not self.ensure_collection_exists(collection): + logger.error(f"Failed to ensure collection '{collection}' exists") + return False + + doc_filename = f"{video_name}_description.json" + + # Delete existing document first to allow re-indexing + self.delete(doc_filename, collection) + + content = f"""Video Name: {video_name} +Video ID: {video_id} +Source: VSS Analysis + +Description: +{description}""" + + doc_data = { + 'content': content, + 'metadata': { + 'content_type': 'video', + 'video_id': video_id, + 'video_name': video_name, + 'source': 'vss_analysis' + } + } + + files = { + 'documents': (doc_filename, json.dumps(doc_data).encode(), 'application/json') + } + + data_config = { + FIELD_COLLECTION_NAME: collection, + FIELD_BLOCKING: True, + FIELD_SPLIT_OPTIONS: { + FIELD_CHUNK_SIZE: COLLECTION_VIDEO_CHUNK_SIZE, + FIELD_CHUNK_OVERLAP: COLLECTION_VIDEO_CHUNK_OVERLAP + }, + FIELD_GENERATE_SUMMARY: False + } + + response = requests.post( + f'{self.base_url}{API_INGESTOR_DOCUMENTS}', + files=files, + data={'data': json.dumps(data_config)}, + timeout=VIDEO_INDEX_TIMEOUT + ) + + if response.status_code in [200, 201, 202]: + logger.info(f"✓ Video description indexed to '{collection}'") + return True + else: + logger.error(f"Failed to index video description: {response.status_code} - {response.text}") + return False + + except requests.RequestException as e: + logger.error(f"Error indexing video description: {e}") + return False + + def _get_content_type(self, filename: str) -> str: + """Get content type from filename.""" + ext = Path(filename).suffix.lower() + return CONTENT_TYPE_MAP.get(ext, DEFAULT_CONTENT_TYPE) diff --git a/examples/rag_event_ingest/kafka_consumer/services/storage.py b/examples/rag_event_ingest/kafka_consumer/services/storage.py new file mode 100644 index 000000000..8f50a1f7b --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/services/storage.py @@ -0,0 +1,195 @@ +# services/storage.py +"""S3-compatible object storage service.""" + +import io +import json +import logging +from abc import ABC, abstractmethod +from typing import Dict, Optional + +from minio import Minio +from minio.error import S3Error + +from config import ( + MINIO_ENDPOINT, + MINIO_ACCESS_KEY, + MINIO_SECRET_KEY, + MINIO_SECURE, + MINIO_DEFAULT_COLLECTION, + MINIO_SOURCES, + CFG_ENDPOINT, + CFG_ACCESS, + CFG_SECRET, + CFG_SECURE, + CFG_COLLECTION, + CFG_BUCKETS, +) + +logger = logging.getLogger(__name__) + + +# ============================================================================= +# Abstract Interface +# ============================================================================= + +class StorageBackend(ABC): + """Abstract interface for object storage operations. + + Implement this to add new backends (Azure Blob, GCS, etc.) + """ + + @abstractmethod + def download(self, bucket: str, key: str) -> bytes: + """Download file from storage.""" + pass + + @abstractmethod + def upload(self, bucket: str, key: str, data: bytes, content_type: Optional[str] = None) -> None: + """Upload file to storage.""" + pass + + @abstractmethod + def delete(self, bucket: str, key: str) -> None: + """Delete file from storage.""" + pass + + @abstractmethod + def exists(self, bucket: str, key: str) -> bool: + """Check if file exists.""" + pass + + +# ============================================================================= +# S3 Implementation +# ============================================================================= + +class S3Backend(StorageBackend): + """S3-compatible storage (MinIO, AWS S3, Wasabi, etc.).""" + + def __init__(self, client: Minio): + self._client = client + + @classmethod + def create( + cls, + endpoint: str, + access_key: str, + secret_key: str, + secure: bool = False, + ) -> 'S3Backend': + """Factory method to create S3 backend.""" + client = Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=secure) + logger.info(f"Created S3 client: {endpoint}") + return cls(client) + + def download(self, bucket: str, key: str) -> bytes: + response = self._client.get_object(bucket, key) + try: + data = response.read() + finally: + response.close() + response.release_conn() + logger.info(f"Downloaded {bucket}/{key} ({len(data)} bytes)") + return data + + def upload(self, bucket: str, key: str, data: bytes, content_type: Optional[str] = None) -> None: + self._client.put_object( + bucket, key, io.BytesIO(data), + length=len(data), + content_type=content_type or 'application/octet-stream' + ) + logger.info(f"Uploaded {bucket}/{key}") + + def delete(self, bucket: str, key: str) -> None: + self._client.remove_object(bucket, key) + logger.info(f"Deleted {bucket}/{key}") + + def exists(self, bucket: str, key: str) -> bool: + try: + self._client.stat_object(bucket, key) + return True + except S3Error: + return False + + +# ============================================================================= +# Object Storage (Factory + Bucket Mapping) +# ============================================================================= + +class ObjectStorage: + """Object storage with bucket-to-collection mapping. + + Handles single or multiple S3 sources via configuration. + """ + + def __init__(self): + self._backends: Dict[str, StorageBackend] = {} + self._bucket_to_backend: Dict[str, str] = {} + self._bucket_to_collection: Dict[str, str] = {} + self._default_collection = MINIO_DEFAULT_COLLECTION + self._configure() + + def _configure(self): + if MINIO_SOURCES: + self._configure_multi_source(MINIO_SOURCES) + else: + self._configure_single_source() + + def _configure_single_source(self): + logger.info(f"Single S3 mode: {MINIO_ENDPOINT}") + self._backends['default'] = S3Backend.create( + MINIO_ENDPOINT, MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_SECURE + ) + + def _configure_multi_source(self, sources_json: str): + config = json.loads(sources_json) + for name, src in config.items(): + self._configure_source(name, src) + + def _configure_source(self, name: str, src: dict): + """Configure a single S3 source and register its buckets.""" + logger.info(f"Configuring S3 source '{name}': {src[CFG_ENDPOINT]}") + + self._backends[name] = S3Backend.create( + src[CFG_ENDPOINT], + src.get(CFG_ACCESS, MINIO_ACCESS_KEY), + src.get(CFG_SECRET, MINIO_SECRET_KEY), + src.get(CFG_SECURE, False) + ) + + collection = src.get(CFG_COLLECTION, name.replace('-', '_')) + self._register_buckets(name, src.get(CFG_BUCKETS, []), collection) + + def _register_buckets(self, backend_name: str, buckets: list, collection: str): + """Register bucket-to-backend and bucket-to-collection mappings.""" + for bucket in buckets: + self._bucket_to_backend[bucket] = backend_name + self._bucket_to_collection[bucket] = collection + logger.info(f" {bucket} → {collection}") + + def _get_backend(self, bucket: str) -> StorageBackend: + if bucket in self._bucket_to_backend: + return self._backends[self._bucket_to_backend[bucket]] + return next(iter(self._backends.values())) + + def download(self, bucket: str, key: str) -> bytes: + return self._get_backend(bucket).download(bucket, key) + + def get_collection_for_bucket(self, bucket: str) -> str: + """Get collection name for bucket. + + Priority: + 1. Explicit mapping from MINIO_SOURCES config + 2. Default collection from COLLECTION_NAME env var + 3. Fallback: bucket name with hyphens → underscores + """ + # Check explicit mapping first + if bucket in self._bucket_to_collection: + return self._bucket_to_collection[bucket] + + # Use default collection if configured + if self._default_collection: + return self._default_collection + + # Fallback to bucket name conversion + return bucket.replace('-', '_') diff --git a/examples/rag_event_ingest/kafka_consumer/services/video_analyzer.py b/examples/rag_event_ingest/kafka_consumer/services/video_analyzer.py new file mode 100644 index 000000000..af7082550 --- /dev/null +++ b/examples/rag_event_ingest/kafka_consumer/services/video_analyzer.py @@ -0,0 +1,326 @@ +# services/video_analyzer.py +"""Video analysis service using VSS 2.4 API.""" + +import json +import re +import logging +from pathlib import Path +from typing import Optional, Tuple +import requests + +from config import ( + API_VSS_FILES, + API_VSS_SUMMARIZE, + CONTENT_TYPE_MAP, + DEFAULT_CONTENT_TYPE, + VSS_DEFAULT_PROMPT, + VSS_SYSTEM_PROMPT, + VSS_CAPTION_SUMMARIZATION_PROMPT, + VSS_SUMMARY_AGGREGATION_PROMPT, + VSS_CHUNK_DURATION, + VSS_CHUNK_OVERLAP, + VSS_NUM_FRAMES_PER_CHUNK, + VSS_MAX_TOKENS, + VSS_MODEL, + VSS_UPLOAD_TIMEOUT, + VSS_STREAM_ENABLED, + # API fields + FIELD_ID, + FIELD_FILE, + FIELD_MODEL, + FIELD_PROMPT, + FIELD_SYSTEM_PROMPT, + FIELD_MAX_TOKENS, + FIELD_CHUNK_DURATION, + FIELD_CHUNK_OVERLAP_DURATION, + FIELD_NUM_FRAMES_PER_CHUNK, + FIELD_CAPTION_SUMMARIZATION_PROMPT, + FIELD_SUMMARY_AGGREGATION_PROMPT, + FIELD_PURPOSE, + FIELD_MEDIA_TYPE, + FIELD_STREAM, + VALUE_VISION, + VALUE_VIDEO, + RESP_CONTENT, + RESP_RESPONSE, + RESP_TEXT, + RESP_CHOICES, + RESP_MESSAGE, + RESP_DELTA, + RESP_DATA_PREFIX, +) + +logger = logging.getLogger(__name__) + + +class VideoAnalyzer: + """Analyzes videos using VSS 2.4 /files and /summarize APIs.""" + + def __init__(self, base_url: str, timeout: int = 1800): + """Initialize video analyzer. + + Args: + base_url: VSS server URL + timeout: Timeout for summarization (default 30 min for long videos) + """ + self.base_url = base_url.rstrip('/') + self.timeout = timeout + + logger.info(f"VideoAnalyzer initialized (VSS 2.4): {self.base_url}") + + def upload_video(self, video_data: bytes, filename: str) -> Tuple[bool, Optional[str]]: + """Upload video to VSS using /files API. + + Args: + video_data: Video file bytes + filename: Original filename + + Returns: + Tuple of (success, file_id) + """ + file_name_sanitized = self._sanitize_filename(filename) + content_type = self._get_content_type(file_name_sanitized) + + logger.info(f"Uploading video to VSS: {file_name_sanitized} ({len(video_data)} bytes)") + + try: + response = requests.post( + f'{self.base_url}{API_VSS_FILES}', + files={FIELD_FILE: (file_name_sanitized, video_data, content_type)}, + data={FIELD_PURPOSE: VALUE_VISION, FIELD_MEDIA_TYPE: VALUE_VIDEO}, + timeout=VSS_UPLOAD_TIMEOUT + ) + except requests.RequestException as e: + logger.error(f"Error uploading video to VSS: {e}") + return False, None + + if response.status_code not in [200, 201]: + logger.error(f"Upload failed: {response.status_code}") + logger.error(f"Response: {response.text}") + return False, None + + result = response.json() + file_id = result.get(FIELD_ID) + + if not file_id: + logger.error(f"No file ID in response: {result}") + return False, None + + logger.info(f"✓ Video uploaded: {file_name_sanitized} (id: {file_id})") + return True, file_id + + def get_video_description(self, file_id: str) -> Optional[str]: + """Get video description using /summarize API. + + Uses streaming mode for real-time progress updates if enabled. + + Args: + file_id: File ID from upload + + Returns: + Video summary text, or None if failed + """ + if VSS_STREAM_ENABLED: + return self._get_description_streaming(file_id) + return self._get_description_blocking(file_id) + + def _build_summarize_payload(self, file_id: str, stream: bool = False) -> dict: + """Build the payload for summarize request.""" + payload = { + FIELD_ID: file_id, + FIELD_MODEL: VSS_MODEL, + FIELD_PROMPT: VSS_DEFAULT_PROMPT, + FIELD_SYSTEM_PROMPT: VSS_SYSTEM_PROMPT, + FIELD_MAX_TOKENS: VSS_MAX_TOKENS, + FIELD_STREAM: stream, + } + + # Add chunking for long videos + if VSS_CHUNK_DURATION > 0: + payload.update({ + FIELD_CHUNK_DURATION: VSS_CHUNK_DURATION, + FIELD_CHUNK_OVERLAP_DURATION: VSS_CHUNK_OVERLAP, + FIELD_NUM_FRAMES_PER_CHUNK: VSS_NUM_FRAMES_PER_CHUNK, + FIELD_CAPTION_SUMMARIZATION_PROMPT: VSS_CAPTION_SUMMARIZATION_PROMPT, + FIELD_SUMMARY_AGGREGATION_PROMPT: VSS_SUMMARY_AGGREGATION_PROMPT, + }) + + return payload + + def _get_description_blocking(self, file_id: str) -> Optional[str]: + """Get video description using blocking mode.""" + logger.info(f"Requesting summarization (blocking) for file: {file_id}") + + payload = self._build_summarize_payload(file_id, stream=False) + + if VSS_CHUNK_DURATION > 0: + logger.info(f"Using chunked processing: {VSS_CHUNK_DURATION}s chunks, " + f"{VSS_NUM_FRAMES_PER_CHUNK} frames/chunk") + + try: + response = requests.post( + f'{self.base_url}{API_VSS_SUMMARIZE}', + json=payload, + headers={'Content-Type': 'application/json'}, + timeout=self.timeout + ) + except requests.RequestException as e: + logger.error(f"Error requesting video summary: {e}") + return None + + if response.status_code != 200: + logger.error(f"Summarize failed: {response.status_code}") + logger.error(f"Response: {response.text}") + return None + + result = response.json() + content = self._extract_content(result) + + if not content: + logger.warning("No content in summarize response") + return None + + logger.info(f"✓ Got video summary ({len(content)} chars)") + return content + + def _get_description_streaming(self, file_id: str) -> Optional[str]: + """Get video description using streaming mode (Server-Sent Events).""" + logger.info(f"Requesting summarization (streaming) for file: {file_id}") + + payload = self._build_summarize_payload(file_id, stream=True) + + if VSS_CHUNK_DURATION > 0: + logger.info(f"Using chunked processing: {VSS_CHUNK_DURATION}s chunks, " + f"{VSS_NUM_FRAMES_PER_CHUNK} frames/chunk") + + try: + response = requests.post( + f'{self.base_url}{API_VSS_SUMMARIZE}', + json=payload, + headers={'Content-Type': 'application/json'}, + timeout=self.timeout, + stream=True # Enable streaming response + ) + except requests.RequestException as e: + logger.error(f"Error requesting video summary (streaming): {e}") + return None + + if response.status_code != 200: + logger.error(f"Summarize failed: {response.status_code}") + logger.error(f"Response: {response.text}") + return None + + # Collect streaming chunks + content = self._collect_stream_content(response) + + if not content: + logger.warning("No content in streaming response") + return None + + logger.info(f"✓ Got video summary via streaming ({len(content)} chars)") + return content + + def _collect_stream_content(self, response) -> Optional[str]: + """Collect content from streaming response (Server-Sent Events).""" + chunks = [] + chunk_count = 0 + + for line in response.iter_lines(decode_unicode=True): + if not line: + continue + + # SSE format: "data: {...json...}" + if line.startswith(RESP_DATA_PREFIX): + json_str = line[len(RESP_DATA_PREFIX):] + + # Handle [DONE] marker + if json_str.strip() == '[DONE]': + logger.info("Stream completed") + break + + try: + data = json.loads(json_str) + content = self._extract_stream_chunk(data) + if content: + chunks.append(content) + chunk_count += 1 + if chunk_count % 10 == 0: + logger.info(f"Received {chunk_count} chunks...") + elif chunk_count == 0: + # Debug: log first non-content chunk to understand format + logger.debug(f"SSE data (no content): {json_str[:200]}") + except json.JSONDecodeError as e: + logger.debug(f"SSE JSON parse error: {e}, line: {json_str[:100]}") + continue + else: + # Non-data line - could be event type or comment + if chunk_count == 0: + logger.debug(f"SSE non-data line: {line[:100]}") + + if not chunks: + logger.warning("No streaming chunks collected, checking full response") + return None + + logger.info(f"Collected {chunk_count} streaming chunks") + return ''.join(chunks) + + def _extract_stream_chunk(self, data: dict) -> Optional[str]: + """Extract content from a single streaming chunk.""" + choices = data.get(RESP_CHOICES, []) + if choices: + delta = choices[0].get(RESP_DELTA, {}) + content = delta.get(RESP_CONTENT) + if content: + return content + + message = choices[0].get(RESP_MESSAGE, {}) + content = message.get(RESP_CONTENT) + if content: + return content + + if RESP_CONTENT in data: + return data[RESP_CONTENT] + + if RESP_RESPONSE in data: + return data[RESP_RESPONSE] + + if RESP_TEXT in data: + return data[RESP_TEXT] + + return None + + def _extract_content(self, result: dict) -> Optional[str]: + """Extract text content from VSS response. + + returns OpenAI-compatible format: + { + "choices": [{"message": {"content": "..."}}] + } + """ + # Try OpenAI format first + choices = result.get(RESP_CHOICES, []) + if choices: + message = choices[0].get(RESP_MESSAGE, {}) + content = message.get(RESP_CONTENT) + if content: + return content.strip() + + if RESP_CONTENT in result: + return result[RESP_CONTENT].strip() + + if RESP_RESPONSE in result: + return result[RESP_RESPONSE].strip() + + return None + + def _sanitize_filename(self, filename: str) -> str: + """Sanitize filename for VSS compatibility.""" + name = Path(filename).name + file_name_sanitized = re.sub(r'[^A-Za-z0-9_.\-]', '_', name) + file_name_sanitized = re.sub(r'_+', '_', file_name_sanitized) + return file_name_sanitized + + def _get_content_type(self, filename: str) -> str: + """Get content type from filename.""" + ext = Path(filename).suffix.lower() + return CONTENT_TYPE_MAP.get(ext, DEFAULT_CONTENT_TYPE) diff --git a/notebooks/rag_event_ingest.ipynb b/notebooks/rag_event_ingest.ipynb new file mode 100644 index 000000000..372a6cfca --- /dev/null +++ b/notebooks/rag_event_ingest.ipynb @@ -0,0 +1,1224 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Video/Document Continuous Ingestion from Object Storage\n", + "\n", + "## Purpose\n", + "\n", + "This notebook demonstrates an **automated document and video ingestion pipeline** that:\n", + "\n", + "1. Monitors emulated object storage for new uploads via Kafka events\n", + "2. Routes files to appropriate AI services based on file type, currently supports document and video ingestion\n", + "5. Enables RAG Agent for semantic search and contextual Q&A over all ingested content\n", + "\n", + "## What Gets Deployed\n", + "\n", + "1. **NVIDIA RAG** - Document indexing, vector search, and AI-powered Q&A (NIMs, Milvus, Ingestor)\n", + "2. **NVIDIA VSS** - Video understanding and summarization (VLM, LLM NIMs, VSS Engine)\n", + "3. **Continuous Ingestion** - Event-driven ingestion pipeline (Kafka, MinIO, Consumer)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "\n", + "### Hardware\n", + "- **GPU**: 4x RTX PRO 6000 Blackwell or 4x H100\n", + "\n", + "### Software (pre-installed required)\n", + "- Ubuntu 22.04 or later\n", + "- Docker 24.0+ with Docker Compose v2\n", + "- NVIDIA Driver 570+\n", + "- NVIDIA Container Toolkit\n", + "\n", + "### API Keys\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
KeyPurposeHow to Get
NGC_API_KEYDocker login, NIM deploymentsNGC Portal → Generate API Key
HF_TOKENDownload VSS modelsHuggingFace Tokens → Create token with Read access
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Table of Contents\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SectionDescription
SetupClone repo, install deps, set API keys, load helpers
Deploy RAGNIMs, Vector DB, Ingestor, RAG Server
Deploy VSSClone VSS, deploy NIMs and VLM
Deploy Continuous IngestionKafka, MinIO, Consumer
TestingUpload documents & videos, query RAG
Clean UpStop services, clean data
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## References\n", + "\n", + "- **RAG Blueprint**: [NVIDIA RAG Documentation](https://github.com/NVIDIA-AI-Blueprints/rag/blob/develop/docs/deploy-docker-self-hosted.md)\n", + "- **VSS**: [Video Search & Summarization Documentation](https://docs.nvidia.com/vss/latest/index.html)\n", + "- **NIM**: [NVIDIA NIM Documentation](https://docs.nvidia.com/nim/index.html)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup\n", + "\n", + "Clone the repository, configure API keys, and load helper functions.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Clone Repository\n", + "\n", + "Clone the RAG Blueprint repo to `~/rag`. This includes the consumer source code, deploy configs, and sample test data.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess, sys, os, shutil\n", + "\n", + "RAG_REPO_DIR = os.path.expanduser(\"~/rag\")\n", + "RAG_REPO_URL = \"https://github.com/NVIDIA-AI-Blueprints/rag.git\"\n", + "\n", + "# Ensure git-lfs is installed before any LFS operations\n", + "if not shutil.which(\"git-lfs\"):\n", + " print(\"[INSTALLING] git-lfs...\")\n", + " subprocess.run(\"sudo apt-get update && sudo apt-get install -y git-lfs && git lfs install\", shell=True, check=True)\n", + "else:\n", + " print(\"[OK] git-lfs found\")\n", + "\n", + "# Clone from correct branch (skip if already exists)\n", + "if not os.path.exists(RAG_REPO_DIR):\n", + " subprocess.run(f\"git clone {RAG_REPO_URL} {RAG_REPO_DIR}\", shell=True, check=True)\n", + "else:\n", + " print(f\"[OK] RAG repo already exists: {RAG_REPO_DIR}\")\n", + "subprocess.run(\"git lfs pull\", shell=True, cwd=RAG_REPO_DIR, check=True)\n", + "\n", + "# Verify\n", + "for path in [\"deploy/compose\", \"examples/rag_event_ingest/kafka_consumer\", \"examples/rag_event_ingest/data\"]:\n", + " status = \"[OK]\" if os.path.exists(os.path.join(RAG_REPO_DIR, path)) else \"[MISSING]\"\n", + " print(f\" {status} {path}\")\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Install Dependencies\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! python3 -m ensurepip --upgrade" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Ensure pip is available (some minimal Python installs lack it)\n", + "subprocess.run([sys.executable, \"-m\", \"ensurepip\", \"--upgrade\"], capture_output=True)\n", + "\n", + "def check_install_system_pkg(cmd: str, install_cmd: str):\n", + " if shutil.which(cmd):\n", + " print(f\" [OK] {cmd} found\")\n", + " return True\n", + " print(f\" [INSTALLING] {cmd}...\")\n", + " result = subprocess.run(install_cmd, shell=True, capture_output=True, text=True)\n", + " if result.returncode == 0:\n", + " print(f\" [OK] {cmd} installed\")\n", + " return True\n", + " print(f\" [ERROR] Failed to install {cmd}. Please install manually: {install_cmd}\")\n", + " return False\n", + "\n", + "check_install_system_pkg(\"git\", \"sudo apt-get update && sudo apt-get install -y git\")\n", + "\n", + "# Install Python packages\n", + "subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"minio\", \"aiohttp\", \"requests\", \"python-dotenv\", \"pyyaml\"])\n", + "print(\"[OK] Ready\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Set API Keys\n", + "\n", + "Configure NGC and HuggingFace API keys for NIM deployments and model downloads.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "\n", + "def set_api_key(env_var: str, prompt: str, required: bool = True):\n", + " if os.environ.get(env_var):\n", + " print(f\" [OK] {env_var} already set ({os.environ[env_var][:10]}...)\")\n", + " return True\n", + " key = getpass.getpass(prompt)\n", + " if key:\n", + " os.environ[env_var] = key\n", + " print(f\" [OK] {env_var} set\")\n", + " return True\n", + " if required:\n", + " print(f\" [ERROR] {env_var} is required\")\n", + " return False\n", + " print(f\" [SKIP] {env_var} (optional)\")\n", + " return True\n", + "\n", + "set_api_key(\"NGC_API_KEY\", \"Enter NGC_API_KEY (starts with 'nvapi-'): \", required=True)\n", + "set_api_key(\"HF_TOKEN\", \"Enter HF_TOKEN (optional, press Enter to skip): \", required=False)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Helper Functions\n", + "\n", + "Shared utilities for deployment, file upload, status checks, and RAG queries.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Install dependencies\n", + "import sys\n", + "!{sys.executable} -m pip install -q minio aiohttp requests python-dotenv" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os, sys, json, re, subprocess, time, socket, asyncio\n", + "import aiohttp, requests\n", + "from typing import List, Optional, Dict\n", + "\n", + "try:\n", + " from minio import Minio\n", + " from minio.error import S3Error\n", + "except ImportError:\n", + " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"minio\"])\n", + " from minio import Minio\n", + " from minio.error import S3Error\n", + "\n", + "# =============================================================================\n", + "# CONFIGURATION\n", + "# =============================================================================\n", + "\n", + "# Paths relative to RAG repo root\n", + "RAG_REPO_DIR = os.path.expanduser(\"~/rag\")\n", + "EXAMPLE_DIR = os.path.join(RAG_REPO_DIR, \"examples/rag_event_ingest\")\n", + "AIDP_COMPOSE_FILE = os.path.join(EXAMPLE_DIR, \"deploy/docker-compose.yaml\")\n", + "DATA_DIR = os.path.join(EXAMPLE_DIR, \"data\")\n", + "RAG_SERVER_URL = \"http://localhost:8081\"\n", + "INGESTOR_URL = \"http://localhost:8082\"\n", + "\n", + "VSS_DIR = os.path.expanduser(\"~/video-search-and-summarization\")\n", + "VSS_UI_PORT = 9110\n", + "VSS_API_PORT = 8110\n", + "VSS_LLM_PORT = 8107\n", + "VSS_EMBED_PORT = 8106\n", + "VSS_RERANK_PORT = 8105\n", + "LOCAL_NIM_CACHE = os.path.expanduser(\"~/.cache/nim\")\n", + "\n", + "MINIO_ENDPOINT = \"localhost:9201\"\n", + "MINIO_ACCESS_KEY = \"minioadmin\"\n", + "MINIO_SECRET_KEY = \"minioadmin\"\n", + "MINIO_BUCKET = \"aidp-bucket\"\n", + "MINIO_COLLECTION = \"aidp_bucket\"\n", + "MINIO_CONSOLE_PORT = 9211\n", + "\n", + "# =============================================================================\n", + "# SHARED UTILITIES\n", + "# =============================================================================\n", + "\n", + "def run_command(cmd: str, capture: bool = False) -> Optional[str]:\n", + " \"\"\"Execute a shell command and print it.\"\"\"\n", + " print(f\"$ {cmd}\")\n", + " result = subprocess.run(cmd, shell=True, capture_output=capture, text=True)\n", + " return result.stdout if capture else None\n", + "\n", + "def get_host_ip() -> str:\n", + " \"\"\"Get host IP address for external access URLs.\"\"\"\n", + " try:\n", + " s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n", + " s.connect((\"8.8.8.8\", 80))\n", + " ip = s.getsockname()[0]\n", + " s.close()\n", + " return ip\n", + " except OSError:\n", + " return \"localhost\"\n", + "\n", + "def get_minio_client() -> Minio:\n", + " \"\"\"Create MinIO client for AIDP bucket operations.\"\"\"\n", + " return Minio(MINIO_ENDPOINT, access_key=MINIO_ACCESS_KEY, secret_key=MINIO_SECRET_KEY, secure=False)\n", + "\n", + "def upload_file(local_path: str, object_name: Optional[str] = None) -> bool:\n", + " \"\"\"Upload a local file to MinIO AIDP bucket.\"\"\"\n", + " if not os.path.exists(local_path):\n", + " print(f\"[ERROR] File not found: {local_path}\")\n", + " return False\n", + " obj = object_name or os.path.basename(local_path)\n", + " try:\n", + " client = get_minio_client()\n", + " if not client.bucket_exists(MINIO_BUCKET):\n", + " client.make_bucket(MINIO_BUCKET)\n", + " client.fput_object(MINIO_BUCKET, obj, local_path)\n", + " print(f\"[OK] Uploaded: {obj}\")\n", + " return True\n", + " except S3Error as e:\n", + " print(f\"[ERROR] {e}\")\n", + " return False\n", + "\n", + "def verify_file_in_storage(object_name: str, bucket: str = MINIO_BUCKET) -> bool:\n", + " \"\"\"Check if a file exists in MinIO bucket and print verification status.\"\"\"\n", + " try:\n", + " client = get_minio_client()\n", + " stat = client.stat_object(bucket, object_name)\n", + " print(f\"[OK] File verified in storage:\")\n", + " print(f\" Bucket: {bucket}\")\n", + " print(f\" Object: {object_name}\")\n", + " print(f\" Size: {stat.size:,} bytes\")\n", + " print(f\" Modified: {stat.last_modified}\")\n", + " return True\n", + " except S3Error as e:\n", + " print(f\"[ERROR] File not found in storage: {object_name}\")\n", + " print(f\" Error: {e}\")\n", + " return False\n", + "\n", + "def get_consumer_logs(lines: int = 30) -> None:\n", + " \"\"\"Show recent Kafka consumer logs.\"\"\"\n", + " run_command(f\"docker logs kafka-consumer --tail {lines}\")\n", + "\n", + "async def query_rag(question: str, collection: str = None) -> Optional[str]:\n", + " \"\"\"Query RAG system and print the answer.\"\"\"\n", + " coll = collection or MINIO_COLLECTION\n", + " print(f\"Q: {question}\\nCollection: {coll}\\n\" + \"-\" * 40)\n", + "\n", + " payload = {\n", + " \"messages\": [{\"role\": \"user\", \"content\": question}],\n", + " \"use_knowledge_base\": True,\n", + " \"collection_names\": [coll],\n", + " }\n", + " try:\n", + " async with aiohttp.ClientSession() as session:\n", + " async with session.post(\n", + " f\"{RAG_SERVER_URL}/generate\", json=payload,\n", + " timeout=aiohttp.ClientTimeout(total=120),\n", + " ) as resp:\n", + " text = await resp.text()\n", + " # Parse SSE response: extract content from each \"data: {...}\" line\n", + " chunks = []\n", + " for line in text.split(\"\\n\"):\n", + " if not line.startswith(\"data: \") or line[6:] == \"[DONE]\":\n", + " continue\n", + " try:\n", + " msg = json.loads(line[6:]).get(\"choices\", [{}])[0].get(\"message\", {})\n", + " if msg.get(\"content\"):\n", + " chunks.append(msg[\"content\"])\n", + " except json.JSONDecodeError:\n", + " pass\n", + " answer = \"\".join(chunks)\n", + " print(f\"Answer: {answer}\")\n", + " return answer\n", + " except aiohttp.ClientError as e:\n", + " print(f\"[ERROR] {e}\")\n", + " return None\n", + "\n", + "print(f\"[OK] Helpers loaded | Host IP: {get_host_ip()}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deploy NVIDIA RAG\n", + "\n", + "Deploy the NVIDIA RAG: NIMs (LLM, Embedding, Reranker), Milvus vector database, Ingestor server, and RAG server.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ngc_key = os.environ.get(\"NGC_API_KEY\")\n", + "if not ngc_key:\n", + " raise RuntimeError(\"NGC_API_KEY not set! Run the API keys cell first.\")\n", + "\n", + "os.chdir(RAG_REPO_DIR)\n", + "\n", + "# Set env vars needed by docker compose\n", + "os.environ[\"NGC_API_KEY\"] = ngc_key\n", + "os.environ[\"USERID\"] = f\"{os.getuid()}:{os.getgid()}\"\n", + "os.environ[\"COLLECTION_NAME\"] = MINIO_COLLECTION\n", + "\n", + "# Load RAG .env defaults (MODEL_DIRECTORY, etc.)\n", + "from dotenv import load_dotenv\n", + "env_file = os.path.join(RAG_REPO_DIR, \"deploy/compose/.env\")\n", + "if os.path.exists(env_file):\n", + " load_dotenv(env_file, override=False)\n", + "\n", + "# Login to nvcr.io\n", + "subprocess.run(f\"echo {ngc_key} | docker login nvcr.io -u '$oauthtoken' --password-stdin\",\n", + " shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + "\n", + "# Deploy components\n", + "for label, compose_file in [\n", + " (\"NIMs\", \"deploy/compose/nims.yaml\"),\n", + " (\"Vector DB\", \"deploy/compose/vectordb.yaml\"),\n", + "]:\n", + " print(f\"Deploying {label}...\")\n", + " run_command(f\"docker compose -f {compose_file} up -d\")\n", + "\n", + "print(\"Waiting 30s for Milvus...\")\n", + "time.sleep(30)\n", + "\n", + "for label, compose_file in [\n", + " (\"Ingestor\", \"deploy/compose/docker-compose-ingestor-server.yaml\"),\n", + " (\"RAG Server\", \"deploy/compose/docker-compose-rag-server.yaml\"),\n", + "]:\n", + " print(f\"Deploying {label}...\")\n", + " run_command(f\"docker compose -f {compose_file} up -d\")\n", + "\n", + "ip = get_host_ip()\n", + "print(f\"\\nRAG deployed: http://{ip}:8081 (server) | http://{ip}:8082 (ingestor) | http://{ip}:8090 (UI)\")\n", + "print(f\"COLLECTION_NAME: {MINIO_COLLECTION}\")\n", + "print(\"Wait 2-5 minutes for NIMs to load models, then run the status check cell.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify RAG services are healthy. Wait 2-5 minutes for NIMs to load models.\n", + "\n", + "The deployment status should be:\n", + "```\n", + "NAMES STATUS\n", + "rag-frontend Up About a minute\n", + "rag-server Up About a minute\n", + "ingestor-server Up About a minute\n", + "milvus-standalone Up 2 minutes (healthy)\n", + "milvus-etcd Up 2 minutes (healthy)\n", + "milvus-minio Up 2 minutes (healthy)\n", + "nim-llm-ms Up 2 minutes (healthy)\n", + "nemoretriever-embedding-ms Up 2 minutes (healthy)\n", + "nemoretriever-ranking-ms Up 2 minutes (healthy)\n", + "```\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check service status and print access URLs\n", + "print(\"Wait 2-5 minutes for services to become healthy.\")\n", + "print(\"Run this cell again after waiting.\\n\")\n", + "\n", + "ip = get_host_ip()\n", + "for name, port, path in [\n", + " (\"RAG Server\", 8081, \"/health\"), (\"Ingestor\", 8082, \"/health\"),\n", + " (\"Frontend\", 8090, \"/\"), (\"Milvus\", 19530, \"/v1/vector/collections\"),\n", + "]:\n", + " try:\n", + " s = \"[OK]\" if requests.get(f\"http://localhost:{port}{path}\", timeout=10).status_code == 200 else \"[WARN]\"\n", + " except requests.ConnectionError:\n", + " s = \"[DOWN]\"\n", + " except requests.Timeout:\n", + " s = \"[TIMEOUT]\"\n", + " print(f\" {s} {name}: http://{ip}:{port}\")\n", + "run_command(\"docker ps --format 'table {{.Names}}\\t{{.Status}}' | grep -E '(rag|milvus|ingestor|nim|nemoretriever|rerankqa|NAMES)'\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deploy NVIDIA VSS\n", + "\n", + "Deploy the NVIDIA VSS: NIMs (LLM, Embedding, Reranker) and VLM for video analysis.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# VSS deployment configuration\n", + "VSS_REPO_URL = \"https://github.com/NVIDIA-AI-Blueprints/video-search-and-summarization.git\"\n", + "VSS_TAG = \"v2.4.1\" # VSS Blueprint version (must match VIA_IMAGE tag)\n", + "VSS_GPU_DEVICE = 2 # GPU for VSS LLM NIM\n", + "VSS_VLM_GPU_DEVICE = 3 # GPU for VLM (via-server with Cosmos-Reason2)\n", + "\n", + "# Only deploy LLM for VSS; embedding and reranker are shared from RAG stack\n", + "# RAG NIMs on nvidia-rag network: nemoretriever-embedding-ms:8000, nemoretriever-ranking-ms:8000\n", + "NIM_IMAGES = {\n", + " \"vss-llm\": (\"nvcr.io/nim/meta/llama-3.1-8b-instruct:1.12.0\", VSS_LLM_PORT),\n", + "}\n", + "\n", + "ngc_key = os.environ.get(\"NGC_API_KEY\", \"\")\n", + "hf_token = os.environ.get(\"HF_TOKEN\", \"\")\n", + "if not ngc_key:\n", + " raise RuntimeError(\"NGC_API_KEY not set!\")\n", + "\n", + "# Docker login\n", + "subprocess.run(f\"echo {ngc_key} | docker login nvcr.io -u '$oauthtoken' --password-stdin\",\n", + " shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + "\n", + "# Clone VSS repo with correct release tag\n", + "if not os.path.exists(VSS_DIR):\n", + " print(f\"Cloning {VSS_REPO_URL} (tag: {VSS_TAG})...\")\n", + " subprocess.run(f\"git clone --branch {VSS_TAG} --depth 1 {VSS_REPO_URL} {VSS_DIR}\", shell=True)\n", + "else:\n", + " print(f\"[OK] VSS repo exists: {VSS_DIR}\")\n", + "\n", + "# Deploy VSS LLM NIM container\n", + "os.makedirs(LOCAL_NIM_CACHE, exist_ok=True)\n", + "for name, (image, port) in NIM_IMAGES.items():\n", + " subprocess.run(f\"docker rm -f {name} 2>/dev/null\", shell=True, capture_output=True)\n", + " cmd = f\"\"\"docker run -d --name {name} \\\n", + " -u $(id -u) --gpus '\"device={VSS_GPU_DEVICE}\"' --shm-size=16GB \\\n", + " --network nvidia-rag -e NGC_API_KEY={ngc_key} \\\n", + " -v \"{LOCAL_NIM_CACHE}:/opt/nim/.cache\" \\\n", + " -p {port}:8000 -e NIM_LOW_MEMORY_MODE=1 -e NIM_RELAX_MEM_CONSTRAINTS=1 \\\n", + " {image}\"\"\"\n", + " result = subprocess.run(cmd, shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + " status = \"[OK]\" if result.returncode == 0 else \"[ERROR]\"\n", + " print(f\" {status} {name} -> port {port}\")\n", + "\n", + "# Deploy VSS application (VLM on separate GPU)\n", + "vss_deploy_dir = f\"{VSS_DIR}/deploy/docker/local_deployment_single_gpu\"\n", + "env_content = f\"\"\"NGC_API_KEY={ngc_key}\n", + "HF_TOKEN={hf_token}\n", + "VIA_IMAGE=nvcr.io/nvidia/blueprint/vss-engine:2.4.1\n", + "FRONTEND_PORT={VSS_UI_PORT}\n", + "BACKEND_PORT={VSS_API_PORT}\n", + "MILVUS_DB_HTTP_PORT=19091\n", + "MILVUS_DB_GRPC_PORT=29530\n", + "MINIO_PORT=9002\n", + "MINIO_WEBUI_PORT=9003\n", + "GRAPH_DB_USERNAME=neo4j\n", + "GRAPH_DB_PASSWORD=password\n", + "ARANGO_DB_USERNAME=arangodb\n", + "ARANGO_DB_PASSWORD=password\n", + "CA_RAG_CONFIG=./config.yaml\n", + "GUARDRAILS_CONFIG=./guardrails\n", + "NVIDIA_VISIBLE_DEVICES={VSS_VLM_GPU_DEVICE}\n", + "VLM_MODEL_TO_USE=cosmos-reason2\n", + "MODEL_PATH=git:https://huggingface.co/nvidia/Cosmos-Reason2-8B\n", + "VLLM_GPU_MEMORY_UTILIZATION=0.4\n", + "VLM_MAX_MODEL_LEN=20480\n", + "DISABLE_GUARDRAILS=true\n", + "DISABLE_CV_PIPELINE=true\n", + "ENABLE_AUDIO=false\n", + "\"\"\"\n", + "with open(f\"{vss_deploy_dir}/.env\", \"w\") as f:\n", + " f.write(env_content)\n", + "\n", + "# Patch config.yaml: LLM points to vss-llm, embedding/reranker reuse RAG NIMs\n", + "config_file = f\"{vss_deploy_dir}/config.yaml\"\n", + "if os.path.exists(config_file):\n", + " cfg = open(config_file).read()\n", + " cfg = re.sub(r\"http://[^:]+:8007/v1\", f\"http://host.docker.internal:{VSS_LLM_PORT}/v1\", cfg)\n", + " cfg = re.sub(r\"http://[^:]+:8006/v1\", \"http://host.docker.internal:9080/v1\", cfg)\n", + " cfg = re.sub(r\"http://[^:]+:8005/v1\", \"http://host.docker.internal:1976/v1\", cfg)\n", + " open(config_file, \"w\").write(cfg)\n", + "\n", + "cmd = f\"cd {vss_deploy_dir} && set -a && source .env && set +a && docker compose up -d\"\n", + "subprocess.run(cmd, shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + "\n", + "ip = get_host_ip()\n", + "print(f\"\\nVSS deployed: http://{ip}:{VSS_UI_PORT} (UI) | http://{ip}:{VSS_API_PORT} (API)\")\n", + "print(\"Embedding/Reranker: shared from RAG stack (nemoretriever-embedding-ms, nemoretriever-ranking-ms)\")\n", + "print(\"Wait 2-5 minutes for NIMs to load models, then run the status check cell.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify VSS services are healthy. Wait 2-5 minutes for NIMs to load models.\n", + "\n", + "> **Note**: Embedding and reranker NIMs are shared from the RAG stack (`nemoretriever-embedding-ms`, `nemoretriever-ranking-ms`) — no separate VSS containers needed.\n", + "\n", + "The deployment status should be:\n", + "```\n", + "NAMES STATUS\n", + "local_deployment_single_gpu-via-server-1 Up About a minute\n", + "local_deployment_single_gpu-elasticsearch-1 Up About a minute\n", + "local_deployment_single_gpu-graph-db-1 Up About a minute\n", + "local_deployment_single_gpu-minio-1 Up About a minute\n", + "local_deployment_single_gpu-arango-db-1 Up About a minute\n", + "local_deployment_single_gpu-milvus-standalone-1 Up About a minute (healthy)\n", + "vss-llm Up About a minute\n", + "```\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check service status and print access URLs\n", + "ip = get_host_ip()\n", + "for name, port, path in [\n", + " (\"VSS UI\", VSS_UI_PORT, \"/\"), (\"VSS API\", VSS_API_PORT, \"/\"),\n", + " (\"VSS LLM NIM\", VSS_LLM_PORT, \"/v1/health/ready\"),\n", + " (\"Embedding (shared from RAG)\", 9080, \"/v1/health/ready\"),\n", + " (\"Reranker (shared from RAG)\", 1976, \"/v1/health/ready\"),\n", + "]:\n", + " try:\n", + " requests.get(f\"http://localhost:{port}{path}\", timeout=10)\n", + " s = \"[OK]\"\n", + " except requests.ConnectionError:\n", + " s = \"[DOWN]\"\n", + " except requests.Timeout:\n", + " s = \"[TIMEOUT]\"\n", + " print(f\" {s} {name}: http://{ip}:{port}\")\n", + "run_command(\"docker ps --format 'table {{.Names}}\\t{{.Status}}' | grep -E '(vss|via|local_deployment|NAMES)'\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deploy Continuous Ingestion from emulated object storage\n", + "\n", + "Deploy the Continuous Ingestion: Kafka message broker, MinIO object storage, and Kafka consumer for automated ingestion.\n", + "\n", + "## 1. Configure Video Analysis Prompts\n", + "\n", + "Customize the prompts used by the Kafka consumer when processing videos through VSS.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Consumer prompts for video analysis - customize these for your video content\n", + "# These are passed to the Kafka consumer which sends them to VSS for video summarization\n", + "\n", + "CONSUMER_VSS_PROMPT = \"\"\"Analyze this sports video. TIMESTAMP FORMAT: Convert seconds to MM:SS (40s=00:40, 80s=01:20, 150s=02:30, 200s=03:20). \\\n", + "Describe: key plays, scoring, turnovers, big gains, defensive stops, celebrations.\"\"\"\n", + "\n", + "CONSUMER_VSS_SYSTEM_PROMPT = \"\"\"You are a sports broadcaster. CRITICAL: Convert all timestamps from seconds to MM:SS format. \\\n", + "Examples: 40 seconds = 00:40, 90 seconds = 01:30, 200 seconds = 03:20, 350 seconds = 05:50. Focus on action, field position, execution, and game momentum.\"\"\"\n", + "\n", + "CONSUMER_VSS_CAPTION_SUMMARIZATION_PROMPT = \"\"\"Format: [MM:SS] Play description. CONVERT seconds to MM:SS (40s=00:40, 80s=01:20, 150s=02:30). \\\n", + "Describe: what happened, how it was executed, the result. Be vivid like a TV broadcast.\"\"\"\n", + "\n", + "CONSUMER_VSS_SUMMARY_AGGREGATION_PROMPT = \"\"\"Create game summary with MM:SS timestamps. CONVERT all times: 40s=00:40, 80s=01:20, 200s=03:20, 350s=05:50. \\\n", + "Highlight scoring plays, turnovers, momentum shifts, spectacular plays. Write like a highlight reel narrator - vivid and exciting.\"\"\"\n", + "\n", + "print(\"Consumer VSS prompts configured:\")\n", + "print(f\" VSS_PROMPT: {len(CONSUMER_VSS_PROMPT)} chars\")\n", + "print(f\" VSS_SYSTEM_PROMPT: {len(CONSUMER_VSS_SYSTEM_PROMPT)} chars\")\n", + "print(f\" VSS_CAPTION_SUMMARIZATION_PROMPT: {len(CONSUMER_VSS_CAPTION_SUMMARIZATION_PROMPT)} chars\")\n", + "print(f\" VSS_SUMMARY_AGGREGATION_PROMPT: {len(CONSUMER_VSS_SUMMARY_AGGREGATION_PROMPT)} chars\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Deploy Services\n", + "\n", + "Deploy Kafka, MinIO, and the Kafka consumer with custom prompts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Verify prerequisites\n", + "net_check = subprocess.run(\"docker network inspect nvidia-rag\", shell=True, capture_output=True)\n", + "if net_check.returncode != 0:\n", + " raise RuntimeError(\"nvidia-rag network not found. Deploy RAG first.\")\n", + "\n", + "ngc_key = os.environ.get(\"NGC_API_KEY\", \"\")\n", + "if not ngc_key:\n", + " raise RuntimeError(\"NGC_API_KEY not set!\")\n", + "\n", + "host_ip = get_host_ip()\n", + "\n", + "# Set environment variables for docker compose (including custom prompts)\n", + "os.environ[\"HOST_IP\"] = host_ip\n", + "os.environ[\"VSS_SERVER_URL\"] = f\"http://{host_ip}:{VSS_API_PORT}\"\n", + "os.environ[\"VSS_PROMPT\"] = CONSUMER_VSS_PROMPT\n", + "os.environ[\"VSS_SYSTEM_PROMPT\"] = CONSUMER_VSS_SYSTEM_PROMPT\n", + "os.environ[\"VSS_CAPTION_SUMMARIZATION_PROMPT\"] = CONSUMER_VSS_CAPTION_SUMMARIZATION_PROMPT\n", + "os.environ[\"VSS_SUMMARY_AGGREGATION_PROMPT\"] = CONSUMER_VSS_SUMMARY_AGGREGATION_PROMPT\n", + "\n", + "# Login + pull + build\n", + "subprocess.run(f\"echo {ngc_key} | docker login nvcr.io -u '$oauthtoken' --password-stdin\",\n", + " shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + "\n", + "compose = f\"docker compose -f {AIDP_COMPOSE_FILE}\"\n", + "subprocess.run(f\"{compose} pull --ignore-pull-failures\", shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + "subprocess.run(f\"{compose} up -d --build\", shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + "\n", + "print(f\"Continuous Ingestion deployed:\")\n", + "print(f\" Kafka UI: http://{host_ip}:8080\")\n", + "print(f\" MinIO Console: http://{host_ip}:{MINIO_CONSOLE_PORT}\")\n", + "print(f\" Credentials: minioadmin / minioadmin\")\n", + "print(f\" Custom prompts: ✓ passed to consumer\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify continuous ingestion services are running.\n", + "\n", + "The deployment status should be:\n", + "```\n", + "NAMES STATUS\n", + "kafka-consumer Up About a minute\n", + "aidp-kafka-ui Up About a minute\n", + "aidp-minio-mc Up About a minute\n", + "aidp-minio Up About a minute (healthy)\n", + "kafka Up About a minute (healthy)\n", + "```\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check service status and print access URLs\n", + "ip = get_host_ip()\n", + "for name, port, path in [\n", + " (\"Kafka UI\", 8080, \"/\"),\n", + " (\"MinIO Console\", MINIO_CONSOLE_PORT, \"/\"),\n", + "]:\n", + " try:\n", + " s = \"[OK]\" if requests.get(f\"http://localhost:{port}{path}\", timeout=10).status_code == 200 else \"[WARN]\"\n", + " except requests.ConnectionError:\n", + " s = \"[DOWN]\"\n", + " except requests.Timeout:\n", + " s = \"[TIMEOUT]\"\n", + " print(f\" {s} {name}: http://{ip}:{port}\")\n", + "\n", + "# Check kafka-consumer container status\n", + "result = subprocess.run(\"docker inspect -f '{{.State.Status}}' kafka-consumer 2>/dev/null\",\n", + " shell=True, capture_output=True, text=True)\n", + "status = result.stdout.strip()\n", + "s = \"[OK]\" if status == \"running\" else \"[DOWN]\"\n", + "print(f\" {s} Kafka Consumer: {status or 'not found'}\")\n", + "\n", + "run_command(\"docker ps --format 'table {{.Names}}\\t{{.Status}}' | grep -E '(kafka|minio|NAMES)'\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Testing\n", + "\n", + "Test the deployment by uploading documents and videos, then querying via RAG.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Document Upload\n", + "\n", + "Upload a PDF document to MinIO, which triggers automatic ingestion via Kafka consumer.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.1 Upload to Storage\n", + "\n", + "Upload the document to MinIO object storage.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Sample documents are included in the repo under examples/rag_event_ingest/data/\n", + "pdf_path = os.path.join(DATA_DIR, \"documents\", \"Seahawks-Patriots in Super Bowl LX_ What We Learned from Seattle's 29-13 win.pdf\")\n", + "upload_file(pdf_path, \"Seahawks-Patriots_SuperBowl_LX_Analysis.pdf\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.2 Verify Document Ingestion" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check consumer logs to verify document processing status.\n", + "\n", + "The logs should show the document being picked up and successfully ingested:\n", + "```\n", + "services.document_indexer - INFO - Task ...: PENDING (0s)\n", + "services.document_indexer - INFO - Task ...: PENDING (5s)\n", + "handlers.base - INFO - [DocumentHandler] ✓ Seahawks-Patriots_SuperBowl_LX_Analysis.pdf → SUCCESS\n", + "consumer - INFO - ✓ SUMMARY: Seahawks-Patriots_SuperBowl_LX_Analysis.pdf | Collection: aidp_bucket | Duration: 12.76s | Status: SUCCESS\n", + "```\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Verify file landed in object storage\n", + "verify_file_in_storage(\"Seahawks-Patriots_SuperBowl_LX_Analysis.pdf\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.3 Verify Document Ingestion\n", + "\n", + "Check consumer logs to verify document processing status.\n", + "\n", + "The logs should show the document being picked up and successfully ingested:\n", + "```\n", + "services.document_indexer - INFO - Task ...: PENDING (0s)\n", + "services.document_indexer - INFO - Task ...: PENDING (5s)\n", + "handlers.base - INFO - [DocumentHandler] ✓ Seahawks-Patriots_SuperBowl_LX_Analysis.pdf → SUCCESS\n", + "consumer - INFO - ✓ SUMMARY: Seahawks-Patriots_SuperBowl_LX_Analysis.pdf | Collection: aidp_bucket | Duration: 12.76s | Status: SUCCESS\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check consumer logs for ingestion status\n", + "print(\"Waiting for document processing...\")\n", + "get_consumer_logs(50)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.4 Query Document via RAG\n", + "\n", + "You can query the ingested document either **programmatically** below or via the **RAG Frontend UI**.\n", + "\n", + "> **💡 RAG Frontend**: Open `http://:8090` in your browser for an interactive Q&A interface.\n", + "> Make sure to select the collection **`aidp_bucket`** in the UI.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Query the document\n", + "await query_rag(\"What was the final score and who won Super Bowl LX?\", MINIO_COLLECTION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ask another question about the document.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Query about key takeaways\n", + "await query_rag(\"What were the key lessons learned from Seattle's victory in Super Bowl LX?\", MINIO_COLLECTION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Video Upload\n", + "\n", + "Upload a video to MinIO, which triggers automatic ingestion via Kafka consumer → VSS for video analysis → RAG for indexing.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.1 Upload to Storage\n", + "\n", + "Upload the video to MinIO object storage.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Sample videos are included in the repo under examples/rag_event_ingest/data/\n", + "video_path = os.path.join(DATA_DIR, \"videos\", \"Seattle Seahawks vs New England Patriots - Super Bowl LX Game Highlights.mp4\")\n", + "upload_file(video_path)\n", + "\n", + "print(\"\\nVideo processing takes longer than documents. Check consumer logs for progress.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.2 Verify Video Ingestion\n", + "\n", + "Check consumer logs to verify video processing status.\n", + "\n", + "The logs should show the video being picked up and processed by VSS:\n", + "```\n", + "handlers.video - INFO - [VideoHandler] Processing video: Seattle Seahawks vs New England Patriots - Super Bowl LX Game Highlights.mp4\n", + "services.video_analyzer - INFO - Submitting video to VSS...\n", + "services.video_analyzer - INFO - VSS processing complete\n", + "handlers.base - INFO - [VideoHandler] ✓ Seattle Seahawks...mp4 → SUCCESS\n", + "consumer - INFO - ✓ SUMMARY: Seattle Seahawks...mp4 | Collection: aidp_bucket | Duration: ~120s | Status: SUCCESS\n", + "```\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Verify video landed in object storage\n", + "video_filename = os.path.basename(video_path)\n", + "verify_file_in_storage(video_filename)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.3 Verify Video Ingestion\n", + "\n", + "Check consumer logs to verify video processing status.\n", + "\n", + "The logs should show the video being picked up and processed by VSS:\n", + "```\n", + "handlers.video - INFO - [VideoHandler] Processing video: Seattle Seahawks vs New England Patriots - Super Bowl LX Game Highlights.mp4\n", + "services.video_analyzer - INFO - Submitting video to VSS...\n", + "services.video_analyzer - INFO - VSS processing complete\n", + "handlers.base - INFO - [VideoHandler] ✓ Seattle Seahawks...mp4 → SUCCESS\n", + "consumer - INFO - ✓ SUMMARY: Seattle Seahawks...mp4 | Collection: aidp_bucket | Duration: ~120s | Status: SUCCESS\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check consumer logs for ingestion status\n", + "print(\"Waiting for video processing...\")\n", + "get_consumer_logs(50)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.4 Query Video via RAG\n", + "\n", + "Query the video content.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Query about the video content\n", + "await query_rag(\"Summarize the video content\", MINIO_COLLECTION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Query about a specific time range in the video.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Query about specific time range\n", + "await query_rag(\"What happened between 15:00 and 20:00?\", MINIO_COLLECTION)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Additional query: analyze key defensive plays and turnovers.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Defensive Analysis\n", + "await query_rag(\"Describe the key defensive plays and turnovers that impacted the game outcome.\", MINIO_COLLECTION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Additional query: identify critical momentum-changing plays in the second half.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Momentum Shifts\n", + "await query_rag(\"What were the critical momentum-changing plays in the second half of the game?\", MINIO_COLLECTION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.5 Delete Video and Verify Removal from RAG\n", + "\n", + "Delete the video from MinIO and confirm the ingested content is removed from the RAG collection." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Delete video from MinIO bucket\n", + "video_filename = os.path.basename(video_path)\n", + "try:\n", + " client = get_minio_client()\n", + " client.remove_object(MINIO_BUCKET, video_filename)\n", + " print(f\"[OK] Deleted '{video_filename}' from MinIO bucket '{MINIO_BUCKET}'\")\n", + "except S3Error as e:\n", + " print(f\"[ERROR] Failed to delete: {e}\")\n", + "\n", + "# Wait for Kafka consumer to process the delete event\n", + "print(\"Waiting 15s for consumer to process delete event...\")\n", + "time.sleep(15)\n", + "get_consumer_logs(20)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Verify the video description document was removed from the ingestor\n", + "video_basename = os.path.basename(video_path)\n", + "desc_filename = f\"{video_basename}_description.json\"\n", + "\n", + "resp = requests.get(f\"{INGESTOR_URL}/documents\", params={\"collection_name\": MINIO_COLLECTION}, timeout=30)\n", + "if resp.status_code == 200:\n", + " docs = resp.json().get(\"documents\", [])\n", + " video_docs = [d for d in docs if desc_filename in str(d)]\n", + " if not video_docs:\n", + " print(f\"[OK] '{desc_filename}' removed from collection '{MINIO_COLLECTION}'\")\n", + " else:\n", + " print(f\"[WARN] '{desc_filename}' still found in collection\")\n", + "else:\n", + " print(f\"[INFO] Could not list documents (status {resp.status_code}), checking via query instead\")\n", + "\n", + "print(\"\\nQuerying RAG (answers may come from the PDF document, not the deleted video):\")\n", + "await query_rag(\"Summarize the video content\", MINIO_COLLECTION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Clean Up\n", + "\n", + "Stop all services and clean up ingested data.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Stop RAG Deployment\n", + "\n", + "Stop all RAG services (NIMs, Milvus, Ingestor, RAG server).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "os.chdir(RAG_REPO_DIR)\n", + "for f in [\n", + " \"deploy/compose/docker-compose-rag-server.yaml\",\n", + " \"deploy/compose/docker-compose-ingestor-server.yaml\",\n", + " \"deploy/compose/vectordb.yaml\",\n", + " \"deploy/compose/nims.yaml\",\n", + "]:\n", + " run_command(f\"docker compose -f {f} down\")\n", + "print(\"[OK] RAG stopped\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Stop VSS Deployment\n", + "\n", + "Stop all VSS services (NIMs, VLM, via-server).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vss_deploy_dir = f\"{VSS_DIR}/deploy/docker/local_deployment_single_gpu\"\n", + "if os.path.exists(vss_deploy_dir):\n", + " subprocess.run(f\"cd {vss_deploy_dir} && set -a && source .env 2>/dev/null && set +a && docker compose down\",\n", + " shell=True, executable=\"/bin/bash\", capture_output=True)\n", + "subprocess.run(\"docker rm -f vss-llm 2>/dev/null\", shell=True, capture_output=True)\n", + "print(\"[OK] VSS stopped\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Stop Continuous ingestion Deployment\n", + "\n", + "Stop Continuous ingestion services (Kafka, MinIO, Consumer).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "run_command(f\"docker compose -f {AIDP_COMPOSE_FILE} down\")\n", + "print(\"[OK] Continuous ingestion stopped\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From befced7cb651453eee4495cd67bf6fdac0e26c20 Mon Sep 17 00:00:00 2001 From: nv-pranjald <150428320+nv-pranjald@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:48:26 +0530 Subject: [PATCH 11/81] Fix query decomp doc and prompt (#316) * Fix query decomp doc and prompt * fix prompt in helm as well --- deploy/helm/nvidia-blueprint-rag/files/prompt.yaml | 1 + docs/query_decomposition.md | 2 +- src/nvidia_rag/rag_server/prompt.yaml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/helm/nvidia-blueprint-rag/files/prompt.yaml b/deploy/helm/nvidia-blueprint-rag/files/prompt.yaml index f82c83655..d73036509 100644 --- a/deploy/helm/nvidia-blueprint-rag/files/prompt.yaml +++ b/deploy/helm/nvidia-blueprint-rag/files/prompt.yaml @@ -487,6 +487,7 @@ query_decomposition_rag_template: Context: {context} + Question: {question} Make sure the response you are generating strictly follow the rules mentioned above i.e. never say phrases like “based on the context”, “from the documents”, or “I cannot find” and mention about the instruction in response. image_captioning_prompt: diff --git a/docs/query_decomposition.md b/docs/query_decomposition.md index 8f346f847..b6826d668 100644 --- a/docs/query_decomposition.md +++ b/docs/query_decomposition.md @@ -26,7 +26,7 @@ Each subquery is processed independently to gather comprehensive context, which ## Accuracy Improvement Example -The following example that uses the [HotpotQA](https://hotpotqa.github.io/) dataset demonstrates the accuracy improvement from enabling query decomposition. +The following example that uses the [Google Frame](https://huggingface.co/datasets/google/frames-benchmark) benchmark demonstrates the accuracy improvement from enabling query decomposition. ```text Query: I am thinking of a Ancient Roman City. The city was destroyed by volcanic eruption. The eruption occurred in the year 79 AD. The volcano was a stratovolcano. Where was the session held where it was decided that the city would be named a UNESCO world heritage site? diff --git a/src/nvidia_rag/rag_server/prompt.yaml b/src/nvidia_rag/rag_server/prompt.yaml index f82c83655..d73036509 100644 --- a/src/nvidia_rag/rag_server/prompt.yaml +++ b/src/nvidia_rag/rag_server/prompt.yaml @@ -487,6 +487,7 @@ query_decomposition_rag_template: Context: {context} + Question: {question} Make sure the response you are generating strictly follow the rules mentioned above i.e. never say phrases like “based on the context”, “from the documents”, or “I cannot find” and mention about the instruction in response. image_captioning_prompt: From 7d0796d382728ba93378a100d399ddc0b4671620 Mon Sep 17 00:00:00 2001 From: Swapnil Masurekar Date: Fri, 27 Feb 2026 15:46:25 +0530 Subject: [PATCH 12/81] Upgrade to Nemotron RC NIMs and NV-Ingest 26.1.2 (#371) Signed-off-by: Swapnil Masurekar --- README.md | 10 +- deploy/compose/.env | 10 +- .../docker-compose-ingestor-server.yaml | 10 +- deploy/compose/docker-compose-rag-server.yaml | 8 +- deploy/compose/nims.yaml | 18 +-- deploy/helm/nvidia-blueprint-rag/Chart.yaml | 2 +- deploy/helm/nvidia-blueprint-rag/endpoints.md | 8 +- deploy/helm/nvidia-blueprint-rag/values.yaml | 42 +++---- deploy/workbench/README.md | 2 +- deploy/workbench/compose.yaml | 38 +++--- deploy/workbench/quickstart.ipynb | 32 ++--- docs/accuracy_perf.md | 2 +- .../openapi_schema_rag_server.json | 8 +- docs/change-model.md | 18 +-- docs/debugging.md | 12 +- docs/deploy-docker-self-hosted.md | 12 +- docs/deploy-helm.md | 8 +- docs/mig-deployment.md | 4 +- docs/multi-collection-retrieval.md | 6 +- docs/nv-ingest-standalone.md | 6 +- docs/python-client.md | 16 +-- docs/retrieval-only-deployment.md | 10 +- docs/service-port-gpu-reference.md | 4 +- docs/text_only_ingest.md | 6 +- docs/troubleshooting.md | 8 +- .../store/__tests__/useSettingsStore.test.tsx | 8 +- notebooks/.env_library | 6 +- notebooks/building_rag_vdb_operator.ipynb | 14 +-- notebooks/config.yaml | 4 +- notebooks/image_input.ipynb | 6 +- notebooks/launchable.ipynb | 4 +- notebooks/nb_metadata.ipynb | 28 ++--- notebooks/rag_library_lite_usage.ipynb | 12 +- notebooks/rag_library_usage.ipynb | 16 +-- notebooks/retriever_api_usage.ipynb | 8 +- notebooks/summarization.ipynb | 16 +-- pyproject.toml | 8 +- src/nvidia_rag/utils/configuration.py | 4 +- tests/integration/notebook_test_config.yaml | 118 ++++++++++++++++++ tests/integration/test_cases/library_usage.py | 3 +- .../env_parity_exemptions.yaml | 4 +- .../test_compose_helm_parity.py | 4 +- .../test_rag_server/test_self_reflection.py | 2 +- tests/unit/test_utils/test_configuration.py | 4 +- tests/unit/test_utils/test_reranker.py | 8 +- uv.lock | 20 +-- variables.env | 10 +- 47 files changed, 362 insertions(+), 245 deletions(-) create mode 100644 tests/integration/notebook_test_config.yaml diff --git a/README.md b/README.md index edea2e72a..72c5ca5a3 100644 --- a/README.md +++ b/README.md @@ -105,9 +105,9 @@ This modular design ensures efficient query processing, accurate retrieval of in - [NVIDIA NIM llama-3_2-nv-embedqa-1b-v2](https://build.nvidia.com/nvidia/llama-3_2-nv-embedqa-1b-v2) - [NVIDIA NIM llama-3_2-nv-rerankqa-1b-v2](https://build.nvidia.com/nvidia/llama-3_2-nv-rerankqa-1b-v2) - - [NeMo Retriever Page Elements NIM](https://build.nvidia.com/nvidia/nemoretriever-page-elements-v3) - - [NeMo Retriever Table Structure NIM](https://build.nvidia.com/nvidia/nemoretriever-table-structure-v1) - - [NeMo Retriever Graphic Elements NIM](https://build.nvidia.com/nvidia/nemoretriever-graphic-elements-v1) + - [NeMo Retriever Page Elements NIM](https://build.nvidia.com/nvidia/nemotron-page-elements-v3) + - [NeMo Retriever Table Structure NIM](https://build.nvidia.com/nvidia/nemotron-table-structure-v1) + - [NeMo Retriever Graphic Elements NIM](https://build.nvidia.com/nvidia/nemotron-graphic-elements-v1) - [NeMo Retriever OCR NIM](https://build.nvidia.com/nvidia/nemoretriever-ocr) - Optional NIMs @@ -202,9 +202,9 @@ Use of the models in this blueprint is governed by the [NVIDIA AI Foundation Mod ## Terms of Use This blueprint is governed by the [NVIDIA Agreements | Enterprise Software | NVIDIA Software License Agreement](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-software-license-agreement/) and the [NVIDIA Agreements | Enterprise Software | Product Specific Terms for AI Product](https://www.nvidia.com/en-us/agreements/enterprise-software/product-specific-terms-for-ai-products/). The models are governed by the [NVIDIA Agreements | Enterprise Software | NVIDIA Community Model License](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-community-models-license/) and the [NVIDIA RAG dataset](./data/multimodal/) which is governed by the [NVIDIA Asset License Agreement](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/data/LICENSE.DATA). -The following models that are built with Llama are governed by the Llama 3.2 Community License Agreement: nvidia/llama-3.2-nv-embedqa-1b-v2 and nvidia/llama-3.2-nv-rerankqa-1b-v2 and llama-3.2-nemoretriever-1b-vlm-embed-v1. +The following models that are built with Llama are governed by the Llama 3.2 Community License Agreement: nvidia/llama-nemotron-embed-1b-v2 and nvidia/llama-nemotron-rerank-1b-v2 and llama-3.2-nemoretriever-1b-vlm-embed-v1. ## Additional Information -The [Llama 3.1 Community License Agreement](https://www.llama.com/llama3_1/license/) for the llama-3.1-nemotron-nano-vl-8b-v1, llama-3.1-nemoguard-8b-content-safety and llama-3.1-nemoguard-8b-topic-control models. The [Llama 3.2 Community License Agreement](https://www.llama.com/llama3_2/license/) for the nvidia/llama-3.2-nv-embedqa-1b-v2, nvidia/llama-3.2-nv-rerankqa-1b-v2 and llama-3.2-nemoretriever-1b-vlm-embed-v1 models. The [Llama 3.3 Community License Agreement](https://github.com/meta-llama/llama-models/blob/main/models/llama3_3/LICENSE) for the llama-3.3-nemotron-super-49b-v1.5 models. Built with Llama. Apache 2.0 for NVIDIA Ingest and for the nemoretriever-page-elements-v2, nemoretriever-table-structure-v1, nemoretriever-graphic-elements-v1, paddleocr and nemoretriever-ocr-v1 models. +The [Llama 3.1 Community License Agreement](https://www.llama.com/llama3_1/license/) for the llama-3.1-nemotron-nano-vl-8b-v1, llama-3.1-nemoguard-8b-content-safety and llama-3.1-nemoguard-8b-topic-control models. The [Llama 3.2 Community License Agreement](https://www.llama.com/llama3_2/license/) for the nvidia/llama-nemotron-embed-1b-v2, nvidia/llama-nemotron-rerank-1b-v2 and llama-3.2-nemoretriever-1b-vlm-embed-v1 models. The [Llama 3.3 Community License Agreement](https://github.com/meta-llama/llama-models/blob/main/models/llama3_3/LICENSE) for the llama-3.3-nemotron-super-49b-v1.5 models. Built with Llama. Apache 2.0 for NVIDIA Ingest and for the nemoretriever-page-elements-v2, nemotron-table-structure-v1, nemotron-graphic-elements-v1, paddleocr and nemoretriever-ocr-v1 models. diff --git a/deploy/compose/.env b/deploy/compose/.env index 9f6ccf796..cc80cfaf0 100644 --- a/deploy/compose/.env +++ b/deploy/compose/.env @@ -22,8 +22,8 @@ export NVIDIA_API_KEY=${NGC_API_KEY} export APP_LLM_SERVERURL=nim-llm:8000 export APP_FILTEREXPRESSIONGENERATOR_SERVERURL=nim-llm:8000 export SUMMARY_LLM_SERVERURL=nim-llm:8000 -export APP_EMBEDDINGS_SERVERURL=nemoretriever-embedding-ms:8000/v1 -export APP_RANKING_SERVERURL=nemoretriever-ranking-ms:8000 +export APP_EMBEDDINGS_SERVERURL=nemotron-embedding-ms:8000/v1 +export APP_RANKING_SERVERURL=nemotron-ranking-ms:8000 export OCR_GRPC_ENDPOINT=nemoretriever-ocr:8001 export OCR_HTTP_ENDPOINT=http://nemoretriever-ocr:8000/v1/infer export OCR_INFER_PROTOCOL=grpc @@ -50,11 +50,11 @@ export YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL=grpc # export OCR_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr # export OCR_INFER_PROTOCOL=http # export OCR_MODEL_NAME=scene_text_ensemble -# export YOLOX_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3 +# export YOLOX_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3 # export YOLOX_INFER_PROTOCOL=http -# export YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-graphic-elements-v1 +# export YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1 # export YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL=http -# export YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-table-structure-v1 +# export YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1 # export YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL=http # export APP_QUERYREWRITER_SERVERURL="" # export APP_QUERYREWRITER_MODELNAME="nvidia/llama-3.3-nemotron-super-49b-v1.5" diff --git a/deploy/compose/docker-compose-ingestor-server.yaml b/deploy/compose/docker-compose-ingestor-server.yaml index fec1fb5aa..be15c17dc 100644 --- a/deploy/compose/docker-compose-ingestor-server.yaml +++ b/deploy/compose/docker-compose-ingestor-server.yaml @@ -75,8 +75,8 @@ services: ##===Embedding Model specific configurations=== # url on which embedding model is hosted. If "", Nvidia hosted API is used - APP_EMBEDDINGS_SERVERURL: ${APP_EMBEDDINGS_SERVERURL-"nemoretriever-embedding-ms:8000/v1"} - APP_EMBEDDINGS_MODELNAME: ${APP_EMBEDDINGS_MODELNAME:-nvidia/llama-3.2-nv-embedqa-1b-v2} + APP_EMBEDDINGS_SERVERURL: ${APP_EMBEDDINGS_SERVERURL-"nemotron-embedding-ms:8000/v1"} + APP_EMBEDDINGS_MODELNAME: ${APP_EMBEDDINGS_MODELNAME:-nvidia/llama-nemotron-embed-1b-v2} # For VLM Embedding Model (Nemoretriever-1b-vlm-embed-v1) # APP_EMBEDDINGS_SERVERURL: ${APP_EMBEDDINGS_SERVERURL-"nemotron-vlm-embedding-ms:8000/v1"} # APP_EMBEDDINGS_MODELNAME: ${APP_EMBEDDINGS_MODELNAME:-nvidia/llama-nemotron-embed-vl-1b-v2} @@ -168,7 +168,7 @@ services: - "6379:6379" nv-ingest-ms-runtime: - image: nvcr.io/nvidia/nemo-microservices/nv-ingest:26.1.1 + image: nvcr.io/nvidia/nemo-microservices/nv-ingest:26.1.2 # cpuset: "0-15" # Uncomment to restrict this container to CPU cores 0–15 shm_size: 40gb # Should be at minimum 30% of assigned memory per Ray documentation volumes: @@ -234,13 +234,13 @@ services: - YOLOX_HTTP_ENDPOINT=${YOLOX_HTTP_ENDPOINT:-http://page-elements:8000/v1/infer} - YOLOX_INFER_PROTOCOL=${YOLOX_INFER_PROTOCOL:-grpc} # build.nvidia.com hosted yolox-graphics-elements endpoints. - #- YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-graphic-elements-v1 + #- YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1 #- YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL=http - YOLOX_GRAPHIC_ELEMENTS_GRPC_ENDPOINT=${YOLOX_GRAPHIC_ELEMENTS_GRPC_ENDPOINT:-graphic-elements:8001} - YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT=${YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT:-http://graphic-elements:8000/v1/infer} - YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL=${YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL:-grpc} # build.nvidia.com hosted yolox-table-elements endpoints. - #- YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-table-structure-v1 + #- YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1 #- YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL=http - YOLOX_TABLE_STRUCTURE_GRPC_ENDPOINT=${YOLOX_TABLE_STRUCTURE_GRPC_ENDPOINT:-table-structure:8001} - YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT=${YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT:-http://table-structure:8000/v1/infer} diff --git a/deploy/compose/docker-compose-rag-server.yaml b/deploy/compose/docker-compose-rag-server.yaml index b0c7d25bd..9f7e59a12 100644 --- a/deploy/compose/docker-compose-rag-server.yaml +++ b/deploy/compose/docker-compose-rag-server.yaml @@ -94,8 +94,8 @@ services: ##===Embedding Model specific configurations=== # url on which embedding model is hosted. If "", Nvidia hosted API is used - APP_EMBEDDINGS_SERVERURL: ${APP_EMBEDDINGS_SERVERURL-"nemoretriever-embedding-ms:8000/v1"} - APP_EMBEDDINGS_MODELNAME: ${APP_EMBEDDINGS_MODELNAME:-nvidia/llama-3.2-nv-embedqa-1b-v2} + APP_EMBEDDINGS_SERVERURL: ${APP_EMBEDDINGS_SERVERURL-"nemotron-embedding-ms:8000/v1"} + APP_EMBEDDINGS_MODELNAME: ${APP_EMBEDDINGS_MODELNAME:-nvidia/llama-nemotron-embed-1b-v2} APP_EMBEDDINGS_DIMENSIONS: ${APP_EMBEDDINGS_DIMENSIONS:-2048} # For VLM Embedding Model (Nemoretriever-1b-vlm-embed-v1) # APP_EMBEDDINGS_SERVERURL: ${APP_EMBEDDINGS_SERVERURL-"nemotron-vlm-embedding-ms:8000/v1"} @@ -103,8 +103,8 @@ services: ##===Reranking Model specific configurations=== # url on which ranking model is hosted. If "", Nvidia hosted API is used - APP_RANKING_SERVERURL: ${APP_RANKING_SERVERURL-"nemoretriever-ranking-ms:8000"} - APP_RANKING_MODELNAME: ${APP_RANKING_MODELNAME:-"nvidia/llama-3.2-nv-rerankqa-1b-v2"} + APP_RANKING_SERVERURL: ${APP_RANKING_SERVERURL-"nemotron-ranking-ms:8000"} + APP_RANKING_MODELNAME: ${APP_RANKING_MODELNAME:-"nvidia/llama-nemotron-rerank-1b-v2"} ENABLE_RERANKER: ${ENABLE_RERANKER:-True} # Default score threshold for filtering documents by reranker relevance (0.0 to 1.0) RERANKER_SCORE_THRESHOLD: ${RERANKER_SCORE_THRESHOLD:-${RERANKER_CONFIDENCE_THRESHOLD:-0.0}} diff --git a/deploy/compose/nims.yaml b/deploy/compose/nims.yaml index f376d9a64..261a705bc 100644 --- a/deploy/compose/nims.yaml +++ b/deploy/compose/nims.yaml @@ -31,9 +31,9 @@ services: retries: 100 profiles: ["", "rag"] - nemoretriever-embedding-ms: - container_name: nemoretriever-embedding-ms - image: nvcr.io/nim/nvidia/llama-3.2-nv-embedqa-1b-v2:1.10.1 + nemotron-embedding-ms: + container_name: nemotron-embedding-ms + image: nvcr.io/nvstaging/nim/llama-nemotron-embed-1b-v2:1.12.0-rc.20260216115852-6fba925dcb63c190 volumes: - ${MODEL_DIRECTORY:-./}:/opt/nim/.cache ports: @@ -91,9 +91,9 @@ services: start_period: 10m profiles: ["vlm-embed", "vlm-ingest"] - nemoretriever-ranking-ms: - container_name: nemoretriever-ranking-ms - image: nvcr.io/nim/nvidia/llama-3.2-nv-rerankqa-1b-v2:1.8.0 + nemotron-ranking-ms: + container_name: nemotron-ranking-ms + image: nvcr.io/nvstaging/nim/llama-nemotron-rerank-1b-v2:1.10.0-rc.20260219005237-7e2370934c6f9b9c volumes: - ${MODEL_DIRECTORY:-./}:/opt/nim/.cache ports: @@ -119,7 +119,7 @@ services: profiles: ["", "rag", "vlm-generation"] page-elements: - image: ${YOLOX_IMAGE:-nvcr.io/nim/nvidia/nemoretriever-page-elements-v3}:${YOLOX_TAG:-1.7.0} + image: ${YOLOX_IMAGE:-nvcr.io/nvstaging/nim/nemotron-page-elements-v3}:${YOLOX_TAG:-1.8.0-rc.20260218170155-6b1b2aa9dfefc9c3} shm_size: 16gb ports: - "8000:8000" @@ -157,7 +157,7 @@ services: profiles: ["", "ingest", "vlm-ingest"] graphic-elements: - image: ${YOLOX_GRAPHIC_ELEMENTS_IMAGE:-nvcr.io/nim/nvidia/nemoretriever-graphic-elements-v1}:${YOLOX_GRAPHIC_ELEMENTS_TAG:-1.6.0} + image: ${YOLOX_GRAPHIC_ELEMENTS_IMAGE:-nvcr.io/nvstaging/nim/nemotron-graphic-elements-v1}:${YOLOX_GRAPHIC_ELEMENTS_TAG:-1.8.0-rc.20260218170144-35d5a3f270662864} shm_size: 16gb ports: - "8003:8000" @@ -183,7 +183,7 @@ services: profiles: ["", "ingest", "vlm-ingest"] table-structure: - image: ${YOLOX_TABLE_STRUCTURE_IMAGE:-nvcr.io/nim/nvidia/nemoretriever-table-structure-v1}:${YOLOX_TABLE_STRUCTURE_TAG:-1.6.0} + image: ${YOLOX_TABLE_STRUCTURE_IMAGE:-nvcr.io/nvstaging/nim/nemotron-table-structure-v1}:${YOLOX_TABLE_STRUCTURE_TAG:-1.8.0-rc.20260218170204-cca5cb850146dbbc} shm_size: 16gb ports: - "8006:8000" diff --git a/deploy/helm/nvidia-blueprint-rag/Chart.yaml b/deploy/helm/nvidia-blueprint-rag/Chart.yaml index 639869e53..afe7a5cbd 100644 --- a/deploy/helm/nvidia-blueprint-rag/Chart.yaml +++ b/deploy/helm/nvidia-blueprint-rag/Chart.yaml @@ -4,7 +4,7 @@ dependencies: - condition: nv-ingest.enabled name: nv-ingest repository: https://helm.ngc.nvidia.com/nvidia/nemo-microservices - version: 26.1.1 + version: 26.1.2 - condition: eck-elasticsearch.enabled name: eck-elasticsearch repository: https://helm.elastic.co diff --git a/deploy/helm/nvidia-blueprint-rag/endpoints.md b/deploy/helm/nvidia-blueprint-rag/endpoints.md index 7609053d5..e62b0d1fb 100644 --- a/deploy/helm/nvidia-blueprint-rag/endpoints.md +++ b/deploy/helm/nvidia-blueprint-rag/endpoints.md @@ -24,11 +24,11 @@ This document describes the configurable endpoints used by the RAG server and it ### Embedding Model - **APP_EMBEDDINGS_SERVERURL**: URL for the embedding model service (default: "nemo-retriever-embedding-ms:8000") -- **APP_EMBEDDINGS_MODELNAME**: Name of the embedding model (default: "nvidia/llama-3.2-nv-embedqa-1b-v2") +- **APP_EMBEDDINGS_MODELNAME**: Name of the embedding model (default: "nvidia/llama-nemotron-embed-1b-v2") ### Reranking Model - **APP_RANKING_SERVERURL**: URL for the ranking model service (default: "nemo-retriever-reranking-ms:8000") -- **APP_RANKING_MODELNAME**: Name of the ranking model (default: "nvidia/llama-3.2-nv-rerankqa-1b-v2") +- **APP_RANKING_MODELNAME**: Name of the ranking model (default: "nvidia/llama-nemotron-rerank-1b-v2") ### Reflection Model - **REFLECTION_LLM_SERVERURL**: URL for the reflection LLM service (default: "nim-llm:8000") @@ -42,8 +42,8 @@ This document describes the configurable endpoints used by the RAG server and it ### Model Configuration - **NEXT_PUBLIC_MODEL_NAME**: Name of the LLM model used in the frontend (default: "nvidia/llama-3.3-nemotron-super-49b-v1.5") -- **VITE_EMBEDDING_MODEL**: Name of the embedding model used in the frontend (default: "nvidia/llama-3.2-nv-embedqa-1b-v2") -- **VITE_RERANKER_MODEL**: Name of the reranker model used in the frontend (default: "nvidia/llama-3.2-nv-rerankqa-1b-v2") +- **VITE_EMBEDDING_MODEL**: Name of the embedding model used in the frontend (default: "nvidia/llama-nemotron-embed-1b-v2") +- **VITE_RERANKER_MODEL**: Name of the reranker model used in the frontend (default: "nvidia/llama-nemotron-rerank-1b-v2") ## Monitoring and Tracing Endpoints diff --git a/deploy/helm/nvidia-blueprint-rag/values.yaml b/deploy/helm/nvidia-blueprint-rag/values.yaml index a02fef3e4..b385b5296 100644 --- a/deploy/helm/nvidia-blueprint-rag/values.yaml +++ b/deploy/helm/nvidia-blueprint-rag/values.yaml @@ -183,14 +183,14 @@ envVars: ##===Embedding Model specific configurations=== # URL on which embedding model is hosted. If "", Nvidia hosted API is used - APP_EMBEDDINGS_SERVERURL: "nemoretriever-embedding-ms:8000/v1" - APP_EMBEDDINGS_MODELNAME: "nvidia/llama-3.2-nv-embedqa-1b-v2" + APP_EMBEDDINGS_SERVERURL: "nemotron-embedding-ms:8000/v1" + APP_EMBEDDINGS_MODELNAME: "nvidia/llama-nemotron-embed-1b-v2" APP_EMBEDDINGS_DIMENSIONS: "2048" ##===Reranking Model specific configurations=== # URL on which ranking model is hosted. If "", Nvidia hosted API is used - APP_RANKING_SERVERURL: "nemoretriever-ranking-ms:8000" - APP_RANKING_MODELNAME: "nvidia/llama-3.2-nv-rerankqa-1b-v2" + APP_RANKING_SERVERURL: "nemotron-ranking-ms:8000" + APP_RANKING_MODELNAME: "nvidia/llama-nemotron-rerank-1b-v2" ENABLE_RERANKER: "True" # Default score threshold for filtering documents by reranker relevance (0.0 to 1.0) RERANKER_SCORE_THRESHOLD: "0.0" @@ -349,8 +349,8 @@ ingestor-server: ## APP_EMBEDDINGS_APIKEY and SUMMARY_LLM_APIKEY are loaded from secrets automatically. # === Embeddings Configurations === - APP_EMBEDDINGS_SERVERURL: "nemoretriever-embedding-ms:8000/v1" - APP_EMBEDDINGS_MODELNAME: "nvidia/llama-3.2-nv-embedqa-1b-v2" + APP_EMBEDDINGS_SERVERURL: "nemotron-embedding-ms:8000/v1" + APP_EMBEDDINGS_MODELNAME: "nvidia/llama-nemotron-embed-1b-v2" APP_EMBEDDINGS_DIMENSIONS: "2048" # === NV-Ingest Configurations === @@ -716,10 +716,10 @@ nimOperator: enabled: true replicas: 1 service: - name: "nemoretriever-embedding-ms" + name: "nemotron-embedding-ms" image: - repository: nvcr.io/nim/nvidia/llama-3.2-nv-embedqa-1b-v2 - tag: "1.10.1" + repository: nvcr.io/nvstaging/nim/llama-nemotron-embed-1b-v2 + tag: "1.12.0-rc.20260216115852-6fba925dcb63c190" pullPolicy: IfNotPresent resources: limits: @@ -795,10 +795,10 @@ nimOperator: enabled: true replicas: 1 service: - name: "nemoretriever-ranking-ms" + name: "nemotron-ranking-ms" image: - repository: nvcr.io/nim/nvidia/llama-3.2-nv-rerankqa-1b-v2 - tag: "1.8.0" + repository: nvcr.io/nvstaging/nim/llama-nemotron-rerank-1b-v2 + tag: "1.10.0-rc.20260219005237-7e2370934c6f9b9c" pullPolicy: IfNotPresent resources: limits: @@ -870,7 +870,7 @@ nv-ingest: create: false image: repository: "nvcr.io/nvidia/nemo-microservices/nv-ingest" - tag: "26.1.1" + tag: "26.1.2" resources: limits: nvidia.com/gpu: 0 @@ -896,8 +896,8 @@ nv-ingest: RAY_num_server_call_thread: "1" RAY_worker_num_grpc_internal_threads: "1" - EMBEDDING_NIM_ENDPOINT: "http://nemoretriever-embedding-ms:8000/v1" - EMBEDDING_NIM_MODEL_NAME: "nvidia/llama-3.2-nv-embedqa-1b-v2" + EMBEDDING_NIM_ENDPOINT: "http://nemotron-embedding-ms:8000/v1" + EMBEDDING_NIM_MODEL_NAME: "nvidia/llama-nemotron-embed-1b-v2" MESSAGE_CLIENT_HOST: "rag-redis-master" MESSAGE_CLIENT_PORT: 6379 MESSAGE_CLIENT_TYPE: "redis" @@ -1049,8 +1049,8 @@ nv-ingest: tolerations: [] replicaCount: 1 image: - repository: nvcr.io/nim/nvidia/nemoretriever-graphic-elements-v1 - tag: "1.6.0" + repository: nvcr.io/nvstaging/nim/nemotron-graphic-elements-v1 + tag: "1.8.0-rc.20260218170144-35d5a3f270662864" env: - name: NIM_HTTP_API_PORT value: "8000" @@ -1082,8 +1082,8 @@ nv-ingest: tolerations: [] replicaCount: 1 image: - repository: nvcr.io/nim/nvidia/nemoretriever-page-elements-v3 - tag: "1.7.0" + repository: nvcr.io/nvstaging/nim/nemotron-page-elements-v3 + tag: "1.8.0-rc.20260218170155-6b1b2aa9dfefc9c3" env: - name: NIM_HTTP_API_PORT value: "8000" @@ -1133,8 +1133,8 @@ nv-ingest: tolerations: [] replicaCount: 1 image: - repository: nvcr.io/nim/nvidia/nemoretriever-table-structure-v1 - tag: "1.6.0" + repository: nvcr.io/nvstaging/nim/nemotron-table-structure-v1 + tag: "1.8.0-rc.20260218170204-cca5cb850146dbbc" env: - name: NIM_HTTP_API_PORT value: "8000" diff --git a/deploy/workbench/README.md b/deploy/workbench/README.md index 6d02a360e..179c32ec5 100644 --- a/deploy/workbench/README.md +++ b/deploy/workbench/README.md @@ -75,4 +75,4 @@ Use of the models in this blueprint is governed by the [NVIDIA AI Foundation Mod ## Terms of Use This blueprint is governed by the [NVIDIA Agreements | Enterprise Software | NVIDIA Software License Agreement](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-software-license-agreement/) and the [NVIDIA Agreements | Enterprise Software | Product Specific Terms for AI Product](https://www.nvidia.com/en-us/agreements/enterprise-software/product-specific-terms-for-ai-products/). The models are governed by the [NVIDIA Agreements | Enterprise Software | NVIDIA Community Model License](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-community-models-license/) and the [NVIDIA RAG dataset](https://github.com/NVIDIA-AI-Blueprints/rag/tree/v2.0.0/data/multimodal) which is governed by the [NVIDIA Asset License Agreement](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/data/LICENSE.DATA). -The following models that are built with Llama are governed by the [Llama 3.2 Community License Agreement](https://www.llama.com/llama3_2/license/): nvidia/llama-3.3-nemotron-super-49b-v1, nvidia/llama-3.2-nv-embedqa-1b-v2, and nvidia/llama-3.2-nv-rerankqa-1b-v2. +The following models that are built with Llama are governed by the [Llama 3.2 Community License Agreement](https://www.llama.com/llama3_2/license/): nvidia/llama-3.3-nemotron-super-49b-v1, nvidia/llama-nemotron-embed-1b-v2, and nvidia/llama-nemotron-rerank-1b-v2. diff --git a/deploy/workbench/compose.yaml b/deploy/workbench/compose.yaml index 1e75d07b7..aa9dd5713 100644 --- a/deploy/workbench/compose.yaml +++ b/deploy/workbench/compose.yaml @@ -28,9 +28,9 @@ services: retries: 100 profiles: ["local"] - nemoretriever-embedding-ms: - container_name: nemoretriever-embedding-ms - image: nvcr.io/nim/nvidia/llama-3.2-nv-embedqa-1b-v2:1.10.1 + nemotron-embedding-ms: + container_name: nemotron-embedding-ms + image: nvcr.io/nvstaging/nim/llama-nemotron-embed-1b-v2:1.12.0-rc.20260216115852-6fba925dcb63c190 volumes: - ${MODEL_DIRECTORY:-/tmp}:/opt/nim/.cache ports: @@ -58,9 +58,9 @@ services: start_period: 10m profiles: ["local"] - nemoretriever-ranking-ms: - container_name: nemoretriever-ranking-ms - image: nvcr.io/nim/nvidia/llama-3.2-nv-rerankqa-1b-v2:1.8.0 + nemotron-ranking-ms: + container_name: nemotron-ranking-ms + image: nvcr.io/nvstaging/nim/llama-nemotron-rerank-1b-v2:1.10.0-rc.20260219005237-7e2370934c6f9b9c volumes: - ${MODEL_DIRECTORY:-/tmp}:/opt/nim/.cache ports: @@ -86,7 +86,7 @@ services: profiles: ["local"] page-elements: - image: ${YOLOX_IMAGE:-nvcr.io/nim/nvidia/nemoretriever-page-elements-v3}:${YOLOX_TAG:-1.7.0} + image: ${YOLOX_IMAGE:-nvcr.io/nvstaging/nim/nemotron-page-elements-v3}:${YOLOX_TAG:-1.8.0-rc.20260218170155-6b1b2aa9dfefc9c3} ports: - "8000:8000" - "8001:8001" @@ -122,7 +122,7 @@ services: profiles: ["local"] graphic-elements: - image: ${YOLOX_GRAPHIC_ELEMENTS_IMAGE:-nvcr.io/nim/nvidia/nemoretriever-graphic-elements-v1}:${YOLOX_GRAPHIC_ELEMENTS_TAG:-1.6.0} + image: ${YOLOX_GRAPHIC_ELEMENTS_IMAGE:-nvcr.io/nvstaging/nim/nemotron-graphic-elements-v1}:${YOLOX_GRAPHIC_ELEMENTS_TAG:-1.8.0-rc.20260218170144-35d5a3f270662864} ports: - "8003:8000" - "8004:8001" @@ -147,7 +147,7 @@ services: profiles: ["local"] table-structure: - image: ${YOLOX_TABLE_STRUCTURE_IMAGE:-nvcr.io/nim/nvidia/nemoretriever-table-structure-v1}:${YOLOX_TABLE_STRUCTURE_TAG:-1.6.0} + image: ${YOLOX_TABLE_STRUCTURE_IMAGE:-nvcr.io/nvstaging/nim/nemotron-table-structure-v1}:${YOLOX_TABLE_STRUCTURE_TAG:-1.8.0-rc.20260218170204-cca5cb850146dbbc} ports: - "8006:8000" - "8007:8001" @@ -256,8 +256,8 @@ services: ##===Embedding Model specific configurations=== # url on which embedding model is hosted. If "", Nvidia hosted API is used - APP_EMBEDDINGS_SERVERURL: ${APP_EMBEDDINGS_SERVERURL-"nemoretriever-embedding-ms:8000/v1"} - APP_EMBEDDINGS_MODELNAME: ${APP_EMBEDDINGS_MODELNAME:-nvidia/llama-3.2-nv-embedqa-1b-v2} + APP_EMBEDDINGS_SERVERURL: ${APP_EMBEDDINGS_SERVERURL-"nemotron-embedding-ms:8000/v1"} + APP_EMBEDDINGS_MODELNAME: ${APP_EMBEDDINGS_MODELNAME:-nvidia/llama-nemotron-embed-1b-v2} APP_EMBEDDINGS_DIMENSIONS: ${APP_EMBEDDINGS_DIMENSIONS:-2048} ##===NV-Ingest Connection Configurations======= @@ -333,7 +333,7 @@ services: profiles: ["ingest"] nv-ingest-ms-runtime: - image: nvcr.io/nvidia/nemo-microservices/nv-ingest:26.1.1 + image: nvcr.io/nvidia/nemo-microservices/nv-ingest:26.1.2 # cpuset: "0-15" # Uncomment to restrict this container to CPU cores 0–15 shm_size: 40gb # Should be at minimum 30% of assigned memory per Ray documentation volumes: @@ -399,20 +399,20 @@ services: - REDIS_MORPHEUS_TASK_QUEUE=morpheus_task_queue # Self-hosted redis endpoints. # build.nvidia.com hosted yolox endpoints. - # - YOLOX_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3 + # - YOLOX_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3 # - YOLOX_INFER_PROTOCOL=http - YOLOX_PAGE_IMAGE_FORMAT=JPEG - YOLOX_GRPC_ENDPOINT=${YOLOX_GRPC_ENDPOINT:-page-elements:8001} - YOLOX_HTTP_ENDPOINT=${YOLOX_HTTP_ENDPOINT:-http://page-elements:8000/v1/infer} - YOLOX_INFER_PROTOCOL=${YOLOX_INFER_PROTOCOL:-grpc} # build.nvidia.com hosted yolox-graphics-elements endpoints. - #- YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-graphic-elements-v1 + #- YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1 #- YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL=http - YOLOX_GRAPHIC_ELEMENTS_GRPC_ENDPOINT=${YOLOX_GRAPHIC_ELEMENTS_GRPC_ENDPOINT:-graphic-elements:8001} - YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT=${YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT:-http://graphic-elements:8000/v1/infer} - YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL=${YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL:-grpc} # build.nvidia.com hosted yolox-table-elements endpoints. - #- YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-table-structure-v1 + #- YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1 #- YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL=http - YOLOX_TABLE_STRUCTURE_GRPC_ENDPOINT=${YOLOX_TABLE_STRUCTURE_GRPC_ENDPOINT:-table-structure:8001} - YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT=${YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT:-http://table-structure:8000/v1/infer} @@ -495,13 +495,13 @@ services: ##===Embedding Model specific configurations=== # url on which embedding model is hosted. If "", Nvidia hosted API is used - APP_EMBEDDINGS_SERVERURL: ${APP_EMBEDDINGS_SERVERURL-"nemoretriever-embedding-ms:8000/v1"} - APP_EMBEDDINGS_MODELNAME: ${APP_EMBEDDINGS_MODELNAME:-nvidia/llama-3.2-nv-embedqa-1b-v2} + APP_EMBEDDINGS_SERVERURL: ${APP_EMBEDDINGS_SERVERURL-"nemotron-embedding-ms:8000/v1"} + APP_EMBEDDINGS_MODELNAME: ${APP_EMBEDDINGS_MODELNAME:-nvidia/llama-nemotron-embed-1b-v2} ##===Reranking Model specific configurations=== # url on which ranking model is hosted. If "", Nvidia hosted API is used - APP_RANKING_SERVERURL: ${APP_RANKING_SERVERURL-"nemoretriever-ranking-ms:8000"} - APP_RANKING_MODELNAME: ${APP_RANKING_MODELNAME:-"nvidia/llama-3.2-nv-rerankqa-1b-v2"} + APP_RANKING_SERVERURL: ${APP_RANKING_SERVERURL-"nemotron-ranking-ms:8000"} + APP_RANKING_MODELNAME: ${APP_RANKING_MODELNAME:-"nvidia/llama-nemotron-rerank-1b-v2"} ENABLE_RERANKER: ${ENABLE_RERANKER:-True} ##===VLM Model specific configurations=== diff --git a/deploy/workbench/quickstart.ipynb b/deploy/workbench/quickstart.ipynb index d9a15a71a..00c524aba 100644 --- a/deploy/workbench/quickstart.ipynb +++ b/deploy/workbench/quickstart.ipynb @@ -966,10 +966,10 @@ " \"enable_citations\": True,\n", " \"model\": \"nvidia/llama-3.3-nemotron-super-49b-v1.5\",\n", " \"llm_endpoint\": \"nim-llm:8000\",\n", - " \"embedding_model\": \"nvidia/llama-3.2-nv-embedqa-1b-v2\",\n", - " \"embedding_endpoint\": \"nemoretriever-embedding-ms:8000/v1\",\n", - " \"reranker_model\": \"nvidia/llama-3.2-nv-rerankqa-1b-v2\",\n", - " \"reranker_endpoint\": \"nemoretriever-ranking-ms:8000\",\n", + " \"embedding_model\": \"nvidia/llama-nemotron-embed-1b-v2\",\n", + " \"embedding_endpoint\": \"nemotron-embedding-ms:8000/v1\",\n", + " \"reranker_model\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", + " \"reranker_endpoint\": \"nemotron-ranking-ms:8000\",\n", " \"stop\": [],\n", "}\n", "\n", @@ -1030,10 +1030,10 @@ " \"enable_citations\": True,\n", " \"model\": \"nvidia/llama-3.3-nemotron-super-49b-v1.5\",\n", " \"llm_endpoint\": \"nim-llm:8000\",\n", - " \"embedding_model\": \"nvidia/llama-3.2-nv-embedqa-1b-v2\",\n", - " \"embedding_endpoint\": \"nemoretriever-embedding-ms:8000/v1\",\n", - " \"reranker_model\": \"nvidia/llama-3.2-nv-rerankqa-1b-v2\",\n", - " \"reranker_endpoint\": \"nemoretriever-ranking-ms:8000\",\n", + " \"embedding_model\": \"nvidia/llama-nemotron-embed-1b-v2\",\n", + " \"embedding_endpoint\": \"nemotron-embedding-ms:8000/v1\",\n", + " \"reranker_model\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", + " \"reranker_endpoint\": \"nemotron-ranking-ms:8000\",\n", " \"stop\": [],\n", "}\n", "\n", @@ -1175,10 +1175,10 @@ " ],\n", " \"enable_query_rewriting\": False,\n", " \"enable_reranker\": False,\n", - " \"embedding_model\": \"nvidia/llama-3.2-nv-embedqa-1b-v2\",\n", - " \"embedding_endpoint\": \"nemoretriever-embedding-ms:8000/v1\",\n", - " \"reranker_model\": \"nvidia/llama-3.2-nv-rerankqa-1b-v2\",\n", - " \"reranker_endpoint\": \"nemoretriever-ranking-ms:8000\",\n", + " \"embedding_model\": \"nvidia/llama-nemotron-embed-1b-v2\",\n", + " \"embedding_endpoint\": \"nemotron-embedding-ms:8000/v1\",\n", + " \"reranker_model\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", + " \"reranker_endpoint\": \"nemotron-ranking-ms:8000\",\n", "}\n", "\n", "\n", @@ -1233,10 +1233,10 @@ " ],\n", " \"enable_query_rewriting\": False,\n", " \"enable_reranker\": True,\n", - " \"embedding_model\": \"nvidia/llama-3.2-nv-embedqa-1b-v2\",\n", - " \"embedding_endpoint\": \"nemoretriever-embedding-ms:8000/v1\",\n", - " \"reranker_model\": \"nvidia/llama-3.2-nv-rerankqa-1b-v2\",\n", - " \"reranker_endpoint\": \"nemoretriever-ranking-ms:8000\",\n", + " \"embedding_model\": \"nvidia/llama-nemotron-embed-1b-v2\",\n", + " \"embedding_endpoint\": \"nemotron-embedding-ms:8000/v1\",\n", + " \"reranker_model\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", + " \"reranker_endpoint\": \"nemotron-ranking-ms:8000\",\n", "}\n", "\n", "\n", diff --git a/docs/accuracy_perf.md b/docs/accuracy_perf.md index 8ee491fa6..8367747ae 100644 --- a/docs/accuracy_perf.md +++ b/docs/accuracy_perf.md @@ -30,7 +30,7 @@ Change the setting if you want different behavior. | Name | Default | Description | Advantages | Disadvantages | |----------------------|------------|---------------------|----------------------|--------------------------| -| - `APP_LLM_MODELNAME`
- `APP_EMBEDDINGS_MODELNAME`
- `APP_RANKING_MODELNAME`
| See description | The default models are the following:
- `nvidia/llama-3.3-nemotron-super-49b-v1.5`
- `nvidia/llama-3.2-nv-embedqa-1b-v2`
- `nvidia/llama-3.2-nv-rerankqa-1b-v2`

You can use larger models. For details, refer to [Change the Inference or Embedding Model](change-model.md). | - Higher accuracy with better reasoning and a larger context length.
| - Slower response time.
- Higher inference cost.
- Higher GPU requirement.
| +| - `APP_LLM_MODELNAME`
- `APP_EMBEDDINGS_MODELNAME`
- `APP_RANKING_MODELNAME`
| See description | The default models are the following:
- `nvidia/llama-3.3-nemotron-super-49b-v1.5`
- `nvidia/llama-nemotron-embed-1b-v2`
- `nvidia/llama-nemotron-rerank-1b-v2`

You can use larger models. For details, refer to [Change the Inference or Embedding Model](change-model.md). | - Higher accuracy with better reasoning and a larger context length.
| - Slower response time.
- Higher inference cost.
- Higher GPU requirement.
| | `APP_VECTORSTORE_SEARCHTYPE` | `dense` | Set to `hybrid` to enable hybrid search. For details, refer to [Hybrid Search Support](hybrid_search.md). | - Can provide better retrieval accuracy for domain-specific content.
| - Can induce higher latency for large number of documents.
| | `ENABLE_GUARDRAILS` | `false` | Set to `true` to enable NeMo Guardrails. For details, refer to [Nemo Guardrails Support](nemo-guardrails.md). | - Applies input/output constraints for better safety and consistency.
| - Significant increased processing overhead for additional LLM calls.
- Needs additional GPUs to deploy guardrails-specific models locally.
| | `ENABLE_QUERYREWRITER` | `false` | Set to `true` to enable query rewriting. For details, refer to [Multi-Turn Conversation Support](multiturn.md). | - Enhances retrieval accuracy for multi-turn scenarios by rephrasing the query.
| - Adds an extra LLM call, increasing latency.
| diff --git a/docs/api_reference/openapi_schema_rag_server.json b/docs/api_reference/openapi_schema_rag_server.json index 63bbb4e33..5bcf2ec7d 100644 --- a/docs/api_reference/openapi_schema_rag_server.json +++ b/docs/api_reference/openapi_schema_rag_server.json @@ -707,7 +707,7 @@ "maxLength": 256, "title": "Embedding Model", "description": "Name of the embedding model used for vectorization.", - "default": "nvdev/nvidia/llama-3.2-nv-embedqa-1b-v2" + "default": "nvdev/nvidia/llama-nemotron-embed-1b-v2" }, "embedding_endpoint": { "type": "string", @@ -721,7 +721,7 @@ "maxLength": 256, "title": "Reranker Model", "description": "Name of the reranker model used for ranking results.", - "default": "nvidia/llama-3.2-nv-rerankqa-1b-v2" + "default": "nvidia/llama-nemotron-rerank-1b-v2" }, "reranker_endpoint": { "anyOf": [ @@ -1342,7 +1342,7 @@ "maxLength": 256, "title": "Embedding Model", "description": "Name of the embedding model used for vectorization.", - "default": "nvdev/nvidia/llama-3.2-nv-embedqa-1b-v2" + "default": "nvdev/nvidia/llama-nemotron-embed-1b-v2" }, "embedding_endpoint": { "anyOf": [ @@ -1363,7 +1363,7 @@ "maxLength": 256, "title": "Reranker Model", "description": "Name of the reranker model used for ranking results.", - "default": "nvidia/llama-3.2-nv-rerankqa-1b-v2" + "default": "nvidia/llama-nemotron-rerank-1b-v2" }, "reranker_endpoint": { "anyOf": [ diff --git a/docs/change-model.md b/docs/change-model.md index d0173462a..27aa80e28 100644 --- a/docs/change-model.md +++ b/docs/change-model.md @@ -77,7 +77,7 @@ Always use same embedding model or model having same tokinizers for both ingesti ### Configure Embedding Dimensions -The default embedding model (`nvidia/llama-3.2-nv-embedqa-1b-v2`) uses **2048 dimensions** by default. When changing to a different embedding model, you may need to update the dimensions to match the model's output. +The default embedding model (`nvidia/llama-nemotron-embed-1b-v2`) uses **2048 dimensions** by default. When changing to a different embedding model, you may need to update the dimensions to match the model's output. **Important:** Some embedding models have **fixed output dimensions** and do not accept a `dimensions` parameter. For example, `nvidia/nv-embedqa-e5-v5` always outputs 1024-dimensional embeddings. If you use such a model without configuring the dimensions, you may encounter an error like: @@ -124,13 +124,13 @@ You can specify the model for NVIDIA NIM containers to use in the [nims.yaml](.. image: nvcr.io/nim/: ... - nemoretriever-embedding-ms: - container_name: nemoretriever-embedding-ms + nemotron-embedding-ms: + container_name: nemotron-embedding-ms image: nvcr.io/nim/: - nemoretriever-ranking-ms: - container_name: nemoretriever-ranking-ms + nemotron-ranking-ms: + container_name: nemotron-ranking-ms image: nvcr.io/nim/: ``` @@ -173,11 +173,11 @@ Use this procedure to change models when you are running self-hosted NVIDIA NIM # === Embeddings === APP_EMBEDDINGS_MODELNAME: "" - APP_EMBEDDINGS_SERVERURL: "nemoretriever-embedding-ms:8000/v1" + APP_EMBEDDINGS_SERVERURL: "nemotron-embedding-ms:8000/v1" # === Reranker === APP_RANKING_MODELNAME: "" - APP_RANKING_SERVERURL: "nemoretriever-ranking-ms:8000" + APP_RANKING_SERVERURL: "nemotron-ranking-ms:8000" ``` 3. Configure the NIM microservices that host those models. Replace `:` with the image you selected (format `nvcr.io/nim/:`) in [values.yaml](../deploy/helm/nvidia-blueprint-rag/values.yaml). @@ -215,7 +215,7 @@ Use this procedure to change models when you are running self-hosted NVIDIA NIM enabled: true replicas: 1 service: - name: "nemoretriever-embedding-ms" + name: "nemotron-embedding-ms" image: # nvcr.io/nim/: repository: nvcr.io/nim/ @@ -237,7 +237,7 @@ Use this procedure to change models when you are running self-hosted NVIDIA NIM enabled: true replicas: 1 service: - name: "nemoretriever-ranking-ms" + name: "nemotron-ranking-ms" image: # nvcr.io/nim/: repository: nvcr.io/nim/ diff --git a/docs/debugging.md b/docs/debugging.md index 66a580419..a9c25debd 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -33,7 +33,7 @@ docker logs -f nim-llm-ms watch -n 10 'du -sh ~/.cache/model-cache/' # Check specific container resource usage -docker stats nim-llm-ms nemoretriever-embedding-ms nemoretriever-ranking-ms +docker stats nim-llm-ms nemotron-embedding-ms nemotron-ranking-ms ``` The expected timeline for Docker (Self-Hosted) deployment is the following: @@ -124,12 +124,12 @@ docker ps | grep -E "(ingestor-server|nv-ingest|nemoretriever-embedding|milvus|r milvus-standalone Up 36 minutes (healthy) milvus-minio Up 35 minutes (healthy) milvus-etcd Up 35 minutes (healthy) - nemoretriever-ranking-ms Up 38 minutes (healthy) + nemotron-ranking-ms Up 38 minutes (healthy) compose-page-elements-1 Up 38 minutes compose-nemoretriever-ocr-1 Up 38 minutes compose-graphic-elements-1 Up 38 minutes compose-table-structure-1 Up 38 minutes - nemoretriever-embedding-ms Up 38 minutes (healthy) + nemotron-embedding-ms Up 38 minutes (healthy) nim-llm-ms Up 38 minutes (healthy) ``` @@ -223,7 +223,7 @@ docker logs ingestor-server --tail 100 docker logs nv-ingest-ms-runtime --tail 100 # Check embedding service logs for model issues -docker logs nemoretriever-embedding-ms --tail 100 +docker logs nemotron-embedding-ms --tail 100 ``` ### 2. Common Ingestion Problems and Solutions @@ -245,7 +245,7 @@ docker logs milvus-standalone --tail 50 **Embedding Service Issues:** ```bash # Check embedding service logs -docker logs nemoretriever-embedding-ms --tail 100 +docker logs nemotron-embedding-ms --tail 100 # Verify GPU availability and memory nvidia-smi @@ -288,7 +288,7 @@ docker logs rag-server --tail 100 docker logs nim-llm-ms --tail 100 # Check ranking service logs for reranking errors -docker logs nemoretriever-ranking-ms --tail 100 +docker logs nemotron-ranking-ms --tail 100 ``` ### 2. Common Retrieval Problems and Solutions diff --git a/docs/deploy-docker-self-hosted.md b/docs/deploy-docker-self-hosted.md index 0efe64ea6..6607aac73 100644 --- a/docs/deploy-docker-self-hosted.md +++ b/docs/deploy-docker-self-hosted.md @@ -110,7 +110,7 @@ Use the following procedure to start all containers needed for this blueprint. USERID=$(id -u) docker compose -f deploy/compose/nims.yaml up -d ``` -5. Check the status of the deployment by running the following code. Wait until all services are up and the `nemoretriever-ranking-ms`, `nemoretriever-embedding-ms` and `nim-llm-ms` NIMs are in healthy state before proceeding further. +5. Check the status of the deployment by running the following code. Wait until all services are up and the `nemotron-ranking-ms`, `nemotron-embedding-ms` and `nim-llm-ms` NIMs are in healthy state before proceeding further. ```bash watch -n 2 'docker ps --format "table {{.Names}}\t{{.Status}}"' @@ -121,10 +121,10 @@ Use the following procedure to start all containers needed for this blueprint. NAMES STATUS nim-llm-ms Up 4 minutes (healthy) - nemoretriever-ranking-ms Up 4 minutes (healthy) + nemotron-ranking-ms Up 4 minutes (healthy) compose-graphic-elements-1 Up 4 minutes compose-page-elements-1 Up 4 minutes - nemoretriever-embedding-ms Up 4 minutes (healthy) + nemotron-embedding-ms Up 4 minutes (healthy) compose-nemoretriever-ocr-1 Up 4 minutes compose-table-structure-1 Up 4 minutes ``` @@ -253,10 +253,10 @@ Use the following procedure to start all containers needed for this blueprint. 340bc8210a0d milvus-minio Up 3 minutes (healthy) 0be702b87ad6 milvus-etcd Up 3 minutes (healthy) 62eabf1d9f65 nim-llm-ms Up 10 minutes (healthy) - fe2751bfa734 nemoretriever-ranking-ms Up 10 minutes (healthy) + fe2751bfa734 nemotron-ranking-ms Up 10 minutes (healthy) 7b5ddabf8be7 compose-graphic-elements-1 Up 10 minutes ecfaa5190302 compose-page-elements-1 Up 10 minutes - ea8c7fdf20d1 nemoretriever-embedding-ms Up 10 minutes (healthy) + ea8c7fdf20d1 nemotron-embedding-ms Up 10 minutes (healthy) 6d62008a9b42 compose-nemoretriever-ocr-1 Up 10 minutes 969b9f5c987c compose-table-structure-1 Up 10 minutes ``` @@ -337,7 +337,7 @@ After the first time you deploy the RAG Blueprint successfully, you can consider - For advanced users who need direct filesystem access to extraction results, refer to [Ingestor Server Volume Mounting](mount-ingestor-volume.md). -- A single NVIDIA A100-80GB or H100-80GB, B200 GPU can be used to start non-LLM NIMs (nemoretriever-embedding-ms, nemoretriever-ranking-ms, and ingestion services like page-elements, ocr, graphic-elements, and table-structure) for ingestion and RAG workflows. You can control which GPU is used for each service by setting these environment variables in `deploy/compose/.env` file before launching. For a complete list of all services and their default GPU assignments, see [Service Port and GPU Reference](service-port-gpu-reference.md). +- A single NVIDIA A100-80GB or H100-80GB, B200 GPU can be used to start non-LLM NIMs (nemotron-embedding-ms, nemotron-ranking-ms, and ingestion services like page-elements, ocr, graphic-elements, and table-structure) for ingestion and RAG workflows. You can control which GPU is used for each service by setting these environment variables in `deploy/compose/.env` file before launching. For a complete list of all services and their default GPU assignments, see [Service Port and GPU Reference](service-port-gpu-reference.md). ```bash EMBEDDING_MS_GPU_ID=0 diff --git a/docs/deploy-helm.md b/docs/deploy-helm.md index f0686c6ca..9e7a39217 100644 --- a/docs/deploy-helm.md +++ b/docs/deploy-helm.md @@ -146,11 +146,11 @@ To verify a deployment, use the following procedure. NAME READY STATUS RESTARTS AGE ingestor-server-6cc886bcdf-6rfwm 1/1 Running 0 54m milvus-standalone-7dd5db4755-ctqzg 1/1 Running 0 54m - nemoretriever-embedding-ms-86f75c8f65-dfhd2 1/1 Running 0 39m + nemotron-embedding-ms-86f75c8f65-dfhd2 1/1 Running 0 39m nemoretriever-graphic-elements-v1-67d9d65bdc-ftbkw 1/1 Running 0 33m nemoretriever-ocr-v1-78f56cddb9-f4852 1/1 Running 0 40m nemoretriever-page-elements-v3-56ddcf9b4b-qsg82 1/1 Running 0 49m - nemoretriever-ranking-ms-5ff774889f-fwrlm 1/1 Running 0 40m + nemotron-ranking-ms-5ff774889f-fwrlm 1/1 Running 0 40m nemoretriever-table-structure-v1-696c9f5665-l9sxn 1/1 Running 0 37m nim-llm-7cb9bdcc89-hwpkq 1/1 Running 0 11m nim-llm-cache-job-77hpc 0/1 Completed 0 94s @@ -209,11 +209,11 @@ To verify a deployment, use the following procedure. NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE ingestor-server ClusterIP 10.107.12.217 8082/TCP 54m milvus ClusterIP 10.99.110.203 19530/TCP,9091/TCP 54m - nemoretriever-embedding-ms ClusterIP 10.104.99.15 8000/TCP,8001/TCP 54m + nemotron-embedding-ms ClusterIP 10.104.99.15 8000/TCP,8001/TCP 54m nemoretriever-graphic-elements-v1 ClusterIP 10.96.115.45 8000/TCP,8001/TCP 54m nemoretriever-ocr-v1 ClusterIP 10.100.107.215 8000/TCP,8001/TCP 54m nemoretriever-page-elements-v3 ClusterIP 10.102.237.196 8000/TCP,8001/TCP 54m - nemoretriever-ranking-ms ClusterIP 10.96.114.244 8000/TCP,8001/TCP 54m + nemotron-ranking-ms ClusterIP 10.96.114.244 8000/TCP,8001/TCP 54m nemoretriever-table-structure-v1 ClusterIP 10.107.227.139 8000/TCP,8001/TCP 54m nim-llm ClusterIP 10.104.60.155 8000/TCP,8001/TCP 54m rag-etcd ClusterIP 10.104.74.116 2379/TCP,2380/TCP 54m diff --git a/docs/mig-deployment.md b/docs/mig-deployment.md index 878681713..149c25366 100644 --- a/docs/mig-deployment.md +++ b/docs/mig-deployment.md @@ -233,14 +233,14 @@ You should see output similar to the following. Resource Requested Limit Allocatable Free nvidia.com/mig-1g.10gb (86%) 6.0 (86%) 6.0 7.0 1.0 ├─ milvus-standalone-... 1.0 1.0 -├─ nemoretriever-embedding-ms-... 1.0 1.0 +├─ nemotron-embedding-ms-... 1.0 1.0 ├─ rag-nv-ingest-... 1.0 1.0 ├─ nemoretriever-graphic-elements-v1-... 1.0 1.0 ├─ nemoretriever-page-elements-v3-... 1.0 1.0 └─ nemoretriever-table-structure-v1-... 1.0 1.0 nvidia.com/mig-1g.20gb (100%) 2.0 (100%) 2.0 2.0 0.0 -├─ nemoretriever-ranking-ms-... 1.0 1.0 +├─ nemotron-ranking-ms-... 1.0 1.0 └─ 1.0 1.0 nvidia.com/mig-3g.40gb (100%) 1.0 (100%) 1.0 1.0 0.0 diff --git a/docs/multi-collection-retrieval.md b/docs/multi-collection-retrieval.md index cd80c337f..07d05be12 100644 --- a/docs/multi-collection-retrieval.md +++ b/docs/multi-collection-retrieval.md @@ -38,10 +38,10 @@ The reranker settings are configured in `deploy/compose/docker-compose-rag-serve export ENABLE_RERANKER=True # Set reranker model (default is already configured) -export APP_RANKING_MODELNAME="nvidia/llama-3.2-nv-rerankqa-1b-v2" +export APP_RANKING_MODELNAME="nvidia/llama-nemotron-rerank-1b-v2" # Reranker service URL (default is already configured) -export APP_RANKING_SERVERURL="nemoretriever-ranking-ms:8000" +export APP_RANKING_SERVERURL="nemotron-ranking-ms:8000" ``` ### For Helm Deployment @@ -54,7 +54,7 @@ envVars: ENABLE_RERANKER: "True" # Reranker model name (default is already configured) - APP_RANKING_MODELNAME: "nvidia/llama-3.2-nv-rerankqa-1b-v2" + APP_RANKING_MODELNAME: "nvidia/llama-nemotron-rerank-1b-v2" # Reranker service URL (default is already configured) APP_RANKING_SERVERURL: "nemoretriever-reranking-ms:8000" diff --git a/docs/nv-ingest-standalone.md b/docs/nv-ingest-standalone.md index f09a37970..691a23607 100644 --- a/docs/nv-ingest-standalone.md +++ b/docs/nv-ingest-standalone.md @@ -118,10 +118,10 @@ ingestor = ingestor.split( ) ingestor = ingestor.embed( - # For self-hosted: "http://nemoretriever-embedding-ms:8000/v1" + # For self-hosted: "http://nemotron-embedding-ms:8000/v1" # For cloud (NVIDIA-hosted): "https://integrate.api.nvidia.com/v1" - endpoint_url="http://nemoretriever-embedding-ms:8000/v1", - model_name="nvidia/llama-3.2-nv-embedqa-1b-v2" + endpoint_url="http://nemotron-embedding-ms:8000/v1", + model_name="nvidia/llama-nemotron-embed-1b-v2" ) ingestor = ingestor.vdb_upload( diff --git a/docs/python-client.md b/docs/python-client.md index 73b432eaf..e052b5e91 100644 --- a/docs/python-client.md +++ b/docs/python-client.md @@ -155,12 +155,12 @@ Verify all containers are running and healthy. ```output NAMES STATUS -nemoretriever-ranking-ms Up ... (healthy) +nemotron-ranking-ms Up ... (healthy) compose-page-elements-1 Up ... compose-nemoretriever-ocr-1 Up ... compose-graphic-elements-1 Up ... compose-table-structure-1 Up ... -nemoretriever-embedding-ms Up ... (healthy) +nemotron-embedding-ms Up ... (healthy) nim-llm-ms Up ... (healthy) ``` @@ -176,19 +176,19 @@ nim-llm-ms Up ... (healthy) - os.environ["OCR_INFER_PROTOCOL"] = "http" os.environ["YOLOX_HTTP_ENDPOINT"] = ( - "https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3" + "https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3" ) - os.environ["YOLOX_INFER_PROTOCOL"] = "http" - os.environ["YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT"] = ( - "https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-graphic-elements-v1" + "https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1" ) - os.environ["YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL"] = "http" - os.environ["YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT"] = ( - "https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-table-structure-v1" + "https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1" ) os.environ["YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL"] = "http" @@ -247,7 +247,7 @@ if DEPLOYMENT_MODE == "cloud": config_ingestor.llm.server_url = "" # Empty uses NVIDIA API catalog config_ingestor.summarizer.server_url = "" # Empty uses NVIDIA API catalog else: - config_ingestor.embeddings.server_url = "http://nemoretriever-embedding-ms:8000/v1" + config_ingestor.embeddings.server_url = "http://nemotron-embedding-ms:8000/v1" ingestor = NvidiaRAGIngestor(config=config_ingestor) ``` @@ -357,11 +357,11 @@ from nvidia_rag.utils.configuration import NvidiaRAGConfig # "server_url": "", # }, # "embeddings": { -# "model_name": "nvidia/llama-3.2-nv-embedqa-1b-v2", +# "model_name": "nvidia/llama-nemotron-embed-1b-v2", # "server_url": "https://integrate.api.nvidia.com/v1", # }, # "ranking": { -# "model_name": "nvidia/llama-3.2-nv-rerankqa-1b-v2", +# "model_name": "nvidia/llama-nemotron-rerank-1b-v2", # "server_url": "", # }, # }) diff --git a/docs/retrieval-only-deployment.md b/docs/retrieval-only-deployment.md index 3cfc5d30a..7f7f94475 100644 --- a/docs/retrieval-only-deployment.md +++ b/docs/retrieval-only-deployment.md @@ -88,11 +88,11 @@ Choose one of the following options based on your deployment preference. Instead of starting all NIMs, use the `text-embed` profile to start only the embedding and reranking services: ```bash -USERID=$(id -u) docker compose -f deploy/compose/nims.yaml up -d nemoretriever-ranking-ms nemoretriever-embedding-ms +USERID=$(id -u) docker compose -f deploy/compose/nims.yaml up -d nemotron-ranking-ms nemotron-embedding-ms ``` :::{note} -The `text-embed` profile starts only `nemoretriever-embedding-ms` and `nemoretriever-ranking-ms `, which is sufficient for retrieval operations. The LLM NIM (`nim-llm-ms`) is not started, saving significant GPU memory. +The `text-embed` profile starts only `nemotron-embedding-ms` and `nemotron-ranking-ms `, which is sufficient for retrieval operations. The LLM NIM (`nim-llm-ms`) is not started, saving significant GPU memory. ::: Wait for the services to become healthy: @@ -105,8 +105,8 @@ Expected output: ```output NAMES STATUS -nemoretriever-ranking-ms Up 5 minutes (healthy) -nemoretriever-embedding-ms Up 5 minutes (healthy) +nemotron-ranking-ms Up 5 minutes (healthy) +nemotron-embedding-ms Up 5 minutes (healthy) ``` #### Option B: NVIDIA-Hosted NIMs @@ -308,7 +308,7 @@ This is expected behavior in retrieval-only mode. The `/generate` endpoint requi Check the embedding NIM logs: ```bash -docker logs nemoretriever-embedding-ms +docker logs nemotron-embedding-ms ``` Ensure the model cache directory has proper permissions: diff --git a/docs/service-port-gpu-reference.md b/docs/service-port-gpu-reference.md index 648d1bd32..f95c9925a 100644 --- a/docs/service-port-gpu-reference.md +++ b/docs/service-port-gpu-reference.md @@ -20,9 +20,9 @@ The following table provides a comprehensive reference of all services, their po | Service | Container Name | Host Port(s) | Container Port(s) | Default GPU ID | Environment Variable | Notes | |---------|---------------|--------------|-------------------|----------------|---------------------|-------| | LLM | `nim-llm-ms` | 8999 | 8000 | 1 | `LLM_MS_GPU_ID` | Main language model | -| Embedding | `nemoretriever-embedding-ms` | 9080 | 8000 | 0 | `EMBEDDING_MS_GPU_ID` | Text embeddings | +| Embedding | `nemotron-embedding-ms` | 9080 | 8000 | 0 | `EMBEDDING_MS_GPU_ID` | Text embeddings | | VLM Embedding | `nemotron-vlm-embedding-ms` | 9081 | 8000 | 0 | `VLM_EMBEDDING_MS_GPU_ID` | Vision-language embeddings (opt-in, profile: vlm-embed) | -| Ranking | `nemoretriever-ranking-ms` | 1976 | 8000 | 0 | `RANKING_MS_GPU_ID` | Reranking model | +| Ranking | `nemotron-ranking-ms` | 1976 | 8000 | 0 | `RANKING_MS_GPU_ID` | Reranking model | | VLM | `nemo-vlm-microservice` | 1977 | 8000 | 5 | `VLM_MS_GPU_ID` | Vision-language model (opt-in, profile: vlm-only, vlm-generation) | | Nemotron Parse | `compose-nemotron-parse-1` | 8015, 8016, 8017 | 8000, 8001, 8002 | 1 | `NEMOTRON_PARSE_MS_GPU_ID` | PDF parsing (opt-in, profile: nemotron-parse) | | RIVA ASR | `compose-audio-1` | 8021, 8022 | 50051, 9000 | 0 | `AUDIO_MS_GPU_ID` | Audio speech recognition (opt-in, profile: audio) | diff --git a/docs/text_only_ingest.md b/docs/text_only_ingest.md index 625416090..10cb6e9b1 100644 --- a/docs/text_only_ingest.md +++ b/docs/text_only_ingest.md @@ -43,8 +43,8 @@ You can enable text-only ingestion for the [NVIDIA RAG Blueprint](readme.md). Fo ```output NAMES STATUS - nemoretriever-ranking-ms Up 14 minutes (healthy) - nemoretriever-embedding-ms Up 14 minutes (healthy) + nemotron-ranking-ms Up 14 minutes (healthy) + nemotron-embedding-ms Up 14 minutes (healthy) nim-llm-ms Up 14 minutes (healthy) ``` @@ -70,7 +70,7 @@ In case you are [interacting with cloud hosted models](deploy-docker-nvidia-host export APP_EMBEDDINGS_SERVERURL="" export APP_LLM_SERVERURL="" export APP_RANKING_SERVERURL="" - export YOLOX_HTTP_ENDPOINT="https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3" + export YOLOX_HTTP_ENDPOINT="https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3" export YOLOX_INFER_PROTOCOL="http" ``` ::: diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 319056bf0..485ae84f1 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -87,10 +87,10 @@ During first-time deployments, large models are downloaded without visible progr docker logs -f nim-llm-ms # Monitor embedding service -docker logs -f nemoretriever-embedding-ms +docker logs -f nemotron-embedding-ms # Monitor ranking service -docker logs -f nemoretriever-ranking-ms +docker logs -f nemotron-ranking-ms ``` **Check disk usage to verify download progress:** @@ -105,7 +105,7 @@ watch -n 10 'du -sh ~/.cache/model-cache/' **Check container stats:** ```bash # View resource usage and verify containers are active -docker stats nim-llm-ms nemoretriever-embedding-ms nemoretriever-ranking-ms +docker stats nim-llm-ms nemotron-embedding-ms nemotron-ranking-ms ``` ### Kubernetes/Helm Deployments @@ -367,7 +367,7 @@ Adding this information may impact response accuracy, especially when partial in ## Helm Deployment Issues ### PVCs in Pending state (StorageClass issues) -If NIM Cache PVCs (e.g., `nemoretriever-embedding-ms-cache-pvc`) remain in `Pending` state, check if they are requesting a `storageClassName: default` that does not exist. +If NIM Cache PVCs (e.g., `nemotron-embedding-ms-cache-pvc`) remain in `Pending` state, check if they are requesting a `storageClassName: default` that does not exist. **Fix:** Ensure you have a default storage class. If using `local-path`, you can create an alias: ```yaml apiVersion: storage.k8s.io/v1 diff --git a/frontend/src/store/__tests__/useSettingsStore.test.tsx b/frontend/src/store/__tests__/useSettingsStore.test.tsx index 0d8165473..541451f20 100644 --- a/frontend/src/store/__tests__/useSettingsStore.test.tsx +++ b/frontend/src/store/__tests__/useSettingsStore.test.tsx @@ -52,7 +52,7 @@ const mockHealthResponse: HealthResponse = { status: 'healthy', latency_ms: 40, error: null, - model: 'nvidia/llama-3.2-nv-rerankqa-1b-v2', + model: 'nvidia/llama-nemotron-rerank-1b-v2', message: null, http_status: 200 } @@ -121,7 +121,7 @@ describe('useHealthInitialization', () => { const state = useSettingsStore.getState(); expect(state.model).toBe('meta/llama-3.1-8b-instruct'); expect(state.embeddingModel).toBe('nvidia/nv-embedqa-e5-v5'); - expect(state.rerankerModel).toBe('nvidia/llama-3.2-nv-rerankqa-1b-v2'); + expect(state.rerankerModel).toBe('nvidia/llama-nemotron-rerank-1b-v2'); }); // Verify endpoints are also set @@ -140,7 +140,7 @@ describe('useHealthInitialization', () => { llmEndpoint: 'http://llm:8000', embeddingModel: 'nvidia/nv-embedqa-e5-v5', embeddingEndpoint: 'http://embeddings:8001', - rerankerModel: 'nvidia/llama-3.2-nv-rerankqa-1b-v2', + rerankerModel: 'nvidia/llama-nemotron-rerank-1b-v2', rerankerEndpoint: 'http://reranker:8002' }) ); @@ -167,7 +167,7 @@ describe('useHealthInitialization', () => { expect(state.model).toBe('user-selected-llm-model'); expect(state.embeddingModel).toBe('user-selected-embedding-model'); // Should still populate undefined fields - expect(state.rerankerModel).toBe('nvidia/llama-3.2-nv-rerankqa-1b-v2'); + expect(state.rerankerModel).toBe('nvidia/llama-nemotron-rerank-1b-v2'); }); }); diff --git a/notebooks/.env_library b/notebooks/.env_library index a0d998752..eb5b68eb5 100644 --- a/notebooks/.env_library +++ b/notebooks/.env_library @@ -16,8 +16,8 @@ export MINIO_ACCESSKEY=minioadmin export MINIO_SECRETKEY=minioadmin # === Embedding Model specific configurations === -export APP_EMBEDDINGS_SERVERURL=nemoretriever-embedding-ms:8000/v1 -export APP_EMBEDDINGS_MODELNAME=nvidia/llama-3.2-nv-embedqa-1b-v2 +export APP_EMBEDDINGS_SERVERURL=nemotron-embedding-ms:8000/v1 +export APP_EMBEDDINGS_MODELNAME=nvidia/llama-nemotron-embed-1b-v2 export APP_EMBEDDINGS_DIMENSIONS=2048 # For VLM Embedding Model (Nemoretriever-1b-vlm-embed-v1) # export APP_EMBEDDINGS_SERVERURL=localhost:9081 @@ -85,7 +85,7 @@ export ENABLE_FILTER_GENERATOR=False # === Reranking Model specific configurations === export APP_RANKING_SERVERURL=localhost:1976 -export APP_RANKING_MODELNAME="nvidia/llama-3.2-nv-rerankqa-1b-v2" +export APP_RANKING_MODELNAME="nvidia/llama-nemotron-rerank-1b-v2" export ENABLE_RERANKER=True # === VLM Model specific configurations === diff --git a/notebooks/building_rag_vdb_operator.ipynb b/notebooks/building_rag_vdb_operator.ipynb index fec360d84..1852f1d9c 100644 --- a/notebooks/building_rag_vdb_operator.ipynb +++ b/notebooks/building_rag_vdb_operator.ipynb @@ -360,12 +360,12 @@ "Ensure all the below are running and healthy before proceeding further\n", "```output\n", "NAMES STATUS\n", - "nemoretriever-ranking-ms Up ... (healthy)\n", + "nemotron-ranking-ms Up ... (healthy)\n", "compose-page-elements-1 Up ...\n", "compose-nemoretriever-ocr-1 Up ...\n", "compose-graphic-elements-1 Up ...\n", "compose-table-structure-1 Up ...\n", - "nemoretriever-embedding-ms Up ... (healthy)\n", + "nemotron-embedding-ms Up ... (healthy)\n", "nim-llm-ms Up ... (healthy)\n", "```" ] @@ -390,15 +390,15 @@ "os.environ[\"OCR_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr\"\n", "os.environ[\"OCR_INFER_PROTOCOL\"] = \"http\"\n", "os.environ[\"YOLOX_HTTP_ENDPOINT\"] = (\n", - " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3\"\n", + " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3\"\n", ")\n", "os.environ[\"YOLOX_INFER_PROTOCOL\"] = \"http\"\n", "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT\"] = (\n", - " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-graphic-elements-v1\"\n", + " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1\"\n", ")\n", "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL\"] = \"http\"\n", "os.environ[\"YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT\"] = (\n", - " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-table-structure-v1\"\n", + " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1\"\n", ")\n", "os.environ[\"YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL\"] = \"http\"" ] @@ -2081,13 +2081,13 @@ "\n", "# IMPORTANT: Two different embedding URLs are needed:\n", "# 1. config_ingestor.embeddings.server_url → Used by nv-ingest (runs in Docker)\n", - "# Must use Docker network hostname: nemoretriever-embedding-ms:8000\n", + "# Must use Docker network hostname: nemotron-embedding-ms:8000\n", "# 2. embedding_model for VDB operator → Used for queries (runs locally in notebook)\n", "# Must use localhost: localhost:9080\n", "\n", "if DEPLOYMENT_MODE == \"on_prem\":\n", " # nv-ingest runs inside Docker, needs Docker network hostname\n", - " config_ingestor.embeddings.server_url = \"http://nemoretriever-embedding-ms:8000/v1\"\n", + " config_ingestor.embeddings.server_url = \"http://nemotron-embedding-ms:8000/v1\"\n", "if DEPLOYMENT_MODE == \"cloud\":\n", " config_ingestor.embeddings.server_url = \"https://integrate.api.nvidia.com/v1\"\n", " config_ingestor.llm.server_url = \"\" # Empty uses NVIDIA API catalog\n", diff --git a/notebooks/config.yaml b/notebooks/config.yaml index ae9551877..f5a8eb53b 100644 --- a/notebooks/config.yaml +++ b/notebooks/config.yaml @@ -55,14 +55,14 @@ filter_expression_generator: # Embedding Configuration embeddings: - model_name: "nvidia/llama-3.2-nv-embedqa-1b-v2" # Model for generating text embeddings + model_name: "nvidia/llama-nemotron-embed-1b-v2" # Model for generating text embeddings dimensions: 2048 # Dimensionality of the embedding vectors server_url: "http://localhost:9080/v1" # URL endpoint for embedding service (on-prem NIM default) # api_key: "" # Optional: API key for embeddings (overrides NVIDIA_API_KEY environment variable) # Ranking Configuration ranking: - model_name: "nvidia/llama-3.2-nv-rerankqa-1b-v2" # Model for reranking retrieved documents + model_name: "nvidia/llama-nemotron-rerank-1b-v2" # Model for reranking retrieved documents server_url: "http://localhost:1976" # URL endpoint for reranking service (on-prem NIM default) enable_reranker: true # Enable reranking of retrieved documents before generation # api_key: "" # Optional: API key for reranking (overrides NVIDIA_API_KEY environment variable) diff --git a/notebooks/image_input.ipynb b/notebooks/image_input.ipynb index 622ec849b..4f778a2fa 100644 --- a/notebooks/image_input.ipynb +++ b/notebooks/image_input.ipynb @@ -293,11 +293,11 @@ "os.environ[\"OCR_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr\"\n", "os.environ[\"OCR_INFER_PROTOCOL\"] = \"http\"\n", "os.environ[\"OCR_MODEL_NAME\"] = \"scene_text_ensemble\"\n", - "os.environ[\"YOLOX_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3\"\n", + "os.environ[\"YOLOX_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3\"\n", "os.environ[\"YOLOX_INFER_PROTOCOL\"] = \"http\"\n", - "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-graphic-elements-v1\"\n", + "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1\"\n", "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL\"] = \"http\"\n", - "os.environ[\"YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-table-structure-v1\"\n", + "os.environ[\"YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1\"\n", "os.environ[\"YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL\"] = \"http\"\n", "os.environ[\"APP_NVINGEST_CAPTIONENDPOINTURL\"] = \"https://integrate.api.nvidia.com/v1/chat/completions\"\n", "\n", diff --git a/notebooks/launchable.ipynb b/notebooks/launchable.ipynb index ee56c3447..1bc8e793b 100644 --- a/notebooks/launchable.ipynb +++ b/notebooks/launchable.ipynb @@ -1295,10 +1295,10 @@ "902445432dde milvus-standalone Up 3 minutes\n", "340bc8210a0d milvus-minio Up 3 minutes (healthy)\n", "0be702b87ad6 milvus-etcd Up 3 minutes (healthy)\n", - "fe2751bfa734 nemoretriever-ranking-ms Up 10 minutes (healthy)\n", + "fe2751bfa734 nemotron-ranking-ms Up 10 minutes (healthy)\n", "7b5ddabf8be7 compose-graphic-elements-1 Up 10 minutes\n", "ecfaa5190302 compose-page-elements-1 Up 10 minutes\n", - "ea8c7fdf20d1 nemoretriever-embedding-ms Up 10 minutes (healthy)\n", + "ea8c7fdf20d1 nemotron-embedding-ms Up 10 minutes (healthy)\n", "6d62008a9b42 compose-nemoretriever-ocr-1 Up 10 minutes\n", "969b9f5c987c compose-table-structure-1 Up 10 minutes\n", "```\n", diff --git a/notebooks/nb_metadata.ipynb b/notebooks/nb_metadata.ipynb index ad79dd7c7..39203d07b 100644 --- a/notebooks/nb_metadata.ipynb +++ b/notebooks/nb_metadata.ipynb @@ -951,8 +951,8 @@ " \"enable_reranker\": True,\n", " \"enable_citations\": True,\n", " \"model\": \"nvidia/llama-3.3-nemotron-super-49b-v1.5\",\n", - " \"reranker_model\": \"nvidia/llama-3.2-nv-rerankqa-1b-v2\",\n", - " \"embedding_model\": \"nvidia/llama-3.2-nv-embedqa-1b-v2\",\n", + " \"reranker_model\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", + " \"embedding_model\": \"nvidia/llama-nemotron-embed-1b-v2\",\n", " # Provide url of the model endpoints if deployed elsewhere\n", " # \"llm_endpoint\": \"\",\n", " #\"embedding_endpoint\": \"\",\n", @@ -1004,8 +1004,8 @@ " \"enable_reranker\": True,\n", " \"enable_citations\": True,\n", " \"model\": \"nvidia/llama-3.3-nemotron-super-49b-v1.5\",\n", - " \"reranker_model\": \"nvidia/llama-3.2-nv-rerankqa-1b-v2\",\n", - " \"embedding_model\": \"nvidia/llama-3.2-nv-embedqa-1b-v2\",\n", + " \"reranker_model\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", + " \"embedding_model\": \"nvidia/llama-nemotron-embed-1b-v2\",\n", " # Provide url of the model endpoints if deployed elsewhere\n", " # \"llm_endpoint\": \"\",\n", " #\"embedding_endpoint\": \"\",\n", @@ -1119,8 +1119,8 @@ " \"enable_citations\": True,\n", " \"enable_filter_generator\": False, # Disable to use manual complex filter\n", " \"model\": \"nvidia/llama-3.3-nemotron-super-49b-v1.5\",\n", - " \"reranker_model\": \"nvidia/llama-3.2-nv-rerankqa-1b-v2\",\n", - " \"embedding_model\": \"nvidia/llama-3.2-nv-embedqa-1b-v2\",\n", + " \"reranker_model\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", + " \"embedding_model\": \"nvidia/llama-nemotron-embed-1b-v2\",\n", " \"stop\": [],\n", " \"filter_expr\": '(content_metadata[\"manufacturer\"] like \"%ford%\" and content_metadata[\"rating\"] > 4.0 and content_metadata[\"created_date\"] between \"2020-01-01\" and \"2024-12-31\" and content_metadata[\"is_public\"] == true) or (content_metadata[\"model\"] like \"%edge%\" and content_metadata[\"year\"] >= 2020 and content_metadata[\"tags\"] in [\"technology\", \"safety\", \"latest\"] and content_metadata[\"rating\"] >= 4.0)'\n", "}\n", @@ -1186,8 +1186,8 @@ " \"enable_reranker\": True,\n", " \"enable_citations\": True,\n", " \"model\": \"nvidia/llama-3.3-nemotron-super-49b-v1.5\",\n", - " \"reranker_model\": \"nvidia/llama-3.2-nv-rerankqa-1b-v2\",\n", - " \"embedding_model\": \"nvidia/llama-3.2-nv-embedqa-1b-v2\",\n", + " \"reranker_model\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", + " \"embedding_model\": \"nvidia/llama-nemotron-embed-1b-v2\",\n", " \"stop\": [],\n", " \"filter_expr\": 'array_contains(content_metadata[\"tags\"], \"eco-friendly\")'\n", "}\n", @@ -1257,8 +1257,8 @@ " \"enable_citations\": True,\n", " \"enable_filter_generator\": True, # 🎯 NEW FEATURE - Enable AI filter generation\n", " \"model\": \"nvidia/llama-3.3-nemotron-super-49b-v1.5\",\n", - " \"reranker_model\": \"nvidia/llama-3.2-nv-rerankqa-1b-v2\",\n", - " \"embedding_model\": \"nvidia/llama-3.2-nv-embedqa-1b-v2\",\n", + " \"reranker_model\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", + " \"embedding_model\": \"nvidia/llama-nemotron-embed-1b-v2\",\n", " \"stop\": [],\n", " \"filter_expr\": \"\" # Will be generated automatically by AI\n", "}\n", @@ -1323,8 +1323,8 @@ " \"enable_reranker\": False,\n", " \"enable_citations\": False,\n", " \"model\": \"nvidia/llama-3.3-nemotron-super-49b-v1.5\",\n", - " \"reranker_model\": \"nvidia/llama-3.2-nv-rerankqa-1b-v2\",\n", - " \"embedding_model\": \"nvidia/llama-3.2-nv-embedqa-1b-v2\",\n", + " \"reranker_model\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", + " \"embedding_model\": \"nvidia/llama-nemotron-embed-1b-v2\",\n", " # Provide url of the model endpoints if deployed elsewhere\n", " # \"llm_endpoint\": \"\",\n", " #\"embedding_endpoint\": \"\",\n", @@ -1391,8 +1391,8 @@ " \"enable_citations\": True,\n", " \"enable_filter_generator\": False,\n", " \"model\": \"nvidia/llama-3.3-nemotron-super-49b-v1.5\",\n", - " \"reranker_model\": \"nvidia/llama-3.2-nv-rerankqa-1b-v2\",\n", - " \"embedding_model\": \"nvidia/llama-3.2-nv-embedqa-1b-v2\",\n", + " \"reranker_model\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", + " \"embedding_model\": \"nvidia/llama-nemotron-embed-1b-v2\",\n", " \"stop\": [],\n", " \"filter_expr\": 'content_metadata[\"nonexistent_field\"] == \"value\"' # This will cause an error\n", "}\n", diff --git a/notebooks/rag_library_lite_usage.ipynb b/notebooks/rag_library_lite_usage.ipynb index eb91c80b6..915eb00ca 100644 --- a/notebooks/rag_library_lite_usage.ipynb +++ b/notebooks/rag_library_lite_usage.ipynb @@ -36,7 +36,7 @@ "\n", "Install nv-ingest library using below command - **OR** - Run the cell below if Jupyter notebook is started in the same environment:\n", "```bash\n", - "uv pip install nv-ingest==26.1.1\n", + "uv pip install nv-ingest==26.1.2\n", "```" ] }, @@ -71,7 +71,7 @@ "# !uv pip install ../dist/nvidia_rag-*-py3-none-any.whl[all]\n", "\n", "# Install NV-Ingest library in the same environment to run NV-Ingest pipeline\n", - "!uv pip install nv-ingest==26.1.1" + "!uv pip install nv-ingest==26.1.2" ] }, { @@ -150,15 +150,15 @@ "os.environ[\"OCR_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr\"\n", "os.environ[\"OCR_INFER_PROTOCOL\"] = \"http\"\n", "os.environ[\"YOLOX_HTTP_ENDPOINT\"] = (\n", - " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3\"\n", + " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3\"\n", ")\n", "os.environ[\"YOLOX_INFER_PROTOCOL\"] = \"http\"\n", "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT\"] = (\n", - " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-graphic-elements-v1\"\n", + " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1\"\n", ")\n", "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL\"] = \"http\"\n", "os.environ[\"YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT\"] = (\n", - " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-table-structure-v1\"\n", + " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1\"\n", ")\n", "os.environ[\"YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL\"] = \"http\"" ] @@ -292,7 +292,7 @@ "\n", "config_ingestor = NvidiaRAGConfig.from_yaml(\"config.yaml\")\n", "# You can update the config object to use different models and endpoints like below\n", - "# config_ingestor.embeddings.model_name = \"nvidia/llama-3.2-nv-embedqa-1b-v2\"\n", + "# config_ingestor.embeddings.model_name = \"nvidia/llama-nemotron-embed-1b-v2\"\n", "# config_ingestor.embeddings.server_url = \"https://integrate.api.nvidia.com/v1\"\n", "\n", "# Set config for rag lite library mode\n", diff --git a/notebooks/rag_library_usage.ipynb b/notebooks/rag_library_usage.ipynb index 83fb0f3b0..894e82620 100644 --- a/notebooks/rag_library_usage.ipynb +++ b/notebooks/rag_library_usage.ipynb @@ -308,12 +308,12 @@ "Ensure all the below are running and healthy before proceeding further\n", "```output\n", "NAMES STATUS\n", - "nemoretriever-ranking-ms Up ... (healthy)\n", + "nemotron-ranking-ms Up ... (healthy)\n", "compose-page-elements-1 Up ...\n", "compose-nemoretriever-ocr-1 Up ...\n", "compose-graphic-elements-1 Up ...\n", "compose-table-structure-1 Up ...\n", - "nemoretriever-embedding-ms Up ... (healthy)\n", + "nemotron-embedding-ms Up ... (healthy)\n", "nim-llm-ms Up ... (healthy)\n", "```" ] @@ -338,15 +338,15 @@ "os.environ[\"OCR_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr\"\n", "os.environ[\"OCR_INFER_PROTOCOL\"] = \"http\"\n", "os.environ[\"YOLOX_HTTP_ENDPOINT\"] = (\n", - " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3\"\n", + " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3\"\n", ")\n", "os.environ[\"YOLOX_INFER_PROTOCOL\"] = \"http\"\n", "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT\"] = (\n", - " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-graphic-elements-v1\"\n", + " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1\"\n", ")\n", "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL\"] = \"http\"\n", "os.environ[\"YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT\"] = (\n", - " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-table-structure-v1\"\n", + " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1\"\n", ")\n", "os.environ[\"YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL\"] = \"http\"" ] @@ -440,7 +440,7 @@ " config_ingestor.llm.server_url = \"\" # Empty uses NVIDIA API catalog\n", " config_ingestor.summarizer.server_url = \"\" # Empty uses NVIDIA API catalog\n", "else:\n", - " config_ingestor.embeddings.server_url = \"http://nemoretriever-embedding-ms:8000/v1\"\n", + " config_ingestor.embeddings.server_url = \"http://nemotron-embedding-ms:8000/v1\"\n", "ingestor = NvidiaRAGIngestor(config=config_ingestor)" ] }, @@ -625,11 +625,11 @@ "# \"server_url\": \"\",\n", "# },\n", "# \"embeddings\": {\n", - "# \"model_name\": \"nvidia/llama-3.2-nv-embedqa-1b-v2\",\n", + "# \"model_name\": \"nvidia/llama-nemotron-embed-1b-v2\",\n", "# \"server_url\": \"https://integrate.api.nvidia.com/v1\",\n", "# },\n", "# \"ranking\": {\n", - "# \"model_name\": \"nvidia/llama-3.2-nv-rerankqa-1b-v2\",\n", + "# \"model_name\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", "# \"server_url\": \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking/v1\",\n", "# },\n", "# }\n", diff --git a/notebooks/retriever_api_usage.ipynb b/notebooks/retriever_api_usage.ipynb index 47466e3ed..9b52647ed 100644 --- a/notebooks/retriever_api_usage.ipynb +++ b/notebooks/retriever_api_usage.ipynb @@ -145,8 +145,8 @@ " \"filter_expr\": \"\",\n", " # Override model endpoints and details if needed\n", " #\"model\": \"nvidia/llama-3.3-nemotron-super-49b-v1.5\",\n", - " #\"reranker_model\": \"nvidia/llama-3.2-nv-rerankqa-1b-v2\",\n", - " #\"embedding_model\": \"nvidia/llama-3.2-nv-embedqa-1b-v2\",\n", + " #\"reranker_model\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", + " #\"embedding_model\": \"nvidia/llama-nemotron-embed-1b-v2\",\n", " #\"llm_endpoint\": \"\",\n", " #\"embedding_endpoint\": \"\",\n", " #\"reranker_endpoint\": \"\",\n", @@ -271,8 +271,8 @@ " \"enable_query_rewriting\": False,\n", " \"enable_reranker\": True,\n", " # Override model endpoints and details if needed\n", - " #\"reranker_model\": \"nvidia/llama-3.2-nv-rerankqa-1b-v2\",\n", - " #\"embedding_model\": \"nvidia/llama-3.2-nv-embedqa-1b-v2\",\n", + " #\"reranker_model\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", + " #\"embedding_model\": \"nvidia/llama-nemotron-embed-1b-v2\",\n", " #\"embedding_endpoint\": \"\",\n", " #\"reranker_endpoint\": \"\",\n", "}\n", diff --git a/notebooks/summarization.ipynb b/notebooks/summarization.ipynb index f824f8e7e..d0c7ef285 100644 --- a/notebooks/summarization.ipynb +++ b/notebooks/summarization.ipynb @@ -388,12 +388,12 @@ "Ensure all the below are running and healthy before proceeding further\n", "```output\n", "NAMES STATUS\n", - "nemoretriever-ranking-ms Up ... (healthy)\n", + "nemotron-ranking-ms Up ... (healthy)\n", "compose-page-elements-1 Up ...\n", "compose-nemoretriever-ocr-1 Up ...\n", "compose-graphic-elements-1 Up ...\n", "compose-table-structure-1 Up ...\n", - "nemoretriever-embedding-ms Up ... (healthy)\n", + "nemotron-embedding-ms Up ... (healthy)\n", "nim-llm-ms Up ... (healthy)\n", "```" ] @@ -419,15 +419,15 @@ "os.environ[\"OCR_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr\"\n", "os.environ[\"OCR_INFER_PROTOCOL\"] = \"http\"\n", "os.environ[\"YOLOX_HTTP_ENDPOINT\"] = (\n", - " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3\"\n", + " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3\"\n", ")\n", "os.environ[\"YOLOX_INFER_PROTOCOL\"] = \"http\"\n", "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT\"] = (\n", - " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-graphic-elements-v1\"\n", + " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1\"\n", ")\n", "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL\"] = \"http\"\n", "os.environ[\"YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT\"] = (\n", - " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-table-structure-v1\"\n", + " \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1\"\n", ")\n", "os.environ[\"YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL\"] = \"http\"" ] @@ -548,8 +548,8 @@ " config.ranking.server_url = \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking/v1\"\n", " config.summarizer.server_url = \"\" # Empty uses NVIDIA API catalog\n", "else:\n", - " config.embeddings.server_url = \"nemoretriever-embedding-ms:8000/v1\"\n", - " config.ranking.server_url = \"nemoretriever-ranking-ms:8000\"\n", + " config.embeddings.server_url = \"nemotron-embedding-ms:8000/v1\"\n", + " config.ranking.server_url = \"nemotron-ranking-ms:8000\"\n", " config.summarizer.server_url = \"nim-llm:8000\"\n", " config.llm.server_url = \"nim-llm:8000\"\n", "\n", @@ -967,7 +967,7 @@ "else:\n", " os.environ[\"SUMMARY_LLM_SERVERURL\"] = \"nim-llm:8000\"\n", " os.environ[\"LLM_SERVER_URL\"] = \"nim-llm:8000\"\n", - " os.environ[\"APP_EMBEDDINGS_SERVERURL\"] = \"nemoretriever-embedding-ms:8000/v1\"\n", + " os.environ[\"APP_EMBEDDINGS_SERVERURL\"] = \"nemotron-embedding-ms:8000/v1\"\n", " print(\"✓ Configured for on-prem NIMs\")\n", "\n", "os.environ[\"LOGLEVEL\"] = \"INFO\"\n", diff --git a/pyproject.toml b/pyproject.toml index 80567fb75..6baa185b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,8 +58,8 @@ rag = [ ] ingest = [ # nv-ingest dependencies (required for ingestion operations) - "nv-ingest-api==26.1.1", - "nv-ingest-client==26.1.1", + "nv-ingest-api==26.1.2", + "nv-ingest-client==26.1.2", "tritonclient==2.57.0", # Other ingest dependencies "langchain-openai>=0.2", @@ -80,8 +80,8 @@ ingest = [ ] all = [ # nv-ingest dependencies (required for ingestion operations) - "nv-ingest-api==26.1.1", - "nv-ingest-client==26.1.1", + "nv-ingest-api==26.1.2", + "nv-ingest-client==26.1.2", "tritonclient==2.57.0", # RAG + Ingest dependencies "langchain-openai>=0.2", diff --git a/src/nvidia_rag/utils/configuration.py b/src/nvidia_rag/utils/configuration.py index 9019b7a84..853fe20d7 100644 --- a/src/nvidia_rag/utils/configuration.py +++ b/src/nvidia_rag/utils/configuration.py @@ -631,7 +631,7 @@ class EmbeddingConfig(_ConfigBase): """Embedding configuration.""" model_name: str = Field( - default="nvidia/llama-3.2-nv-embedqa-1b-v2", + default="nvidia/llama-nemotron-embed-1b-v2", env="APP_EMBEDDINGS_MODELNAME", description="Model for generating text embeddings", ) @@ -671,7 +671,7 @@ class RankingConfig(_ConfigBase): """Ranking configuration.""" model_name: str = Field( - default="nvidia/llama-3.2-nv-rerankqa-1b-v2", + default="nvidia/llama-nemotron-rerank-1b-v2", env="APP_RANKING_MODELNAME", description="Model for reranking retrieved documents", ) diff --git a/tests/integration/notebook_test_config.yaml b/tests/integration/notebook_test_config.yaml new file mode 100644 index 000000000..83bc001d5 --- /dev/null +++ b/tests/integration/notebook_test_config.yaml @@ -0,0 +1,118 @@ +# NVIDIA RAG Configuration +# This file contains configurable parameters with their values +# You can override any of these values, and they take precedence over environment variables + +# Vector Store Configuration +vector_store: + name: "milvus" # Name of the vector store backend (e.g., milvus, elasticsearch) + url: "http://localhost:19530" # URL endpoint for the vector store service + index_type: "GPU_CAGRA" # Type of vector index (e.g., GPU_CAGRA, IVF_FLAT) + search_type: "dense" # Type of search to perform (dense, hybrid) + enable_gpu_index: true # Enable GPU acceleration for index building + enable_gpu_search: true # Enable GPU acceleration for search operations + default_collection_name: "test_native" # Default collection/index name for storing vectors + +# NV-Ingest Configuration +nv_ingest: + message_client_hostname: "localhost" # Hostname for NV-Ingest message client + message_client_port: 7670 # Port for NV-Ingest message client + extract_text: true # Enable text extraction from documents + extract_infographics: false # Enable infographic extraction from documents + extract_tables: true # Enable table extraction from documents + extract_charts: true # Enable chart extraction from documents + extract_images: false # Enable image extraction from documents + pdf_extract_method: null # Method to use for PDF extraction + text_depth: "page" # Granularity level for text extraction (page, document) + chunk_size: 512 # Maximum size of text chunks in tokens + chunk_overlap: 150 # Number of overlapping tokens between chunks + caption_model_name: "nvidia/nemotron-nano-12b-v2-vl" # Model name for generating image captions + caption_endpoint_url: "http://localhost:1977/v1/chat/completions" # API endpoint for caption generation service + enable_pdf_splitter: true # Enable PDF page splitting during ingestion + +# LLM Configuration +llm: + server_url: "http://localhost:8999" # URL endpoint for the LLM inference service (on-prem NIM default) + model_name: "nvidia/llama-3.3-nemotron-super-49b-v1.5" # Name of the language model to use for generation + # api_key: "" # Optional: API key for LLM service (overrides NVIDIA_API_KEY environment variable) + parameters: + max_tokens: 32768 # Maximum number of tokens to generate in response + temperature: 0.0 # Sampling temperature for controlling randomness (0.0 = deterministic) + top_p: 1.0 # Nucleus sampling threshold for token selection + +# Query Rewriter Configuration +query_rewriter: + model_name: "nvidia/llama-3.3-nemotron-super-49b-v1.5" # Model for rewriting user queries to improve retrieval + server_url: "localhost:8999" # URL endpoint for query rewriter service + enable_query_rewriter: false # Enable automatic query rewriting before retrieval + # api_key: "" # Optional: API key for query rewriter (overrides NVIDIA_API_KEY environment variable) + +# Filter Expression Generator Configuration +filter_expression_generator: + model_name: "nvidia/llama-3.3-nemotron-super-49b-v1.5" # Model for generating metadata filter expressions from queries + server_url: "localhost:8999" # URL endpoint for filter expression generator service + enable_filter_generator: false # Enable automatic filter expression generation from natural language + # api_key: "" # Optional: API key for filter generator (overrides NVIDIA_API_KEY environment variable) + +# Embedding Configuration +embeddings: + model_name: "nvidia/llama-3.2-nv-embedqa-1b-v2" # Model for generating text embeddings + dimensions: 2048 # Dimensionality of the embedding vectors + server_url: "http://localhost:9080/v1" # URL endpoint for embedding service (on-prem NIM default) + # api_key: "" # Optional: API key for embeddings (overrides NVIDIA_API_KEY environment variable) + +# Ranking Configuration +ranking: + model_name: "nvidia/llama-3.2-nv-rerankqa-1b-v2" # Model for reranking retrieved documents + server_url: "http://localhost:1976" # URL endpoint for reranking service (on-prem NIM default) + enable_reranker: true # Enable reranking of retrieved documents before generation + # api_key: "" # Optional: API key for reranking (overrides NVIDIA_API_KEY environment variable) + +# Retriever Configuration +retriever: + top_k: 10 # Number of top documents to return after retrieval and reranking + vdb_top_k: 100 # Number of documents to retrieve from vector database before reranking + score_threshold: 0.25 # Minimum similarity score threshold for retrieved documents + +# Tracing Configuration +tracing: + enabled: false # Enable distributed tracing and metrics collection + otlp_http_endpoint: "http://localhost:4318/v1/traces" # OpenTelemetry HTTP endpoint for traces + otlp_grpc_endpoint: "grpc://localhost:4317" # OpenTelemetry gRPC endpoint for traces + +# Vision-Language Model Configuration +vlm: + server_url: "http://localhost:1977/v1" # URL endpoint for Vision-Language Model service + model_name: "nvidia/nemotron-nano-12b-v2-vl" # Vision-Language Model for processing images and text + # api_key: "" # Optional: API key for VLM service (overrides NVIDIA_API_KEY environment variable) + +# MinIO Configuration +minio: + endpoint: "localhost:9010" # MinIO object storage endpoint + access_key: "minioadmin" # MinIO access key for authentication + secret_key: "minioadmin" # MinIO secret key for authentication + +# Summarizer Configuration +summarizer: + model_name: "nvidia/llama-3.3-nemotron-super-49b-v1.5" # Model for generating document summaries + server_url: "localhost:8999" # URL endpoint for summarization service + max_chunk_length: 50000 # Maximum character length for chunks to summarize + chunk_overlap: 200 # Character overlap between chunks during summarization + temperature: 0.0 # Sampling temperature for summary generation + top_p: 1.0 # Nucleus sampling threshold for summary generation + # api_key: "" # Optional: API key for summarization (overrides NVIDIA_API_KEY environment variable) + +# Reflection Configuration +reflection: + enable_reflection: false # Enable self-reflection to improve answer quality + max_loops: 3 # Maximum number of reflection iterations + model_name: "nvidia/llama-3.3-nemotron-super-49b-v1.5" # Model for reflection and quality assessment + server_url: "" # URL endpoint for reflection service + context_relevance_threshold: 1 # Minimum relevance score for context to be considered useful + response_groundedness_threshold: 1 # Minimum groundedness score for response to be considered factual + # api_key: "" # Optional: API key for reflection (overrides NVIDIA_API_KEY environment variable) + +# Top-level Configuration Flags +enable_guardrails: false # Enable safety guardrails for input/output filtering +enable_citations: true # Include source citations in generated responses +enable_vlm_inference: false # Enable Vision-Language Model for multimodal queries +temp_dir: "./tmp-data/" # Temporary directory for file processing and storage \ No newline at end of file diff --git a/tests/integration/test_cases/library_usage.py b/tests/integration/test_cases/library_usage.py index d67ef4942..c562201ca 100644 --- a/tests/integration/test_cases/library_usage.py +++ b/tests/integration/test_cases/library_usage.py @@ -49,8 +49,7 @@ def _get_config(self): """Get or create shared config object with common settings""" if self._config is None: from nvidia_rag.utils.configuration import NvidiaRAGConfig - - config_path = Path(__file__).parent.parent.parent.parent / "notebooks" / "config.yaml" + config_path = Path(__file__).parent.parent.parent.parent / "tests" / "integration" / "notebook_test_config.yaml" self._config = NvidiaRAGConfig.from_yaml(str(config_path)) # Common configuration for all library tests diff --git a/tests/unit/test_compose_helm_parity/env_parity_exemptions.yaml b/tests/unit/test_compose_helm_parity/env_parity_exemptions.yaml index 80cea693c..ed9f45c06 100644 --- a/tests/unit/test_compose_helm_parity/env_parity_exemptions.yaml +++ b/tests/unit/test_compose_helm_parity/env_parity_exemptions.yaml @@ -53,8 +53,8 @@ ngcApiKeyPresenceExemptions: perService: nims.yaml: nim-llm: true - nemoretriever-embedding-ms: true - nemoretriever-ranking-ms: true + nemotron-embedding-ms: true + nemotron-ranking-ms: true vlm-ms: true diff --git a/tests/unit/test_compose_helm_parity/test_compose_helm_parity.py b/tests/unit/test_compose_helm_parity/test_compose_helm_parity.py index 713e50a6c..15a36bbbf 100644 --- a/tests/unit/test_compose_helm_parity/test_compose_helm_parity.py +++ b/tests/unit/test_compose_helm_parity/test_compose_helm_parity.py @@ -300,7 +300,7 @@ def test_compose_helm_image_and_env_parity(): "ngcAPIKey", ], }, - "nemoretriever-embedding-ms": { + "nemotron-embedding-ms": { "values_image_repo_path": [ "nimOperator", "nvidia-nim-llama-32-nv-embedqa-1b-v2", @@ -320,7 +320,7 @@ def test_compose_helm_image_and_env_parity(): "ngcAPIKey", ], }, - "nemoretriever-ranking-ms": { + "nemotron-ranking-ms": { "values_image_repo_path": [ "nimOperator", "nvidia-nim-llama-32-nv-rerankqa-1b-v2", diff --git a/tests/unit/test_rag_server/test_self_reflection.py b/tests/unit/test_rag_server/test_self_reflection.py index f7a6d54c6..6acf3b0e0 100644 --- a/tests/unit/test_rag_server/test_self_reflection.py +++ b/tests/unit/test_rag_server/test_self_reflection.py @@ -142,7 +142,7 @@ async def test_check_context_relevance(mocker): and structured prompts for consistent, reproducible reflection results. """ # Set up a local ranker for reranking documents - local_ranker = get_ranking_model(model="nvidia/llama-3.2-nv-rerankqa-1b-v2", url="") + local_ranker = get_ranking_model(model="nvidia/llama-nemotron-rerank-1b-v2", url="") # Create a mock VDBRag object mock_vdb_op = mocker.MagicMock(spec=VDBRag) diff --git a/tests/unit/test_utils/test_configuration.py b/tests/unit/test_utils/test_configuration.py index 3525664b7..e865b7326 100644 --- a/tests/unit/test_utils/test_configuration.py +++ b/tests/unit/test_utils/test_configuration.py @@ -197,7 +197,7 @@ def test_default_values(self): """Test default configuration values.""" config = EmbeddingConfig() - assert config.model_name == "nvidia/llama-3.2-nv-embedqa-1b-v2" + assert config.model_name == "nvidia/llama-nemotron-embed-1b-v2" assert config.model_engine == "nvidia-ai-endpoints" assert config.dimensions == 2048 assert config.server_url == "" @@ -210,7 +210,7 @@ def test_default_values(self): """Test default configuration values.""" config = RankingConfig() - assert config.model_name == "nvidia/llama-3.2-nv-rerankqa-1b-v2" + assert config.model_name == "nvidia/llama-nemotron-rerank-1b-v2" assert config.model_engine == "nvidia-ai-endpoints" assert config.server_url == "" assert config.enable_reranker is True diff --git a/tests/unit/test_utils/test_reranker.py b/tests/unit/test_utils/test_reranker.py index f012a2a5c..ac60c67fe 100644 --- a/tests/unit/test_utils/test_reranker.py +++ b/tests/unit/test_utils/test_reranker.py @@ -66,11 +66,11 @@ def test_get_ranking_model_nvidia_endpoints_with_model_name( mock_nvidia_rerank.return_value = mock_reranker result = _get_ranking_model( - "nvidia/llama-3.2-nv-rerankqa-1b-v2", "", 10, config=mock_config + "nvidia/llama-nemotron-rerank-1b-v2", "", 10, config=mock_config ) mock_nvidia_rerank.assert_called_once_with( - model="nvidia/llama-3.2-nv-rerankqa-1b-v2", + model="nvidia/llama-nemotron-rerank-1b-v2", api_key="test-api-key", top_n=10, truncate="END", @@ -297,7 +297,7 @@ def test_complete_ranking_workflow_with_url( # Test the workflow model = get_ranking_model( - "nvidia/llama-3.2-nv-rerankqa-1b-v2", "rerank-service:8080", 10 + "nvidia/llama-nemotron-rerank-1b-v2", "rerank-service:8080", 10 ) # Test that the model can be used @@ -325,7 +325,7 @@ def test_complete_ranking_workflow_api_catalog( mock_get_model.return_value = mock_reranker # Test the workflow - model = get_ranking_model("nvidia/llama-3.2-nv-rerankqa-1b-v2", "", 5) + model = get_ranking_model("nvidia/llama-nemotron-rerank-1b-v2", "", 5) # Test that the model can be used documents = ["doc1", "doc2"] diff --git a/uv.lock b/uv.lock index 7c267bf95..9e77bddac 100644 --- a/uv.lock +++ b/uv.lock @@ -1753,7 +1753,7 @@ wheels = [ [[package]] name = "nv-ingest-api" -version = "26.1.1" +version = "26.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff" }, @@ -1767,14 +1767,14 @@ dependencies = [ { name = "tritonclient" }, { name = "universal-pathlib" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/41/eb/e0469e918d617175e1d3bcf952f0ca8e9b7756fce7817d5386ac4ddca154/nv_ingest_api-26.1.1.tar.gz", hash = "sha256:063d51f1d560bf03d7a595ff3ecebac1bffae45607cf6bd01e4fa8ca2265a884", size = 259532, upload-time = "2026-01-13T23:44:11.112Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/bd/e6e885cca94b89723468d4c32d52d30ccc0235ebe2f1db33b0605402d6b8/nv_ingest_api-26.1.2.tar.gz", hash = "sha256:fea08f9bda064938a5876f1610ef0b92c6a1e4943130c564f329b0c87efa3daf", size = 259604, upload-time = "2026-01-21T14:06:27.092Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/42/51/cd93750a1c5797d8d12e843bb13645b0930e4490f96ca49f458aeb641018/nv_ingest_api-26.1.1-py3-none-any.whl", hash = "sha256:e4f8b860765cedba72622782692e2ffc69a100fd61956e8b8a81a47e6c852d66", size = 357481, upload-time = "2026-01-13T23:44:07.943Z" }, + { url = "https://files.pythonhosted.org/packages/78/66/21e30e658578b7e5ab30857b99e9a0a5c91728ffdca13dadc3d3dba58b98/nv_ingest_api-26.1.2-py3-none-any.whl", hash = "sha256:8e7539a6b7d52afd821c0030e3197cfffddc011d26a8b093cf7b5ffa8addf02d", size = 357537, upload-time = "2026-01-21T14:06:24.321Z" }, ] [[package]] name = "nv-ingest-client" -version = "26.1.1" +version = "26.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "build" }, @@ -1789,9 +1789,9 @@ dependencies = [ { name = "setuptools" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cb/c4/ae5e2b00a8ffdfc1a3cf660ded68c188140ca433b22446adbb72ccfc455d/nv_ingest_client-26.1.1.tar.gz", hash = "sha256:26d6844eac946b4fdb8da2f5f1e77feb22b52ac21e6d772cf6b1c8c21cef4bb8", size = 126865, upload-time = "2026-01-13T23:44:15.615Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/62/386bc4a336b91df9c65afc18d905bb1fe3dd44ef1ce038895a701cba6035/nv_ingest_client-26.1.2.tar.gz", hash = "sha256:7ea4a35d4e7051031c273eb2b15170a0555462b702602e5c4fdce947bd39d446", size = 126061, upload-time = "2026-01-21T14:06:31.836Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/64/ea/042ad6d8ddfa887667159af8b473835bfcc5b9f51ba6e82ff14078e59a3e/nv_ingest_client-26.1.1-py3-none-any.whl", hash = "sha256:df9906c7021e6a1ae64140fbe7a345679b1cfad7dfa486442e38ca75d64f2b39", size = 147197, upload-time = "2026-01-13T23:44:12.761Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a0/6f082f42ba1ba4e3de751deee98099114405489389d82b840024fa26512e/nv_ingest_client-26.1.2-py3-none-any.whl", hash = "sha256:49ec81c2e2470509fc527d778d886caebe05a7f3012918eef2fd67922ea9b8b4", size = 146091, upload-time = "2026-01-21T14:06:28.586Z" }, ] [[package]] @@ -1924,10 +1924,10 @@ requires-dist = [ { name = "langchain-openai", marker = "extra == 'rag'", specifier = ">=0.2" }, { name = "lark", specifier = ">=1.2.2" }, { name = "minio", specifier = ">=7.2,<8.0" }, - { name = "nv-ingest-api", marker = "extra == 'all'", specifier = "==26.1.1" }, - { name = "nv-ingest-api", marker = "extra == 'ingest'", specifier = "==26.1.1" }, - { name = "nv-ingest-client", marker = "extra == 'all'", specifier = "==26.1.1" }, - { name = "nv-ingest-client", marker = "extra == 'ingest'", specifier = "==26.1.1" }, + { name = "nv-ingest-api", marker = "extra == 'all'", specifier = "==26.1.2" }, + { name = "nv-ingest-api", marker = "extra == 'ingest'", specifier = "==26.1.2" }, + { name = "nv-ingest-client", marker = "extra == 'all'", specifier = "==26.1.2" }, + { name = "nv-ingest-client", marker = "extra == 'ingest'", specifier = "==26.1.2" }, { name = "opentelemetry-api", marker = "extra == 'all'", specifier = ">=1.29,<2.0" }, { name = "opentelemetry-api", marker = "extra == 'ingest'", specifier = ">=1.29,<2.0" }, { name = "opentelemetry-api", marker = "extra == 'rag'", specifier = ">=1.29,<2.0" }, diff --git a/variables.env b/variables.env index 790e1ad7c..b9003e7b5 100644 --- a/variables.env +++ b/variables.env @@ -15,8 +15,8 @@ DOCKER_VOLUME_DIRECTORY=vectordb # ==== Endpoints for using on-prem NIMs ==== APP_LLM_SERVERURL=nim-llm:8000 -APP_EMBEDDINGS_SERVERURL=nemoretriever-embedding-ms:8000/v1 -APP_RANKING_SERVERURL=nemoretriever-ranking-ms:8000 +APP_EMBEDDINGS_SERVERURL=nemotron-embedding-ms:8000/v1 +APP_RANKING_SERVERURL=nemotron-ranking-ms:8000 OCR_GRPC_ENDPOINT=nemoretriever-ocr:8001 OCR_HTTP_ENDPOINT=http://nemoretriever-ocr:8000/v1/infer OCR_INFER_PROTOCOL=grpc @@ -35,11 +35,11 @@ YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL=grpc # OCR_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr # OCR_INFER_PROTOCOL=http # OCR_MODEL_NAME=scene_text_ensemble -# YOLOX_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3 +# YOLOX_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3 # YOLOX_INFER_PROTOCOL=http -# YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-graphic-elements-v1 +# YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1 # YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL=http -# YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-table-structure-v1 +# YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1 # YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL=http From a3c0438a00a15363b4cb0856a1ab896cd12fe459 Mon Sep 17 00:00:00 2001 From: kumar-punit Date: Fri, 27 Feb 2026 16:55:19 +0530 Subject: [PATCH 13/81] Preserve filename case in filter; fix syntax error examples to use double quotes (#337) --- docs/custom-metadata.md | 2 +- src/nvidia_rag/utils/metadata_validation.py | 19 ++++++++------- .../test_filter_validator.py | 24 +++++++++++++++++++ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/docs/custom-metadata.md b/docs/custom-metadata.md index dba96e461..c02b80eeb 100644 --- a/docs/custom-metadata.md +++ b/docs/custom-metadata.md @@ -232,7 +232,7 @@ The system automatically manages certain metadata fields that are added to all c | Field Name | Type | Description | Auto-Populated | User Override | |------------|------|-------------|----------------|---------------| -| **`filename`** | `string` | Name of the uploaded file | ✅ RAG system | ✅ Yes - define in schema | +| **`filename`** | `string` | Name of the uploaded file (case-sensitive when filtering) | ✅ RAG system | ✅ Yes - define in schema | | **`page_number`** | `integer` | Page number where content appears (1-indexed) | ✅ nv-ingest | ✅ Yes - define in schema | | **`start_time`** | `integer` | Start timestamp in milliseconds for audio/video segments | ✅ nv-ingest | ✅ Yes - define in schema | | **`end_time`** | `integer` | End timestamp in milliseconds for audio/video segments | ✅ nv-ingest | ✅ Yes - define in schema | diff --git a/src/nvidia_rag/utils/metadata_validation.py b/src/nvidia_rag/utils/metadata_validation.py index e1bdef417..3ff404483 100644 --- a/src/nvidia_rag/utils/metadata_validation.py +++ b/src/nvidia_rag/utils/metadata_validation.py @@ -2164,7 +2164,10 @@ def comparison(self, args) -> str: logger.debug(f"[comparison] Failed to normalize datetime: {e}") value_val = str(value_token) elif field_info and is_string_type(field_info.type): - value_val = str(value_token).lower() + if field_name == "filename": + value_val = str(value_token) + else: + value_val = str(value_token).lower() else: value_val = str(value_token) else: @@ -2677,15 +2680,15 @@ def _get_error_context(filter_expr: str, error: UnexpectedInput) -> str: error_msg = ( f"Syntax error at line {line_num}, column {col_num}: '{snippet}'" ) - error_msg += "\n\nExamples of valid filter expressions:" - error_msg += "\n• content_metadata[\"title\"] == 'value'" - error_msg += "\n• content_metadata[\"title\"] = 'value'" + error_msg += "\n\nExamples of valid filter expressions (use double quotes for string values):" + error_msg += '\n• content_metadata["title"] == "value"' + error_msg += '\n• content_metadata["title"] = "value"' error_msg += '\n• content_metadata["rating"] > 5' - error_msg += "\n• content_metadata[\"category\"] like '%tech%'" - error_msg += "\n• content_metadata[\"tags\"] in ['important', 'urgent']" - error_msg += "\n• content_metadata[\"created_date\"] between '2024-01-01' and '2024-12-31'" + error_msg += '\n• content_metadata["category"] like "%tech%"' + error_msg += '\n• content_metadata["tags"] in ["important", "urgent"]' + error_msg += '\n• content_metadata["created_date"] between "2024-01-01" and "2024-12-31"' error_msg += '\n• content_metadata["is_public"] == true' - error_msg += '\n• content_metadata["file_size"] > 1000 and content_metadata["type"] == \'pdf\'' + error_msg += '\n• content_metadata["file_size"] > 1000 and content_metadata["type"] == "pdf"' return error_msg diff --git a/tests/unit/test_metadata_validation/test_filter_validator.py b/tests/unit/test_metadata_validation/test_filter_validator.py index 0bf174973..3f6b34759 100644 --- a/tests/unit/test_metadata_validation/test_filter_validator.py +++ b/tests/unit/test_metadata_validation/test_filter_validator.py @@ -3222,6 +3222,30 @@ def test_string_basic_operations(self, mock_config, string_schema): assert result["status"] is True assert "error_message" not in result + def test_filename_preserves_case_other_string_lowercased(self, mock_config): + """Filename filter preserves case; other string fields are lowercased for matching.""" + schema = MetadataSchema( + schema=[ + MetadataField(name="filename", type="string", required=False), + MetadataField(name="title", type="string", required=False), + ] + ) + parser = FilterExpressionParser(schema, mock_config) + + # Filename must preserve case (ingestion stores original case). + result = parser.process_filter_expression( + 'content_metadata["filename"] == "Report.PDF"' + ) + assert result["status"] is True + assert '"Report.PDF"' in result["processed_expression"] + + # Other string fields are normalized to lowercase. + result = parser.process_filter_expression( + 'content_metadata["title"] == "Technical"' + ) + assert result["status"] is True + assert '"technical"' in result["processed_expression"] + def test_string_like_operations(self, mock_config, string_schema): """Test string LIKE operations.""" parser = FilterExpressionParser(string_schema, mock_config) From 85d41b6ac63a71df1039708fe1ebea45aea8acc3 Mon Sep 17 00:00:00 2001 From: Kurt Heiss Date: Fri, 27 Feb 2026 09:42:57 -0800 Subject: [PATCH 14/81] conf.py fix (#391) * confirming presence of switcher text in conf.py file * docs: adjust conf.py for 2.5.0 --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0bc42fd7b..6f95103b4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -74,8 +74,7 @@ "icon": "fa-brands fa-github", } ], - # Version switcher disabled: set "switcher": {"json_url": "...", "version_match": release} - # and ensure versions1.json is at the json_url path when using versioned doc deployments. + "switcher": {"json_url": "../versions1.json", "version_match": release}, "extra_head": { """ @@ -88,6 +87,7 @@ }, } + # Add any paths that contain custom static files (such as style sheets) here, html_css_files = ["swagger-nvidia.css"] From 2136f02c4738e50f57f61134dac354e05c999fa7 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Fri, 27 Feb 2026 11:43:10 -0600 Subject: [PATCH 15/81] seeing collections --- src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py index 2ace149bb..12d3e6ed6 100644 --- a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py +++ b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py @@ -181,7 +181,7 @@ def __init__( with self._pool.acquire() as conn: with conn.cursor() as cursor: cursor.execute("SELECT 1 FROM DUAL") - logger.info(f"Connected to Oracle at {self._oracle_cs}") + logger.info(f"Connected to Oracle with connection {self._oracle_cs}") except oracledb.Error as e: logger.exception("Failed to connect to Oracle at %s: %s", self._oracle_cs, e) raise APIError( @@ -630,7 +630,7 @@ def get_metadata_schema(self, collection_name: str) -> list[dict[str, Any]]: ) row = cursor.fetchone() if row and row[0]: - return json.loads(row[0]) + return row[0] logger.info(f"No metadata schema found for {table_name}") return [] @@ -709,8 +709,7 @@ def _get_aggregated_document_info( ) row = cursor.fetchone() if row and row[0]: - existing = json.loads(row[0]) - return perform_document_info_aggregation(existing, info_value) + return perform_document_info_aggregation(row[0][0], info_value) except Exception as e: logger.warning(f"Error getting aggregated info: {e}") @@ -737,7 +736,7 @@ def get_document_info( ) row = cursor.fetchone() if row and row[0]: - return json.loads(row[0]) + return row[0] return {} From 8c4e6d4b2224058ef9467b73f1d364eda177328b Mon Sep 17 00:00:00 2001 From: kumar-punit Date: Mon, 2 Mar 2026 11:36:23 +0530 Subject: [PATCH 16/81] Added vdb serialization if parallel ingestion which helps in high concurrent batch ingestion having indexing issues (#389) --- src/nvidia_rag/ingestor_server/main.py | 28 +++- src/nvidia_rag/utils/vdb/vdb_ingest_base.py | 40 +++++- tests/unit/test_utils/test_configuration.py | 3 +- .../test_vdb/test_vdb_ingest_base.py | 129 ++++++++++++++++++ 4 files changed, 189 insertions(+), 11 deletions(-) create mode 100644 tests/unit/test_utils/test_vdb/test_vdb_ingest_base.py diff --git a/src/nvidia_rag/ingestor_server/main.py b/src/nvidia_rag/ingestor_server/main.py index f28750e38..88df2c957 100644 --- a/src/nvidia_rag/ingestor_server/main.py +++ b/src/nvidia_rag/ingestor_server/main.py @@ -94,6 +94,7 @@ from nvidia_rag.utils.summary_status_handler import SUMMARY_STATUS_HANDLER from nvidia_rag.utils.vdb import DEFAULT_DOCUMENT_INFO_COLLECTION, _get_vdb_op from nvidia_rag.utils.vdb.vdb_base import VDBRag +from nvidia_rag.utils.vdb.vdb_ingest_base import SerializedVDBWrapper # Initialize logger logger = logging.getLogger(__name__) @@ -752,10 +753,13 @@ async def __build_ingestion_response( uploaded_documents = [] for filepath in filepaths: if os.path.basename(filepath) not in failures_filepaths: - doc_type_counts, _, total_elements, raw_text_elements_size = ( - self._get_document_type_counts( - [filename_to_result_map.get(os.path.basename(filepath), [])] - ) + ( + doc_type_counts, + _, + total_elements, + raw_text_elements_size, + ) = self._get_document_type_counts( + [filename_to_result_map.get(os.path.basename(filepath), [])] ) document_info = create_document_metadata( @@ -2265,6 +2269,13 @@ async def __run_nvingest_batched_ingestion( logger.info( f"Processing batches in parallel with concurrency: {state_manager.concurrent_batches}" ) + + if vdb_op is not None and SerializedVDBWrapper is not None: + vdb_op = SerializedVDBWrapper(vdb_op) + logger.info( + "VDB write serialization enabled — extraction runs in parallel, VDB writes are sequential" + ) + all_results = [] all_failures = [] tasks = [] @@ -2850,9 +2861,12 @@ def _log_result_info( Returns: dict[str, Any]: Document info with metrics """ - doc_type_counts, total_documents, total_elements, raw_text_elements_size = ( - self._get_document_type_counts(results) - ) + ( + doc_type_counts, + total_documents, + total_elements, + raw_text_elements_size, + ) = self._get_document_type_counts(results) document_info = { "doc_type_counts": doc_type_counts, diff --git a/src/nvidia_rag/utils/vdb/vdb_ingest_base.py b/src/nvidia_rag/utils/vdb/vdb_ingest_base.py index 190d69ad4..bbbb61911 100644 --- a/src/nvidia_rag/utils/vdb/vdb_ingest_base.py +++ b/src/nvidia_rag/utils/vdb/vdb_ingest_base.py @@ -14,7 +14,8 @@ # limitations under the License. """ -This module provides VDBRagIngest, a VDBRag subclass with nv_ingest support. +This module provides VDBRagIngest, a VDBRag subclass with nv_ingest support, +and SerializedVDBWrapper for serializing concurrent VDB write operations. VDBRagIngest combines VDBRag (pure abstract base) with VDB from nv_ingest_client, providing full ingestion capabilities. This class should be used by ingestor_server @@ -25,6 +26,7 @@ """ import logging +import threading from nvidia_rag.utils.vdb.vdb_base import VDBRag @@ -52,6 +54,40 @@ class VDBRagIngest(VDBRag, VDB): pass + class SerializedVDBWrapper: + """Wraps a VDB op to serialize write operations while keeping reads parallel. + + When multiple batches run extraction concurrently, their VDB writes can + overlap and cause indexing timeouts (e.g., GPU_CAGRA JIT compilation takes + longer than the client's patience window). This wrapper uses a threading + lock to ensure only one batch writes to the VDB at a time. + """ + + def __init__(self, vdb_op): + self._vdb_op = vdb_op + self._write_lock = threading.Lock() + + def run_async(self, records): + with self._write_lock: + return self._vdb_op.run_async(records) + + def run(self, records): + with self._write_lock: + return self._vdb_op.run(records) + + def write_to_index(self, records, **kwargs): + with self._write_lock: + return self._vdb_op.write_to_index(records, **kwargs) + + def create_index(self, **kwargs): + with self._write_lock: + return self._vdb_op.create_index(**kwargs) + + def __getattr__(self, name): + return getattr(self._vdb_op, name) + + VDB.register(SerializedVDBWrapper) + except ImportError: logger.warning( "Optional nv_ingest_client module not installed. " @@ -59,4 +95,4 @@ class VDBRagIngest(VDBRag, VDB): ) # Fallback: VDBRagIngest is just VDBRag without nv_ingest support VDBRagIngest = VDBRag - + SerializedVDBWrapper = None diff --git a/tests/unit/test_utils/test_configuration.py b/tests/unit/test_utils/test_configuration.py index e865b7326..8c8f6651d 100644 --- a/tests/unit/test_utils/test_configuration.py +++ b/tests/unit/test_utils/test_configuration.py @@ -23,8 +23,6 @@ import pytest import yaml -from pydantic import SecretStr, ValidationError - from nvidia_rag.utils.configuration import ( EmbeddingConfig, FilterExpressionGeneratorConfig, @@ -44,6 +42,7 @@ VectorStoreConfig, VLMConfig, ) +from pydantic import SecretStr, ValidationError class TestVectorStoreConfig: diff --git a/tests/unit/test_utils/test_vdb/test_vdb_ingest_base.py b/tests/unit/test_utils/test_vdb/test_vdb_ingest_base.py new file mode 100644 index 000000000..a13877d05 --- /dev/null +++ b/tests/unit/test_utils/test_vdb/test_vdb_ingest_base.py @@ -0,0 +1,129 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for SerializedVDBWrapper from vdb_ingest_base module.""" + +import threading +from concurrent.futures import ThreadPoolExecutor +from unittest.mock import MagicMock + +import pytest +from nvidia_rag.utils.vdb.vdb_ingest_base import SerializedVDBWrapper + + +@pytest.fixture +def mock_vdb_op(): + """Create a mock VDB operation object.""" + op = MagicMock() + op.run_async.return_value = "run_async_result" + op.run.return_value = "run_result" + op.write_to_index.return_value = "write_result" + op.create_index.return_value = "index_result" + op.some_read_method.return_value = "read_result" + return op + + +@pytest.fixture +def wrapper(mock_vdb_op): + """Create a SerializedVDBWrapper around a mock VDB op.""" + return SerializedVDBWrapper(mock_vdb_op) + + +@pytest.mark.skipif( + SerializedVDBWrapper is None, + reason="nv_ingest_client not installed", +) +class TestSerializedVDBWrapper: + """Test cases for SerializedVDBWrapper.""" + + def test_run_async_delegates_to_wrapped_op(self, wrapper, mock_vdb_op): + """Test that run_async delegates to the wrapped VDB op.""" + records = [{"data": "test"}] + result = wrapper.run_async(records) + + mock_vdb_op.run_async.assert_called_once_with(records) + assert result == "run_async_result" + + def test_run_delegates_to_wrapped_op(self, wrapper, mock_vdb_op): + """Test that run delegates to the wrapped VDB op.""" + records = [{"data": "test"}] + result = wrapper.run(records) + + mock_vdb_op.run.assert_called_once_with(records) + assert result == "run_result" + + def test_write_to_index_delegates_with_kwargs(self, wrapper, mock_vdb_op): + """Test that write_to_index passes kwargs to the wrapped VDB op.""" + records = [{"data": "test"}] + result = wrapper.write_to_index(records, collection_name="test") + + mock_vdb_op.write_to_index.assert_called_once_with( + records, collection_name="test" + ) + assert result == "write_result" + + def test_create_index_delegates_with_kwargs(self, wrapper, mock_vdb_op): + """Test that create_index passes kwargs to the wrapped VDB op.""" + result = wrapper.create_index(collection_name="test") + + mock_vdb_op.create_index.assert_called_once_with(collection_name="test") + assert result == "index_result" + + def test_getattr_delegates_non_overridden_methods(self, wrapper, mock_vdb_op): + """Test that non-write methods pass through to the wrapped VDB op.""" + result = wrapper.some_read_method() + mock_vdb_op.some_read_method.assert_called_once() + assert result == "read_result" + + def test_isinstance_check_with_vdb(self, wrapper): + """Test that wrapper passes isinstance check for VDB (was a real bug).""" + from nv_ingest_client.util.vdb.adt_vdb import VDB + + assert isinstance(wrapper, VDB) + + def test_write_methods_are_serialized(self, mock_vdb_op): + """Test that concurrent write calls are serialized by the lock.""" + execution_log = [] + lock_held = threading.Event() + + def locked_write(records): + batch = records[0] + execution_log.append(f"acquired_{batch}") + if batch == "batch_1": + lock_held.set() + threading.Event().wait(0.1) + execution_log.append(f"released_{batch}") + return "done" + + mock_vdb_op.run.side_effect = locked_write + wrapper = SerializedVDBWrapper(mock_vdb_op) + + with ThreadPoolExecutor(max_workers=2) as executor: + f1 = executor.submit(wrapper.run, ["batch_1"]) + lock_held.wait(timeout=2) + f2 = executor.submit(wrapper.run, ["batch_2"]) + f1.result(timeout=5) + f2.result(timeout=5) + + assert execution_log.index("released_batch_1") < execution_log.index( + "acquired_batch_2" + ) + + def test_wrapper_propagates_exceptions(self, wrapper, mock_vdb_op): + """Test that exceptions from the wrapped op propagate through the lock.""" + mock_vdb_op.run.side_effect = ValueError("indexing failed") + + with pytest.raises(ValueError, match="indexing failed"): + wrapper.run([{"data": "test"}]) From c91de07e86e9c5d222cb1618b9355453c8409200 Mon Sep 17 00:00:00 2001 From: nv-pranjald <150428320+nv-pranjald@users.noreply.github.com> Date: Mon, 2 Mar 2026 11:59:15 +0530 Subject: [PATCH 17/81] Update prompt and unify reasoning budged and enable thinking (#386) * Prompt tuining, low reasoning and reasoning budget * Filter out think token when enable filter is on * Use default prompt * Fix unit test * Add doc for nemotron thinking budget * Add question back in prompt.yaml --- deploy/compose/docker-compose-rag-server.yaml | 10 +- deploy/helm/nvidia-blueprint-rag/values.yaml | 10 +- docs/accuracy_perf.md | 2 +- docs/enable-nemotron-thinking.md | 155 +++++---- src/nvidia_rag/rag_server/main.py | 16 +- src/nvidia_rag/utils/configuration.py | 22 +- src/nvidia_rag/utils/llm.py | 320 ++++++++++-------- .../test_rag_main_advanced_features.py | 12 +- tests/unit/test_utils/test_configuration.py | 12 +- tests/unit/test_utils/test_llm.py | 108 +++--- 10 files changed, 381 insertions(+), 286 deletions(-) diff --git a/deploy/compose/docker-compose-rag-server.yaml b/deploy/compose/docker-compose-rag-server.yaml index 9f7e59a12..d6bafd3ed 100644 --- a/deploy/compose/docker-compose-rag-server.yaml +++ b/deploy/compose/docker-compose-rag-server.yaml @@ -74,11 +74,11 @@ services: LLM_MAX_TOKENS: ${LLM_MAX_TOKENS:-32768} LLM_TEMPERATURE: ${LLM_TEMPERATURE:-0} LLM_TOP_P: ${LLM_TOP_P:-1.0} - - # Enable/disable thinking/reasoning for nemotron-3-nano models (30b variant) - # Set to "true" to enable reasoning mode with reasoning_budget - # Set to "false" to disable reasoning and get direct answers - ENABLE_NEMOTRON_3_NANO_THINKING: ${ENABLE_NEMOTRON_3_NANO_THINKING:-true} + + # Reasoning configuration (supported by Nemotron 3 and other reasoning models) + LLM_ENABLE_THINKING: ${LLM_ENABLE_THINKING:-false} + LLM_REASONING_BUDGET: ${LLM_REASONING_BUDGET:-0} + LLM_LOW_EFFORT: ${LLM_LOW_EFFORT:-false} ##===Query Rewriter Model specific configurations=== APP_QUERYREWRITER_MODELNAME: ${APP_QUERYREWRITER_MODELNAME:-"nvidia/llama-3.3-nemotron-super-49b-v1.5"} diff --git a/deploy/helm/nvidia-blueprint-rag/values.yaml b/deploy/helm/nvidia-blueprint-rag/values.yaml index b385b5296..00c0ecd1d 100644 --- a/deploy/helm/nvidia-blueprint-rag/values.yaml +++ b/deploy/helm/nvidia-blueprint-rag/values.yaml @@ -164,11 +164,6 @@ envVars: LLM_TEMPERATURE: "0" LLM_TOP_P: "1.0" - # Enable/disable thinking/reasoning for nemotron-3-nano models (30b variant) - # Set to "true" to enable reasoning mode with reasoning_budget - # Set to "false" to disable reasoning and get direct answers - ENABLE_NEMOTRON_3_NANO_THINKING: "true" - ##===Query Rewriter Model specific configurations=== APP_QUERYREWRITER_MODELNAME: "nvidia/llama-3.3-nemotron-super-49b-v1.5" # URL on which query rewriter model is hosted. If "", Nvidia hosted API is used @@ -260,6 +255,11 @@ envVars: # Whether to filter content within tags in model responses FILTER_THINK_TOKENS: "true" + # Reasoning configuration (supported by Nemotron 3 and other reasoning models) + LLM_ENABLE_THINKING: "false" + LLM_REASONING_BUDGET: "0" + LLM_LOW_EFFORT: "false" + NEMO_GUARDRAILS_URL: "nemo-guardrails:7331" # enable iterative query decomposition diff --git a/docs/accuracy_perf.md b/docs/accuracy_perf.md index 8367747ae..fa4f6b7a7 100644 --- a/docs/accuracy_perf.md +++ b/docs/accuracy_perf.md @@ -37,7 +37,7 @@ Change the setting if you want different behavior. | `ENABLE_REFLECTION` | `false` | Set to `true` to enable self-reflection. For details, refer to [Self-Reflection Support](self-reflection.md). | - Can improve the response quality by refining intermediate retrieval and final LLM output.
| - Significantly higher latency due to multiple iterations of LLM model call.
- You might need to deploy a separate judge LLM model, increasing GPU requirement.
| | `ENABLE_RERANKER` | `true` | Set to `true` to use the reranking model. | - Improves accuracy by selecting better documents for response generation.
| - Increases latency due to additional processing.
- Additional hardware requirements for self-hosted on premises deployment.
| | `ENABLE_VLM_INFERENCE` | `false` | Set to `true` to use the Vision-Language Model (VLM) for response generation. For details, refer to [VLM for Generation](vlm.md). | - Enables analysis of retrieved images alongside text for richer, multimodal responses.
- Can process up to 4 images per citation.
- Useful for document Q&A, visual search, and multimodal chatbots.
| - Requires additional GPU resources for VLM model deployment.
- Increases latency due to image processing.
| -| Reasoning in `llama-3.3-nemotron-super-49b-v1.5` | `/no_think` | Use `/think` to enable reasoning. For details, refer to [Enable Reasoning](enable-nemotron-thinking.md). | - Improves response quality through enhanced reasoning capabilities.
- Yields more precise responses. The default model is verbose and works best with reasoning enabled.
| - Can increase response latency due to additional thinking process.
- Can increase token usage and computational overhead.
| +| `LLM_ENABLE_THINKING` | `false` | Set to `true` to enable reasoning for Nemotron 3 models. Use `LLM_REASONING_BUDGET` and `LLM_LOW_EFFORT` for fine-grained control. For Nemotron 1.5 models, use the `/think` system prompt instead. For details, refer to [Enable Reasoning](enable-nemotron-thinking.md). | - Improves response quality through enhanced reasoning capabilities.
- Yields more precise responses.
| - Can increase response latency due to additional thinking process.
- Can increase token usage and computational overhead.
| | `RERANKER_SCORE_THRESHOLD` | `0.0` | Filters out retrieved chunks if reranker relevance is lower than this threshold. We recommend that you set this value between `0.3` and `0.5` to balance quality and coverage. For details, refer to [Use the Python Package](python-client.md). | - Faster retrieval by processing fewer documents.
- Can improve accuracy by excluding low-relevance documents.
| - Requires `ENABLE_RERANKER` set to `true` for effective filtering.
- Might filter out too many chunks if the threshold is set high, causing no response from the RAG server.
| | `RERANKER TOP K` | 10 | Increase `reranker TOP K` to increase the probability of relevant context being part of the top-k contexts. | Increasing the value can improve accuracy. | Increasing the value can increase latency. | | `VDB TOP K` | 100 | Increase `VDB TOP K` to provide a larger candidate pool for reranking. | Increasing the value can improve accuracy. | Increasing the value can increase latency. | diff --git a/docs/enable-nemotron-thinking.md b/docs/enable-nemotron-thinking.md index 182a09ea1..d9f9b5383 100644 --- a/docs/enable-nemotron-thinking.md +++ b/docs/enable-nemotron-thinking.md @@ -19,9 +19,102 @@ This guide explains how to enable reasoning for different Nemotron models, each | Model | Control Method | Thinking Budget Parameters | |-------|----------------|----------------------------| +| Nemotron 3 (Nano 30B, and others) | Environment variables | `LLM_ENABLE_THINKING`, `LLM_REASONING_BUDGET`, `LLM_LOW_EFFORT` | | Nemotron 1.5 | System prompts | None | | Nemotron-3-Nano 9B | System prompts | min/max thinking tokens | -| Nemotron-3-Nano 30B | Environment variable | max thinking tokens only | + +## Enable Reasoning for Nemotron 3 Models + +Nemotron 3 models (such as `nvidia/nemotron-3-nano-30b-a3b`) use environment variables to control reasoning. + +Set the following environment variables on the RAG server container (via Docker Compose, Helm values, or shell export): + +**`LLM_ENABLE_THINKING`** +: Enable or disable the reasoning phase. When `true`, the model emits reasoning tokens before the final answer. Default: `false`. + +**`LLM_REASONING_BUDGET`** +: Maximum number of tokens allocated for reasoning. Only used when `LLM_ENABLE_THINKING` is `true`. Default: `0`. + +**`LLM_LOW_EFFORT`** +: Low-effort reasoning mode for faster, cheaper responses with shorter reasoning. Only used when `LLM_ENABLE_THINKING` is `true`. Default: `false`. + +**`FILTER_THINK_TOKENS`** +: Filter content between `` and `` tags in model responses. Keep `true` for production to return only the final answer. Set `false` to see the full reasoning process. Default: `true`. + +## Enable Reasoning for Nemotron 3 Models + +Nemotron 3 models (such as `nvidia/nemotron-3-super-120b-a12b` and `nvidia/nemotron-3-nano-30b-a3b`) use environment variables to control reasoning. + +### Basic Configuration + +```bash +export LLM_ENABLE_THINKING=true +``` + +### Configure Reasoning Budget (Optional) + +Limit the number of reasoning tokens to control latency and cost: + +```bash +export LLM_ENABLE_THINKING=true +export LLM_REASONING_BUDGET=8192 +``` + +### Low-Effort Mode (Optional) + +For faster responses where deep reasoning is unnecessary: + +```bash +export LLM_ENABLE_THINKING=true +export LLM_LOW_EFFORT=true +``` + +### Configure Model Parameters + +After you enable reasoning, configure the model parameters for optimal reasoning performance: + +```bash +export LLM_TEMPERATURE=0.6 +export LLM_TOP_P=0.95 +``` + +### Nemotron-3-Nano 30B + +For `nvidia/nemotron-3-nano-30b-a3b`, reasoning is controlled with the same `LLM_ENABLE_THINKING` variable. The reasoning budget can be set with either `LLM_REASONING_BUDGET` or `LLM_MAX_THINKING_TOKENS`: + +```bash +export LLM_ENABLE_THINKING=true +export LLM_REASONING_BUDGET=8192 +``` + +The 30B model also supports a maximum thinking token limit directly in API requests: + +```json +{ + "model": "nvidia/nemotron-3-nano-30b-a3b", + "messages": [ + { + "role": "user", + "content": "What is the capital of France?" + } + ], + "max_thinking_tokens": 8192 +} +``` + +**Thinking budget parameters:** + +**`max_thinking_tokens`** +: Maximum number of reasoning tokens allowed before generating the final answer. + +:::{important} +The key differences for the 30B model are the following: + +- Uses only `max_thinking_tokens` (not `min_thinking_tokens`) +- Reasoning is available in the model output's `reasoning_content` field (not wrapped in `` tags) +- The `reasoning_content` field is present in the model output but isn't exposed in the generate API response +- No filtering is needed because reasoning is already separated from the final answer +::: ## Enable Reasoning for Nemotron 1.5 @@ -81,7 +174,7 @@ export FILTER_THINK_TOKENS=false For most production use cases, keep `FILTER_THINK_TOKENS=true` (default) to provide cleaner responses to end users. ::: -## Enable Reasoning for Nemotron-3-Nano 9B +## Enable Reasoning for Nemotron Nano 9B The `nvidia/nvidia-nemotron-nano-9b-v2` model uses system prompts to control reasoning similar to Nemotron 1.5. It also adds support for thinking budget parameters to control the extent of reasoning. @@ -132,63 +225,6 @@ The key differences for the 9B model are the following: - No filtering is needed because reasoning is already separated from the final answer ::: -## Enable Reasoning for Nemotron-3-Nano 30B - -The `nvidia/nemotron-3-nano-30b-a3b` model uses a different approach for reasoning control. Instead of system prompts, you control reasoning through an environment variable. - -### Enable Reasoning Through an Environment Variable - -Set the environment variable to enable or disable reasoning: - -```bash -# Enable reasoning (default) -export ENABLE_NEMOTRON_3_NANO_THINKING=true - -# Disable reasoning -export ENABLE_NEMOTRON_3_NANO_THINKING=false -``` - -### Configure Thinking Budget (Optional) - -The 30B model supports a maximum thinking token limit to control the reasoning phase: - -```json -{ - "model": "nvidia/nemotron-3-nano-30b-a3b", - "messages": [ - { - "role": "user", - "content": "What is the capital of France?" - } - ], - "max_thinking_tokens": 8192 -} -``` - -**Thinking budget parameters:** - -**`max_thinking_tokens`** -: Maximum number of reasoning tokens allowed before generating the final answer. - -:::{important} -The key differences for the 30B model are the following: - -- Uses only `max_thinking_tokens` (not `min_thinking_tokens`) -- Reasoning is available in the model output's `reasoning_content` field (not wrapped in `` tags) -- The `reasoning_content` field is present in the model output but isn't exposed in the generate API response -- No filtering is needed because reasoning is already separated from the final answer -::: - -### Model Naming - -Use the correct model name based on your deployment: - -**Locally deployed NIMs** -: `nvidia/nemotron-3-nano` - -**NVIDIA-hosted models** -: `nvidia/nemotron-3-nano-30b-a3b` - ## Deploy with Reasoning Enabled After you configure reasoning settings in `prompt.yaml` or environment variables, redeploy your services: @@ -220,6 +256,7 @@ Adjust the thinking budget based on your use case: - **Lower values (1024-4096)**: Faster responses for simpler questions - **Higher values (8192-16384)**: More thorough reasoning for complex queries +- **Low-effort mode**: Use `LLM_LOW_EFFORT=true` for fast, low-cost reasoning when deep thought is not required ::: ## Related Topics diff --git a/src/nvidia_rag/rag_server/main.py b/src/nvidia_rag/rag_server/main.py index 044120926..3e9ddf810 100644 --- a/src/nvidia_rag/rag_server/main.py +++ b/src/nvidia_rag/rag_server/main.py @@ -262,7 +262,9 @@ def __init__( # Load prompts and other utilities self.prompts = get_prompts(prompts) self.vdb_top_k = int(self.config.retriever.vdb_top_k) - self.StreamingFilterThinkParser = get_streaming_filter_think_parser_async() + self.StreamingFilterThinkParser = get_streaming_filter_think_parser_async( + enable_thinking=self.config.llm.parameters.enable_thinking + ) if self._init_errors: logger.warning( @@ -1368,18 +1370,6 @@ def _handle_prompt_processing( conversation_history = [] user_message = [] - is_nemotron_v1 = str(model).endswith("llama-3.3-nemotron-super-49b-v1") - - # Nemotron controls thinking using system prompt, if nemotron v1 model is used update system prompt to enable/disable think - if is_nemotron_v1: - logger.info("Nemotron v1 model detected, updating system prompt") - if os.environ.get("ENABLE_NEMOTRON_THINKING", "false").lower() == "true": - logger.info("Setting system prompt as detailed thinking on") - system_prompt = "detailed thinking on" - else: - logger.info("Setting system prompt as detailed thinking off") - system_prompt = "detailed thinking off" - # Process chat history for message in chat_history: # Overwrite system message if provided in conversation history diff --git a/src/nvidia_rag/utils/configuration.py b/src/nvidia_rag/utils/configuration.py index 853fe20d7..3eb738ea0 100644 --- a/src/nvidia_rag/utils/configuration.py +++ b/src/nvidia_rag/utils/configuration.py @@ -412,15 +412,30 @@ class ModelParametersConfig(_ConfigBase): env="LLM_MIN_TOKENS", description="Minimum number of tokens to generate in response", ) + enable_thinking: bool = Field( + default=False, + env="LLM_ENABLE_THINKING", + description="Enable reasoning/thinking mode. Model emits reasoning tokens before the final answer.", + ) + reasoning_budget: int = Field( + default=0, + env="LLM_REASONING_BUDGET", + description="Token budget for reasoning (0 = no budget, model decides depth). Only used when enable_thinking is true.", + ) + low_effort: bool = Field( + default=False, + env="LLM_LOW_EFFORT", + description="Low-effort reasoning mode for faster, cheaper responses with shorter reasoning. Only used when enable_thinking is true.", + ) max_thinking_tokens: int = Field( default=0, env="LLM_MAX_THINKING_TOKENS", - description="Maximum thinking tokens to allocate for reasoning models (0 = disabled by default)", + description="Maximum thinking tokens for reasoning models. Used directly by nemotron-nano-9b-v2; for other models acts as an alternative to reasoning_budget (0 = disabled).", ) min_thinking_tokens: int = Field( default=0, env="LLM_MIN_THINKING_TOKENS", - description="Minimum thinking tokens to allocate for reasoning models (0 = disabled by default)", + description="Minimum thinking tokens for reasoning models. Only used by nemotron-nano-9b-v2 (0 = disabled).", ) ignore_eos: bool = Field( default=False, @@ -502,6 +517,9 @@ def get_model_parameters(self) -> dict: "min_tokens": self.parameters.min_tokens, "ignore_eos": self.parameters.ignore_eos, "max_tokens": self.parameters.max_tokens, + "enable_thinking": self.parameters.enable_thinking, + "reasoning_budget": self.parameters.reasoning_budget, + "low_effort": self.parameters.low_effort, "min_thinking_tokens": self.parameters.min_thinking_tokens, "max_thinking_tokens": self.parameters.max_thinking_tokens, "temperature": self.parameters.temperature, diff --git a/src/nvidia_rag/utils/llm.py b/src/nvidia_rag/utils/llm.py index 639c87d70..0148f3ce2 100644 --- a/src/nvidia_rag/utils/llm.py +++ b/src/nvidia_rag/utils/llm.py @@ -33,6 +33,7 @@ import yaml from langchain_core.language_models.llms import LLM from langchain_core.language_models.chat_models import SimpleChatModel +from langchain_core.messages import AIMessageChunk from langchain_nvidia_ai_endpoints import ChatNVIDIA from nvidia_rag.rag_server.response_generator import APIError, ErrorCodeMapping @@ -128,117 +129,112 @@ def _is_nvidia_endpoint(url: str | None) -> bool: return True -def _bind_thinking_tokens_if_configured( - llm: LLM | SimpleChatModel, **kwargs -) -> LLM | SimpleChatModel: - """ - If min_thinking_tokens or max_thinking_tokens are > 0 in kwargs, bind them to the LLM. - - Supports multiple reasoning/thinking model variants: - - 1. nvidia/nvidia-nemotron-nano-9b-v2: - - Uses min_thinking_tokens and max_thinking_tokens parameters - - Reasoning content is not available for this model - - 2. nemotron-3-nano variants (nemotron-3-nano-30b-a3b, nvidia/nemotron-3-nano): - - Uses reasoning_budget parameter (mapped from max_thinking_tokens) - - reasoning_budget is ONLY set when enable_thinking is true - - Outputs reasoning in a separate 'reasoning_content' field (not in content) - - Does NOT use tags - - Can be controlled via ENABLE_NEMOTRON_3_NANO_THINKING env var - - Raises: - ValueError: If min_thinking_tokens or max_thinking_tokens is passed but model - is not a supported Nemotron thinking model, or if any of these - parameters have invalid values (0 or negative). - """ - min_think = kwargs.get("min_thinking_tokens", None) - max_think = kwargs.get("max_thinking_tokens", None) - model = kwargs.get("model", None) +def _is_nemotron_3(model: str | None) -> bool: + """Detect Nemotron 3 model variants by checking for 'nemotron-3' in the model name.""" + if not model: + return False + return "nemotron-3" in model.lower() - # Validate model compatibility for thinking tokens - has_thinking_tokens = (min_think is not None and min_think > 0) or ( - max_think is not None and max_think > 0 - ) - if not has_thinking_tokens: - return llm +def _is_nemotron_3_nano(model: str | None) -> bool: + """Detect Nemotron 3 Nano models (30b-a3b and locally hosted variants).""" + if not model: + return False + m = model.lower() + return "nemotron-3-nano" in m - # Check if model is a supported reasoning model (various name formats) - # Note: For locally hosted models, use "nvidia/nemotron-3-nano" - # For NVIDIA-hosted models, use "nvidia/nemotron-3-nano-30b-a3b" - is_nano_9b_v2 = model and "nvidia/nvidia-nemotron-nano-9b-v2" in model - is_nemotron_3_nano = model and ( - "nemotron-3-nano" in model.lower() or - "nvidia/nemotron-3-nano" in model or - "nemotron-3-nano-30b-a3b" in model - ) - - if has_thinking_tokens and not (is_nano_9b_v2 or is_nemotron_3_nano): - raise ValueError( - "min_thinking_tokens and max_thinking_tokens are only supported for models " - "'nvidia/nvidia-nemotron-nano-9b-v2' and nemotron-3-nano variants " - "(e.g., 'nemotron-3-nano-30b-a3b', 'nvidia/nemotron-3-nano'), " - f"but got model '{model}'" - ) - bind_args = {} - if is_nano_9b_v2: - # nvidia/nvidia-nemotron-nano-9b-v2: Uses thinking token parameters directly - if min_think is not None and min_think > 0: - bind_args["min_thinking_tokens"] = min_think - else: - raise ValueError( - f"min_thinking_tokens must be a positive integer, but got {min_think}" - ) - if max_think is not None and max_think > 0: - bind_args["max_thinking_tokens"] = max_think - else: - raise ValueError( - f"max_thinking_tokens must be a positive integer, but got {max_think}" - ) - logger.info( - "nvidia-nemotron-nano-9b-v2: Setting min_thinking_tokens=%d, max_thinking_tokens=%d", - min_think, max_think +def _is_nemotron_nano_9b_v2(model: str | None) -> bool: + """Detect legacy Nemotron Nano 9B v2.""" + if not model: + return False + return "nvidia/nvidia-nemotron-nano-9b-v2" in model + + +def _resolve_enable_thinking(config: NvidiaRAGConfig | None = None, **kwargs) -> bool: + """Resolve enable_thinking from config, kwargs, or deprecated env var fallback.""" + if config is not None: + enable = config.llm.parameters.enable_thinking + if enable: + return True + enable = kwargs.get("enable_thinking", False) + if enable: + return True + deprecated = os.getenv("ENABLE_NEMOTRON_3_NANO_THINKING") + if deprecated is not None: + logger.warning( + "ENABLE_NEMOTRON_3_NANO_THINKING is deprecated, use LLM_ENABLE_THINKING instead" ) - elif is_nemotron_3_nano: - enable_thinking = os.getenv("ENABLE_NEMOTRON_3_NANO_THINKING", "true").lower() == "true" - if not enable_thinking: - raise ValueError( - "ENABLE_NEMOTRON_3_NANO_THINKING must be set to 'true' to use reasoning budget" - ) + return deprecated.lower() == "true" + return False - # For nemotron-3-nano variants, min_thinking_tokens is not supported - if min_think is not None and min_think > 0: - logger.warning( - "min_thinking_tokens is not supported for nemotron-3-nano variants, " - "only max_thinking_tokens (mapped to reasoning_budget or nvext) is supported" - ) - if max_think is not None and max_think > 0: - # Check if llm_endpoint is provided (locally hosted model) - llm_endpoint = kwargs.get("llm_endpoint", None) +def _bind_reasoning_config( + llm: LLM | SimpleChatModel, config: NvidiaRAGConfig | None = None, **kwargs +) -> LLM | SimpleChatModel: + """ + Bind reasoning parameters to the LLM based on model type and configuration. + + Reads enable_thinking, reasoning_budget, and low_effort from the config + object (LLM_ENABLE_THINKING, LLM_REASONING_BUDGET, LLM_LOW_EFFORT env vars). + kwargs can still override these for backward compatibility. + + Supports: + - Nemotron 3 variants: enable_thinking, reasoning_budget, low_effort via chat_template_kwargs + - Nemotron 3 Nano: enable_thinking + reasoning_budget (or nvext for local NIM) + - Nemotron Nano 9B v2: legacy min_thinking_tokens / max_thinking_tokens + - Other models: no reasoning features bound + """ + model = kwargs.get("model", "") + enable_thinking = _resolve_enable_thinking(config=config, **kwargs) + params = config.llm.parameters if config is not None else None + reasoning_budget = kwargs.get("reasoning_budget") or (params.reasoning_budget if params else 0) + low_effort = kwargs.get("low_effort") or (params.low_effort if params else False) + min_think = kwargs.get("min_thinking_tokens") or (params.min_thinking_tokens if params else 0) or 0 + max_think = kwargs.get("max_thinking_tokens") or (params.max_thinking_tokens if params else 0) or 0 + + # Check specific variants first, then fall through to the general nemotron-3 check + + if _is_nemotron_3_nano(model): + llm = llm.bind(chat_template_kwargs={"enable_thinking": enable_thinking}) + if enable_thinking and (reasoning_budget > 0 or max_think > 0): + budget = reasoning_budget if reasoning_budget > 0 else max_think + llm_endpoint = kwargs.get("llm_endpoint", "") if llm_endpoint: - # For locally hosted models, use nvext syntax - bind_args["nvext"] = {"max_thinking_tokens": max_think} - logger.info( - "nemotron-3-nano (locally hosted): Setting max_thinking_tokens=%d via nvext", - max_think - ) + llm = llm.bind(nvext={"max_thinking_tokens": budget}) + logger.info("nemotron-3-nano (local): enable_thinking=%s, nvext.max_thinking_tokens=%d", enable_thinking, budget) else: - # For API catalog models, use reasoning_budget - bind_args["reasoning_budget"] = max_think - logger.info( - "nemotron-3-nano (API catalog): Setting reasoning_budget=%d", - max_think - ) + llm = llm.bind(reasoning_budget=budget) + logger.info("nemotron-3-nano (API): enable_thinking=%s, reasoning_budget=%d", enable_thinking, budget) else: + logger.info("nemotron-3-nano: enable_thinking=%s", enable_thinking) + return llm + + if _is_nemotron_nano_9b_v2(model): + if min_think > 0 and max_think > 0: + llm = llm.bind(min_thinking_tokens=min_think, max_thinking_tokens=max_think) + logger.info("nemotron-nano-9b-v2: min_thinking_tokens=%d, max_thinking_tokens=%d", min_think, max_think) + elif min_think > 0 or max_think > 0: raise ValueError( - f"max_thinking_tokens must be a positive integer, but got {max_think}" + "nemotron-nano-9b-v2 requires both min_thinking_tokens and max_thinking_tokens " + f"to be positive, got min={min_think}, max={max_think}" ) + return llm + + if _is_nemotron_3(model): + template_kwargs: dict = {"enable_thinking": enable_thinking} + if enable_thinking and low_effort: + template_kwargs["low_effort"] = True + budget = reasoning_budget if reasoning_budget > 0 else max_think + if enable_thinking and budget > 0: + template_kwargs["reasoning_budget"] = budget + llm = llm.bind(chat_template_kwargs=template_kwargs) + logger.info( + "nemotron-3: enable_thinking=%s, reasoning_budget=%d, low_effort=%s", + enable_thinking, budget, low_effort, + ) + return llm - if bind_args: - return llm.bind(**bind_args) return llm @@ -289,16 +285,18 @@ def get_llm(config: NvidiaRAGConfig | None = None, **kwargs) -> LLM | SimpleChat default_headers = {**NVIDIA_API_DEFAULT_HEADERS} if api_key: default_headers["X-Model-Authorization"] = api_key - return ChatOpenAI( - model_name=kwargs.get("model"), - openai_api_base=f"{guardrails_url}/v1/guardrail", - openai_api_key="dummy-value", - default_headers=default_headers, - temperature=kwargs.get("temperature", None), - top_p=kwargs.get("top_p", None), - max_tokens=kwargs.get("max_tokens", None), - stop=kwargs.get("stop", []), - ) + openai_kwargs = { + "model_name": kwargs.get("model"), + "openai_api_base": f"{guardrails_url}/v1/guardrail", + "openai_api_key": "dummy-value", + "default_headers": default_headers, + "temperature": kwargs.get("temperature", None), + "top_p": kwargs.get("top_p", None), + "max_tokens": kwargs.get("max_tokens", None), + } + if kwargs.get("stop"): + openai_kwargs["stop"] = kwargs["stop"] + return ChatOpenAI(**openai_kwargs) except (requests.RequestException, requests.ConnectionError) as e: error_msg = f"Guardrails NIM unavailable at {guardrails_url}. Please verify the service is running and accessible." logger.exception( @@ -318,13 +316,15 @@ def get_llm(config: NvidiaRAGConfig | None = None, **kwargs) -> LLM | SimpleChat # Build kwargs dict, only including parameters that are set # For non-NVIDIA endpoints, exclude NVIDIA-specific parameters + # Do not pass stop=[] - some Nemotron 3 APIs reject empty stop arrays chat_nvidia_kwargs = { "base_url": url, "model": kwargs.get("model"), "api_key": api_key, - "stop": kwargs.get("stop", []), "default_headers": NVIDIA_API_DEFAULT_HEADERS, } + if kwargs.get("stop"): + chat_nvidia_kwargs["stop"] = kwargs["stop"] if kwargs.get("temperature") is not None: chat_nvidia_kwargs["temperature"] = kwargs["temperature"] if kwargs.get("top_p") is not None: @@ -342,15 +342,8 @@ def get_llm(config: NvidiaRAGConfig | None = None, **kwargs) -> LLM | SimpleChat chat_nvidia_kwargs["model_kwargs"] = model_kwargs llm = ChatNVIDIA(**chat_nvidia_kwargs) - # Only bind thinking tokens for NVIDIA endpoints if is_nvidia: - llm = _bind_thinking_tokens_if_configured(llm, **kwargs) - # For nemotron-3-nano models, set enable_thinking from env var - model = kwargs.get("model") - if model and ("nemotron-3-nano" in model.lower() or "nvidia/nemotron-3-nano" in model or "nemotron-3-nano-30b-a3b" in model): - enable_thinking = os.getenv("ENABLE_NEMOTRON_3_NANO_THINKING", "true").lower() == "true" - llm = llm.bind(chat_template_kwargs={"enable_thinking": enable_thinking}) - logger.info("nemotron-3-nano: Setting enable_thinking=%s (from ENABLE_NEMOTRON_3_NANO_THINKING)", enable_thinking) + llm = _bind_reasoning_config(llm, config=config, **kwargs) return llm logger.debug("Using llm model %s from api catalog", kwargs.get("model")) @@ -363,23 +356,20 @@ def get_llm(config: NvidiaRAGConfig | None = None, **kwargs) -> LLM | SimpleChat if kwargs.get("ignore_eos") is not None: model_kwargs["ignore_eos"] = kwargs["ignore_eos"] - llm = ChatNVIDIA( - model=kwargs.get("model"), - api_key=api_key, - temperature=kwargs.get("temperature", None), - top_p=kwargs.get("top_p", None), - max_completion_tokens=kwargs.get("max_tokens", None), - stop=kwargs.get("stop", []), - default_headers=NVIDIA_API_DEFAULT_HEADERS, + # Do not pass stop=[] - some Nemotron 3 APIs reject empty stop arrays + chat_nvidia_kwargs = { + "model": kwargs.get("model"), + "api_key": api_key, + "temperature": kwargs.get("temperature", None), + "top_p": kwargs.get("top_p", None), + "max_completion_tokens": kwargs.get("max_tokens", None), + "default_headers": NVIDIA_API_DEFAULT_HEADERS, **({"model_kwargs": model_kwargs} if model_kwargs else {}), - ) - llm = _bind_thinking_tokens_if_configured(llm, **kwargs) - # For nemotron-3-nano models, set enable_thinking from env var - model = kwargs.get("model") - if model and ("nemotron-3-nano" in model.lower() or "nvidia/nemotron-3-nano" in model or "nemotron-3-nano-30b-a3b" in model): - enable_thinking = os.getenv("ENABLE_NEMOTRON_3_NANO_THINKING", "true").lower() == "true" - llm = llm.bind(chat_template_kwargs={"enable_thinking": enable_thinking}) - logger.info("nemotron-3-nano: Setting enable_thinking=%s (from ENABLE_NEMOTRON_3_NANO_THINKING)", enable_thinking) + } + if kwargs.get("stop"): + chat_nvidia_kwargs["stop"] = kwargs["stop"] + llm = ChatNVIDIA(**chat_nvidia_kwargs) + llm = _bind_reasoning_config(llm, config=config, **kwargs) return llm raise RuntimeError( @@ -477,7 +467,8 @@ def streaming_filter_think(chunks: Iterable[str]) -> Iterable[str]: chunk_count = 0 for chunk in chunks: - content = chunk.content + reasoning, content = extract_reasoning_and_content(chunk) + content = content or reasoning chunk_count += 1 # Let's first check for full tags - this is the most reliable approach @@ -611,14 +602,20 @@ def get_streaming_filter_think_parser(): return RunnablePassthrough() -async def streaming_filter_think_async(chunks): +async def streaming_filter_think_async(chunks, enable_thinking: bool = False): """ Async version of streaming_filter_think. This async generator filters content between think tags in streaming LLM responses. It handles both complete tags in a single chunk and tags split across multiple tokens. + When enable_thinking is True and the model uses a separate reasoning_content field + (e.g. Nemotron 3), reasoning tokens are dropped and only content is forwarded. + The tag filter still runs to handle models that embed reasoning in content. + Args: chunks: Async iterable of chunks from a streaming LLM response + enable_thinking: When True, drop reasoning_content (genuine chain-of-thought). + When False, fall back to reasoning_content if content is empty (model quirk). Yields: str: Filtered content with think blocks removed @@ -644,7 +641,8 @@ async def streaming_filter_think_async(chunks): chunk_count = 0 async for chunk in chunks: - content = chunk.content + reasoning, content = extract_reasoning_and_content(chunk) + content = content if enable_thinking else (content or reasoning) chunk_count += 1 # Let's first check for full tags - this is the most reliable approach @@ -753,27 +751,61 @@ async def streaming_filter_think_async(chunks): ) -def get_streaming_filter_think_parser_async(): +async def _content_fallback_async(chunks, enable_thinking: bool = False): + """ + Pass through LLM chunks WITHOUT filtering thinking tokens. + Used when FILTER_THINK_TOKENS=false - the user wants to see everything. + + - When enable_thinking=true: forwards both reasoning_content and content so + the user can see the chain-of-thought followed by the answer. + - When enable_thinking=false: falls back to reasoning_content if content is + empty (NIM quirk where the answer lands in reasoning_content). + + Args: + chunks: Async iterable of LLM response chunks + enable_thinking: Whether the model is producing genuine reasoning tokens. + """ + async for chunk in chunks: + reasoning, content = extract_reasoning_and_content(chunk) + + if enable_thinking: + if reasoning: + yield AIMessageChunk(content=reasoning) + if content: + yield AIMessageChunk(content=content) + else: + text = content or reasoning + if text: + yield AIMessageChunk(content=text) + + +def get_streaming_filter_think_parser_async(enable_thinking: bool = False): """ Creates and returns an async RunnableGenerator for filtering think tokens. If FILTER_THINK_TOKENS environment variable is set to "true" (case-insensitive), returns a parser that filters out content between and tags. - Otherwise, returns a pass-through parser that doesn't modify the content. + Otherwise, returns a parser that normalizes content (content or reasoning_content) + so models like Nemotron 3 that put reply in reasoning_content still yield text. + + Args: + enable_thinking: When True, reasoning_content is genuine chain-of-thought and + will be dropped. When False, reasoning_content is used as a fallback if + content is empty (workaround for model quirk). Returns: - RunnableGenerator: An async parser for filtering (or not filtering) think tokens + RunnableGenerator: An async parser for filtering or content normalization """ + from functools import partial from langchain_core.runnables import RunnableGenerator, RunnablePassthrough # Check environment variable filter_enabled = os.getenv("FILTER_THINK_TOKENS", "true").lower() == "true" if filter_enabled: - logger.info("Think token filtering is enabled (async)") - return RunnableGenerator(streaming_filter_think_async) + logger.info("Think token filtering is enabled (async), enable_thinking=%s", enable_thinking) + return RunnableGenerator(partial(streaming_filter_think_async, enable_thinking=enable_thinking)) else: - logger.info("Think token filtering is disabled (async)") - # If filtering is disabled, use a passthrough that passes content as-is - return RunnablePassthrough() + logger.info("Think token filtering is disabled (async), enable_thinking=%s", enable_thinking) + return RunnableGenerator(partial(_content_fallback_async, enable_thinking=enable_thinking)) \ No newline at end of file diff --git a/tests/unit/test_rag_server/test_rag_main_advanced_features.py b/tests/unit/test_rag_server/test_rag_main_advanced_features.py index 4331586a7..30f4ace20 100644 --- a/tests/unit/test_rag_server/test_rag_main_advanced_features.py +++ b/tests/unit/test_rag_server/test_rag_main_advanced_features.py @@ -1192,26 +1192,20 @@ def test_handle_prompt_processing_basic(self): assert user_message == [("user", "Test human prompt")] def test_handle_prompt_processing_with_nemotron_v1_model(self): - """Test prompt processing with Nemotron v1 model.""" + """Test prompt processing with Nemotron v1 model uses the chat_template system prompt.""" rag = NvidiaRAG() chat_history = [] - # Mock instance attribute - mock_prompts = Mock() with patch.dict(os.environ, {"ENABLE_NEMOTRON_THINKING": "true"}): - mock_prompts.get.return_value = { - "system": "Test system prompt", - "human": "Test human prompt", - } - result = rag._handle_prompt_processing( chat_history, "llama-3.3-nemotron-super-49b-v1", "chat_template" ) assert len(result) == 3 system_message, conversation_history, user_message = result - assert system_message == [("system", "detailed thinking on")] + expected_system = rag.prompts.get("chat_template", {}).get("system", "") + assert system_message == [("system", expected_system)] def test_handle_prompt_processing_with_system_message_in_history(self): """Test prompt processing with system message in chat history.""" diff --git a/tests/unit/test_utils/test_configuration.py b/tests/unit/test_utils/test_configuration.py index 8c8f6651d..04c81ba9d 100644 --- a/tests/unit/test_utils/test_configuration.py +++ b/tests/unit/test_utils/test_configuration.py @@ -134,14 +134,16 @@ def test_get_model_parameters_default(self): config = LLMConfig() params = config.get_model_parameters() - # Default model contains "llama-3.3-nemotron-super-49b" so it triggers nemotron logic expected = { "min_tokens": 0, "ignore_eos": False, "max_tokens": 32768, + "enable_thinking": False, + "reasoning_budget": 0, + "low_effort": False, "min_thinking_tokens": 0, "max_thinking_tokens": 0, - "temperature": 0, + "temperature": 0.0, "top_p": 1.0, } assert params == expected @@ -151,14 +153,16 @@ def test_get_model_parameters_generic(self): config = LLMConfig(model_name="meta/llama-3.1-8b-instruct") params = config.get_model_parameters() - # Generic model should use the base parameter values expected = { "min_tokens": 0, "ignore_eos": False, "max_tokens": 32768, + "enable_thinking": False, + "reasoning_budget": 0, + "low_effort": False, "min_thinking_tokens": 0, "max_thinking_tokens": 0, - "temperature": 0, + "temperature": 0.0, "top_p": 1.0, } assert params == expected diff --git a/tests/unit/test_utils/test_llm.py b/tests/unit/test_utils/test_llm.py index f691edb9d..377f0b68d 100644 --- a/tests/unit/test_utils/test_llm.py +++ b/tests/unit/test_utils/test_llm.py @@ -240,7 +240,6 @@ def test_get_llm_nvidia_endpoints_with_url(self, mock_chatnvidia, mock_sanitize) base_url="http://test-url:8000", model="test-model", api_key="test-api-key", - stop=[], default_headers={"source": "rag-blueprint"}, temperature=0.7, top_p=0.9, @@ -272,7 +271,6 @@ def test_get_llm_nvidia_endpoints_api_catalog(self, mock_chatnvidia, mock_saniti temperature=None, top_p=None, max_completion_tokens=None, - stop=[], default_headers={"source": "rag-blueprint"}, ) @@ -325,7 +323,6 @@ def test_get_llm_with_guardrails_success( temperature=0.7, top_p=None, max_tokens=None, - stop=[], ) @patch("requests.get") @@ -419,7 +416,6 @@ def test_get_llm_none_parameters(self, mock_sanitize): temperature=None, top_p=None, max_completion_tokens=None, - stop=[], default_headers={"source": "rag-blueprint"}, model_kwargs={"ignore_eos": False}, ) @@ -432,6 +428,7 @@ def create_mock_chunk(self, content): """Helper to create mock chunk with content attribute.""" chunk = Mock() chunk.content = content + chunk.additional_kwargs = {} return chunk def test_streaming_filter_think_no_tags(self): @@ -683,7 +680,6 @@ def test_llm_creation_with_all_parameters(self, mock_chatnvidia): base_url="http://test:8000", model="meta/llama-3.1-8b-instruct", api_key="test-api-key", - stop=[], default_headers={"source": "rag-blueprint"}, temperature=0.7, top_p=0.9, @@ -829,64 +825,80 @@ def create_mock_chunk(self, content): """Helper to create mock chunk with content attribute.""" chunk = Mock() chunk.content = content + chunk.additional_kwargs = {} return chunk -class TestThinkingBudgetNemotron3Nano30B: - """Tests for thinking budget behavior with nvidia/nemotron-3-nano-30b-a3b.""" +class TestBindReasoningConfigNemotron3Nano: + """Tests for _bind_reasoning_config with nemotron-3-nano models.""" - @patch.dict(os.environ, {"ENABLE_NEMOTRON_3_NANO_THINKING": "true"}) - def test_bind_thinking_tokens_for_nemotron_30b_maps_reasoning_budget(self): - """max_thinking_tokens for nemotron-3-nano-30b-a3b maps to reasoning_budget.""" - from nvidia_rag.utils.llm import _bind_thinking_tokens_if_configured + @patch.dict(os.environ, {"LLM_ENABLE_THINKING": "true"}) + def test_bind_reasoning_config_nemotron_3_nano_with_budget(self): + """enable_thinking + reasoning_budget for nemotron-3-nano binds chat_template_kwargs and reasoning_budget.""" + from nvidia_rag.utils.llm import _bind_reasoning_config mock_llm = Mock() - bound_llm = _bind_thinking_tokens_if_configured( + mock_llm.bind.return_value = mock_llm + config = Mock() + config.llm.parameters.enable_thinking = True + config.llm.parameters.reasoning_budget = 8192 + config.llm.parameters.low_effort = False + config.llm.parameters.min_thinking_tokens = 0 + config.llm.parameters.max_thinking_tokens = 0 + + bound_llm = _bind_reasoning_config( mock_llm, + config=config, model="nvidia/nemotron-3-nano-30b-a3b", - max_thinking_tokens=8192, ) - mock_llm.bind.assert_called_once_with( - reasoning_budget=8192, + calls = mock_llm.bind.call_args_list + assert any( + call.kwargs.get("chat_template_kwargs", {}).get("enable_thinking") is True + for call in calls ) - assert bound_llm is mock_llm.bind.return_value - def test_min_thinking_tokens_alone_raises_for_nemotron_30b(self): - """min_thinking_tokens alone raises ValueError for nemotron-3-nano-30b-a3b (max_thinking_tokens required).""" - from nvidia_rag.utils.llm import _bind_thinking_tokens_if_configured + def test_bind_reasoning_config_unsupported_model_returns_original(self): + """Unsupported model returns original LLM without binding.""" + from nvidia_rag.utils.llm import _bind_reasoning_config mock_llm = Mock() - with pytest.raises(ValueError, match="max_thinking_tokens must be a positive integer"): - _bind_thinking_tokens_if_configured( - mock_llm, - model="nvidia/nemotron-3-nano-30b-a3b", - min_thinking_tokens=1, - ) - - def test_thinking_tokens_unsupported_model_raises(self): - """Using thinking tokens with unsupported model raises ValueError.""" - from nvidia_rag.utils.llm import _bind_thinking_tokens_if_configured + config = Mock() + config.llm.parameters.enable_thinking = False + config.llm.parameters.reasoning_budget = 0 + config.llm.parameters.low_effort = False + config.llm.parameters.min_thinking_tokens = 0 + config.llm.parameters.max_thinking_tokens = 0 + + bound_llm = _bind_reasoning_config( + mock_llm, + config=config, + model="meta/llama-3.1-8b-instruct", + ) - mock_llm = Mock() - with pytest.raises(ValueError): - _bind_thinking_tokens_if_configured( - mock_llm, - model="meta/llama-3.1-8b-instruct", - max_thinking_tokens=10, - ) + mock_llm.bind.assert_not_called() + assert bound_llm is mock_llm -class TestThinkingBudgetNemotronNano9B: - """Tests for thinking budget behavior with nvidia/nvidia-nemotron-nano-9b-v2.""" +class TestBindReasoningConfigNemotronNano9B: + """Tests for _bind_reasoning_config with nvidia/nvidia-nemotron-nano-9b-v2.""" - def test_bind_thinking_tokens_for_nano_9b_binds_min_and_max(self): + def test_bind_reasoning_config_nano_9b_binds_min_and_max(self): """Both min_thinking_tokens and max_thinking_tokens bind for nano-9b.""" - from nvidia_rag.utils.llm import _bind_thinking_tokens_if_configured + from nvidia_rag.utils.llm import _bind_reasoning_config mock_llm = Mock() - bound_llm = _bind_thinking_tokens_if_configured( + mock_llm.bind.return_value = mock_llm + config = Mock() + config.llm.parameters.enable_thinking = False + config.llm.parameters.reasoning_budget = 0 + config.llm.parameters.low_effort = False + config.llm.parameters.min_thinking_tokens = 1 + config.llm.parameters.max_thinking_tokens = 8192 + + bound_llm = _bind_reasoning_config( mock_llm, + config=config, model="nvidia/nvidia-nemotron-nano-9b-v2", min_thinking_tokens=1, max_thinking_tokens=8192, @@ -898,13 +910,21 @@ def test_bind_thinking_tokens_for_nano_9b_binds_min_and_max(self): ) assert bound_llm is mock_llm.bind.return_value - def test_no_thinking_tokens_for_nano_9b_returns_original_llm(self): + def test_bind_reasoning_config_nano_9b_no_tokens_returns_original(self): """If no thinking tokens are provided, nano-9b returns original LLM.""" - from nvidia_rag.utils.llm import _bind_thinking_tokens_if_configured + from nvidia_rag.utils.llm import _bind_reasoning_config mock_llm = Mock() - bound_llm = _bind_thinking_tokens_if_configured( + config = Mock() + config.llm.parameters.enable_thinking = False + config.llm.parameters.reasoning_budget = 0 + config.llm.parameters.low_effort = False + config.llm.parameters.min_thinking_tokens = 0 + config.llm.parameters.max_thinking_tokens = 0 + + bound_llm = _bind_reasoning_config( mock_llm, + config=config, model="nvidia/nvidia-nemotron-nano-9b-v2", ) From 1f72a5735a37c357bfd72fd66730c0dacdd3c89f Mon Sep 17 00:00:00 2001 From: Swapnil Masurekar Date: Mon, 2 Mar 2026 13:19:44 +0530 Subject: [PATCH 18/81] Concatenate multimodal content for VLM Embed (#362) (#392) Signed-off-by: Swapnil Masurekar --- src/nvidia_rag/rag_server/main.py | 53 ++++++---- .../test_rag_server/test_query_rewriting.py | 99 ++++++++++++++++++- .../test_rag_main_core_components.py | 5 +- .../test_rag_main_integration.py | 4 +- 4 files changed, 139 insertions(+), 22 deletions(-) diff --git a/src/nvidia_rag/rag_server/main.py b/src/nvidia_rag/rag_server/main.py index 3e9ddf810..55aaabede 100644 --- a/src/nvidia_rag/rag_server/main.py +++ b/src/nvidia_rag/rag_server/main.py @@ -1873,27 +1873,46 @@ def _build_retriever_query_from_content(self, content: Any) -> tuple[str, bool]: tuple[str, bool]: Query string that may include base64 image data for VLM embeddings bool: True if image URL is provided, False otherwise """ + is_image_query = False if isinstance(content, str): - return content, False + return content, is_image_query elif isinstance(content, list): - # Build multimodal query with both text and base64 images - query_parts = [] - for item in content: - if isinstance(item, dict): - if item.get("type") == "text": - text_content = item.get("text", "").strip() - if text_content: - query_parts.append(text_content) - elif item.get("type") == "image_url": - image_url = item.get("image_url", {}).get("url", "") - if image_url: - # If image URL is provided, return it as is - return image_url, True - # If no image URL is provided, return the text content - return "\n\n".join(query_parts), False + # Build multimodal query with both text and base64 images. + + # Process text types first, then image_url types. + text_items = [ + item for item in content + if isinstance(item, dict) and item.get("type") == "text" + ] + image_items = [ + item for item in content + if isinstance(item, dict) and item.get("type") == "image_url" + ] + + # Extract text and image parts in separate lists + text_parts = [] + image_parts = [] + for item in text_items: + text_content = item.get("text", "").strip() + if text_content: + text_parts.append(text_content) + for item in image_items: + image_url = item.get("image_url", {}).get("url", "") + if image_url: + image_parts.append(image_url) + is_image_query = True + break # only one image is supported + + text_query = "\n\n".join(text_parts) + if image_parts: + image_str = " ".join(image_parts) + final_query = (text_query + " " + image_str) if text_query else image_str + else: + final_query = text_query + return final_query, is_image_query else: # Fallback for any other content type - return (str(content) if content is not None else ""), False + return (str(content) if content is not None else ""), is_image_query async def _rag_chain( self, diff --git a/tests/unit/test_rag_server/test_query_rewriting.py b/tests/unit/test_rag_server/test_query_rewriting.py index 129d82124..78e010474 100644 --- a/tests/unit/test_rag_server/test_query_rewriting.py +++ b/tests/unit/test_rag_server/test_query_rewriting.py @@ -17,6 +17,8 @@ Test suite for query rewriting functionality in the RAG server. """ +from unittest.mock import AsyncMock, patch + import pytest @@ -64,6 +66,7 @@ class DummyVDB: """A minimal VDB stub used via monkeypatch on __prepare_vdb_op.""" last_query = None + last_retrieval_method = None def check_collection_exists(self, collection_name: str) -> bool: return True @@ -77,6 +80,15 @@ def get_metadata_schema(self, collection_name: str): def retrieval_langchain(self, query, collection_name, vectorstore=None, top_k=None, filter_expr="", otel_ctx=None): """Sync method - called in ThreadPoolExecutor or directly.""" DummyVDB.last_query = query + DummyVDB.last_retrieval_method = "langchain" + return [] + + def retrieval_image_langchain( + self, query, collection_name, vectorstore=None, top_k=None, reranker_top_k=None + ): + """Called when query contains images (multimodal).""" + DummyVDB.last_query = query + DummyVDB.last_retrieval_method = "image" return [] @@ -253,6 +265,43 @@ async def test_search_combines_history_when_multiturn_enabled(monkeypatch): assert fake_vdb.last_query == "What is RAG?. How does it work?" +@pytest.mark.asyncio +async def test_search_skips_query_rewriter_for_image_query(monkeypatch): + """When query is multimodal with image, query rewriting is skipped and retrieval_image_langchain is used.""" + from nvidia_rag.rag_server.main import NvidiaRAG + + monkeypatch.setenv("CONVERSATION_HISTORY", "5") + monkeypatch.setenv("ENABLE_REFLECTION", "false") + + fake_vdb = DummyVDB() + rag = NvidiaRAG() + monkeypatch.setattr(NvidiaRAG, "_prepare_vdb_op", lambda self, **kw: fake_vdb) + + multimodal_query = [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,x"}}, + ] + messages = [ + {"role": "user", "content": "Previous question"}, + {"role": "assistant", "content": "Previous answer"}, + ] + + await rag.search( + query=multimodal_query, + messages=messages, + collection_names=["test"], + enable_query_rewriting=True, + enable_reranker=False, + filter_expr="", + ) + + # Assert: query rewriting skipped - last_query is text + image URL (no "REWRITTEN(...)") + assert fake_vdb.last_query == "What is in this image? data:image/png;base64,x" + assert "REWRITTEN" not in str(fake_vdb.last_query) + # Assert: retrieval_image_langchain was used (not retrieval_langchain) + assert fake_vdb.last_retrieval_method == "image" + + @pytest.mark.asyncio async def test_generate_uses_query_rewriter_when_enabled(monkeypatch): """Test that query rewriting is used in generate when enabled with conversation history.""" @@ -319,6 +368,55 @@ async def test_generate_uses_only_current_query_when_history_disabled(monkeypatc assert fake_vdb.last_query == "How does it work?" +@pytest.mark.asyncio +async def test_generate_skips_query_rewriter_for_image_query(monkeypatch): + """When messages contain multimodal content with image, query rewriting is skipped.""" + from nvidia_rag.rag_server.main import NvidiaRAG + + monkeypatch.setenv("CONVERSATION_HISTORY", "5") + monkeypatch.setenv("ENABLE_REFLECTION", "false") + monkeypatch.setenv("MULTITURN_RETRIEVER_SIMPLE", "False") + + fake_vdb = DummyVDB() + rag = NvidiaRAG() + monkeypatch.setattr(NvidiaRAG, "_prepare_vdb_op", lambda self, **kw: fake_vdb) + + messages = [ + {"role": "user", "content": "What is RAG?"}, + {"role": "assistant", "content": "A retrieval-augmented framework."}, + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,x"}}, + ], + }, + ] + + async def _stream(*a, **k): + yield "ok" + + with patch("nvidia_rag.rag_server.main.VLM") as mock_vlm_class: + mock_vlm_instance = mock_vlm_class.return_value + mock_vlm_instance.stream_with_messages = _stream + + stream = await rag.generate( + messages=messages, + use_knowledge_base=True, + collection_names=["test"], + enable_query_rewriting=True, + enable_reranker=False, + enable_vlm_inference=True, + filter_expr="", + ) + + # Assert: query rewriting skipped - last_query is text + image URL (no "REWRITTEN(...)") + assert fake_vdb.last_query == "What is in this image? data:image/png;base64,x" + assert "REWRITTEN" not in str(fake_vdb.last_query) + # Assert: retrieval_image_langchain was used + assert fake_vdb.last_retrieval_method == "image" + + @pytest.mark.asyncio async def test_generate_combines_history_when_multiturn_enabled(monkeypatch): """Test that when multiturn_retrieval_simple is True, history is concatenated.""" @@ -353,4 +451,3 @@ async def test_generate_combines_history_when_multiturn_enabled(monkeypatch): # last previous user query is combined with current retriever_query # Expected concatenation: "What is RAG?. How does it work?" assert fake_vdb.last_query == "What is RAG?. How does it work?" - diff --git a/tests/unit/test_rag_server/test_rag_main_core_components.py b/tests/unit/test_rag_server/test_rag_main_core_components.py index cc14c7c08..d4567399c 100644 --- a/tests/unit/test_rag_server/test_rag_main_core_components.py +++ b/tests/unit/test_rag_server/test_rag_main_core_components.py @@ -416,8 +416,8 @@ def test_build_retriever_query_from_multimodal_list(self): ] result = rag._build_retriever_query_from_content(content) - # When image_url is present, the method returns the image URL - assert result == ("http://example.com/image.jpg", True) + # Text parts joined with \n\n first, then image URL with space separator + assert result == ("Hello\n\nworld http://example.com/image.jpg", True) def test_build_retriever_query_from_list_without_text(self): """Test building retriever query from list without text items.""" @@ -428,6 +428,7 @@ def test_build_retriever_query_from_list_without_text(self): ] result = rag._build_retriever_query_from_content(content) + # Image-only: no text, so final query is just the image URL (no leading space) assert result == ("http://example.com/image.jpg", True) def test_build_retriever_query_from_other_type(self): diff --git a/tests/unit/test_rag_server/test_rag_main_integration.py b/tests/unit/test_rag_server/test_rag_main_integration.py index 3e3cd8be6..9de59b630 100644 --- a/tests/unit/test_rag_server/test_rag_main_integration.py +++ b/tests/unit/test_rag_server/test_rag_main_integration.py @@ -445,8 +445,8 @@ def test_build_retriever_query_from_content_multimodal(self): ] result = rag._build_retriever_query_from_content(content) - # When image_url is present, the method returns the image URL - assert result == ("http://example.com/image.jpg", True) + # Text parts joined with \n\n first, then image URL with space separator + assert result == ("Hello\n\nworld http://example.com/image.jpg", True) def test_print_conversation_history(self): """Test __print_conversation_history method.""" From a1d1144823c514cca36943d5303e8b42a6baab26 Mon Sep 17 00:00:00 2001 From: Shubhadeep Das <149712532+shubhadeepd@users.noreply.github.com> Date: Mon, 2 Mar 2026 22:39:58 +0530 Subject: [PATCH 19/81] security: Fix frontend CVEs (#396) --- frontend/Dockerfile | 2 +- frontend/package.json | 7 + frontend/pnpm-lock.yaml | 276 ++++++++++++++++++++++++---------------- 3 files changed, 175 insertions(+), 110 deletions(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 1c0a6961d..c6f3e3b01 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -94,7 +94,7 @@ RUN if [ "$DOWNLOAD_LEGAL_COMPLIANCE" = "true" ] && [ -d /legal ]; then \ # Production stage - NVIDIA distroless (pre-approved) # Updated to latest version to address CVE-2025-9230 (libssl3) -FROM nvcr.io/nvidia/distroless/node:24-v3.1.3 +FROM nvcr.io/nvidia/distroless/node:24-v4.0.2 # Copy built application and config for production preview WORKDIR /app/frontend diff --git a/frontend/package.json b/frontend/package.json index fe77b7146..bdb3500f5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -51,5 +51,12 @@ }, "resolutions": { "@kui/foundations": "./src/assets/kui-foundations-react-external-0.504.1.tgz" + }, + "pnpm": { + "overrides": { + "rollup": ">=4.59.0", + "minimatch@3.1.2": "3.1.4", + "minimatch@9.0.5": "9.0.7" + } } } \ No newline at end of file diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml index ea8207293..cc1a633fc 100644 --- a/frontend/pnpm-lock.yaml +++ b/frontend/pnpm-lock.yaml @@ -6,6 +6,9 @@ settings: overrides: '@kui/foundations': ./src/assets/kui-foundations-react-external-0.504.1.tgz + rollup: '>=4.59.0' + minimatch@3.1.2: 3.1.4 + minimatch@9.0.5: 9.0.7 importers: @@ -1233,113 +1236,141 @@ packages: '@rolldown/pluginutils@1.0.0-beta.27': resolution: {integrity: sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==} - '@rollup/rollup-android-arm-eabi@4.53.3': - resolution: {integrity: sha512-mRSi+4cBjrRLoaal2PnqH82Wqyb+d3HsPUN/W+WslCXsZsyHa9ZeQQX/pQsZaVIWDkPcpV6jJ+3KLbTbgnwv8w==} + '@rollup/rollup-android-arm-eabi@4.59.0': + resolution: {integrity: sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==} cpu: [arm] os: [android] - '@rollup/rollup-android-arm64@4.53.3': - resolution: {integrity: sha512-CbDGaMpdE9sh7sCmTrTUyllhrg65t6SwhjlMJsLr+J8YjFuPmCEjbBSx4Z/e4SmDyH3aB5hGaJUP2ltV/vcs4w==} + '@rollup/rollup-android-arm64@4.59.0': + resolution: {integrity: sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==} cpu: [arm64] os: [android] - '@rollup/rollup-darwin-arm64@4.53.3': - resolution: {integrity: sha512-Nr7SlQeqIBpOV6BHHGZgYBuSdanCXuw09hon14MGOLGmXAFYjx1wNvquVPmpZnl0tLjg25dEdr4IQ6GgyToCUA==} + '@rollup/rollup-darwin-arm64@4.59.0': + resolution: {integrity: sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==} cpu: [arm64] os: [darwin] - '@rollup/rollup-darwin-x64@4.53.3': - resolution: {integrity: sha512-DZ8N4CSNfl965CmPktJ8oBnfYr3F8dTTNBQkRlffnUarJ2ohudQD17sZBa097J8xhQ26AwhHJ5mvUyQW8ddTsQ==} + '@rollup/rollup-darwin-x64@4.59.0': + resolution: {integrity: sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==} cpu: [x64] os: [darwin] - '@rollup/rollup-freebsd-arm64@4.53.3': - resolution: {integrity: sha512-yMTrCrK92aGyi7GuDNtGn2sNW+Gdb4vErx4t3Gv/Tr+1zRb8ax4z8GWVRfr3Jw8zJWvpGHNpss3vVlbF58DZ4w==} + '@rollup/rollup-freebsd-arm64@4.59.0': + resolution: {integrity: sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==} cpu: [arm64] os: [freebsd] - '@rollup/rollup-freebsd-x64@4.53.3': - resolution: {integrity: sha512-lMfF8X7QhdQzseM6XaX0vbno2m3hlyZFhwcndRMw8fbAGUGL3WFMBdK0hbUBIUYcEcMhVLr1SIamDeuLBnXS+Q==} + '@rollup/rollup-freebsd-x64@4.59.0': + resolution: {integrity: sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==} cpu: [x64] os: [freebsd] - '@rollup/rollup-linux-arm-gnueabihf@4.53.3': - resolution: {integrity: sha512-k9oD15soC/Ln6d2Wv/JOFPzZXIAIFLp6B+i14KhxAfnq76ajt0EhYc5YPeX6W1xJkAdItcVT+JhKl1QZh44/qw==} + '@rollup/rollup-linux-arm-gnueabihf@4.59.0': + resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} cpu: [arm] os: [linux] + libc: [glibc] - '@rollup/rollup-linux-arm-musleabihf@4.53.3': - resolution: {integrity: sha512-vTNlKq+N6CK/8UktsrFuc+/7NlEYVxgaEgRXVUVK258Z5ymho29skzW1sutgYjqNnquGwVUObAaxae8rZ6YMhg==} + '@rollup/rollup-linux-arm-musleabihf@4.59.0': + resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} cpu: [arm] os: [linux] + libc: [musl] - '@rollup/rollup-linux-arm64-gnu@4.53.3': - resolution: {integrity: sha512-RGrFLWgMhSxRs/EWJMIFM1O5Mzuz3Xy3/mnxJp/5cVhZ2XoCAxJnmNsEyeMJtpK+wu0FJFWz+QF4mjCA7AUQ3w==} + '@rollup/rollup-linux-arm64-gnu@4.59.0': + resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} cpu: [arm64] os: [linux] + libc: [glibc] - '@rollup/rollup-linux-arm64-musl@4.53.3': - resolution: {integrity: sha512-kASyvfBEWYPEwe0Qv4nfu6pNkITLTb32p4yTgzFCocHnJLAHs+9LjUu9ONIhvfT/5lv4YS5muBHyuV84epBo/A==} + '@rollup/rollup-linux-arm64-musl@4.59.0': + resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} cpu: [arm64] os: [linux] + libc: [musl] - '@rollup/rollup-linux-loong64-gnu@4.53.3': - resolution: {integrity: sha512-JiuKcp2teLJwQ7vkJ95EwESWkNRFJD7TQgYmCnrPtlu50b4XvT5MOmurWNrCj3IFdyjBQ5p9vnrX4JM6I8OE7g==} + '@rollup/rollup-linux-loong64-gnu@4.59.0': + resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} cpu: [loong64] os: [linux] + libc: [glibc] - '@rollup/rollup-linux-ppc64-gnu@4.53.3': - resolution: {integrity: sha512-EoGSa8nd6d3T7zLuqdojxC20oBfNT8nexBbB/rkxgKj5T5vhpAQKKnD+h3UkoMuTyXkP5jTjK/ccNRmQrPNDuw==} + '@rollup/rollup-linux-loong64-musl@4.59.0': + resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} + cpu: [loong64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-ppc64-gnu@4.59.0': + resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} cpu: [ppc64] os: [linux] + libc: [glibc] - '@rollup/rollup-linux-riscv64-gnu@4.53.3': - resolution: {integrity: sha512-4s+Wped2IHXHPnAEbIB0YWBv7SDohqxobiiPA1FIWZpX+w9o2i4LezzH/NkFUl8LRci/8udci6cLq+jJQlh+0g==} + '@rollup/rollup-linux-ppc64-musl@4.59.0': + resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} + cpu: [ppc64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-riscv64-gnu@4.59.0': + resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} cpu: [riscv64] os: [linux] + libc: [glibc] - '@rollup/rollup-linux-riscv64-musl@4.53.3': - resolution: {integrity: sha512-68k2g7+0vs2u9CxDt5ktXTngsxOQkSEV/xBbwlqYcUrAVh6P9EgMZvFsnHy4SEiUl46Xf0IObWVbMvPrr2gw8A==} + '@rollup/rollup-linux-riscv64-musl@4.59.0': + resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} cpu: [riscv64] os: [linux] + libc: [musl] - '@rollup/rollup-linux-s390x-gnu@4.53.3': - resolution: {integrity: sha512-VYsFMpULAz87ZW6BVYw3I6sWesGpsP9OPcyKe8ofdg9LHxSbRMd7zrVrr5xi/3kMZtpWL/wC+UIJWJYVX5uTKg==} + '@rollup/rollup-linux-s390x-gnu@4.59.0': + resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} cpu: [s390x] os: [linux] + libc: [glibc] - '@rollup/rollup-linux-x64-gnu@4.53.3': - resolution: {integrity: sha512-3EhFi1FU6YL8HTUJZ51imGJWEX//ajQPfqWLI3BQq4TlvHy4X0MOr5q3D2Zof/ka0d5FNdPwZXm3Yyib/UEd+w==} + '@rollup/rollup-linux-x64-gnu@4.59.0': + resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} cpu: [x64] os: [linux] + libc: [glibc] - '@rollup/rollup-linux-x64-musl@4.53.3': - resolution: {integrity: sha512-eoROhjcc6HbZCJr+tvVT8X4fW3/5g/WkGvvmwz/88sDtSJzO7r/blvoBDgISDiCjDRZmHpwud7h+6Q9JxFwq1Q==} + '@rollup/rollup-linux-x64-musl@4.59.0': + resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} cpu: [x64] os: [linux] + libc: [musl] + + '@rollup/rollup-openbsd-x64@4.59.0': + resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} + cpu: [x64] + os: [openbsd] - '@rollup/rollup-openharmony-arm64@4.53.3': - resolution: {integrity: sha512-OueLAWgrNSPGAdUdIjSWXw+u/02BRTcnfw9PN41D2vq/JSEPnJnVuBgw18VkN8wcd4fjUs+jFHVM4t9+kBSNLw==} + '@rollup/rollup-openharmony-arm64@4.59.0': + resolution: {integrity: sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==} cpu: [arm64] os: [openharmony] - '@rollup/rollup-win32-arm64-msvc@4.53.3': - resolution: {integrity: sha512-GOFuKpsxR/whszbF/bzydebLiXIHSgsEUp6M0JI8dWvi+fFa1TD6YQa4aSZHtpmh2/uAlj/Dy+nmby3TJ3pkTw==} + '@rollup/rollup-win32-arm64-msvc@4.59.0': + resolution: {integrity: sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==} cpu: [arm64] os: [win32] - '@rollup/rollup-win32-ia32-msvc@4.53.3': - resolution: {integrity: sha512-iah+THLcBJdpfZ1TstDFbKNznlzoxa8fmnFYK4V67HvmuNYkVdAywJSoteUszvBQ9/HqN2+9AZghbajMsFT+oA==} + '@rollup/rollup-win32-ia32-msvc@4.59.0': + resolution: {integrity: sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==} cpu: [ia32] os: [win32] - '@rollup/rollup-win32-x64-gnu@4.53.3': - resolution: {integrity: sha512-J9QDiOIZlZLdcot5NXEepDkstocktoVjkaKUtqzgzpt2yWjGlbYiKyp05rWwk4nypbYUNoFAztEgixoLaSETkg==} + '@rollup/rollup-win32-x64-gnu@4.59.0': + resolution: {integrity: sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==} cpu: [x64] os: [win32] - '@rollup/rollup-win32-x64-msvc@4.53.3': - resolution: {integrity: sha512-UhTd8u31dXadv0MopwGgNOBpUVROFKWVQgAg5N1ESyCz8AuBcMqm4AuTjrwgQKGDfoFuz02EuMRHQIw/frmYKQ==} + '@rollup/rollup-win32-x64-msvc@4.59.0': + resolution: {integrity: sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==} cpu: [x64] os: [win32] @@ -1381,24 +1412,28 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [glibc] '@tailwindcss/oxide-linux-arm64-musl@4.1.17': resolution: {integrity: sha512-HvZLfGr42i5anKtIeQzxdkw/wPqIbpeZqe7vd3V9vI3RQxe3xU1fLjss0TjyhxWcBaipk7NYwSrwTwK1hJARMg==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [musl] '@tailwindcss/oxide-linux-x64-gnu@4.1.17': resolution: {integrity: sha512-M3XZuORCGB7VPOEDH+nzpJ21XPvK5PyjlkSFkFziNHGLc5d6g3di2McAAblmaSUNl8IOmzYwLx9NsE7bplNkwQ==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [glibc] '@tailwindcss/oxide-linux-x64-musl@4.1.17': resolution: {integrity: sha512-k7f+pf9eXLEey4pBlw+8dgfJHY4PZ5qOUFDyNf7SI6lHjQ9Zt7+NcscjpwdCEbYi6FI5c2KDTDWyf2iHcCSyyQ==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [musl] '@tailwindcss/oxide-wasm32-wasi@4.1.17': resolution: {integrity: sha512-cEytGqSSoy7zK4JRWiTCx43FsKP/zGr0CsuMawhH67ONlH+T79VteQeJQRO/X7L0juEUA8ZyuYikcRBf0vsxhg==} @@ -1677,6 +1712,10 @@ packages: balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + balanced-match@4.0.4: + resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} + engines: {node: 18 || 20 || >=22} + baseline-browser-mapping@2.9.6: resolution: {integrity: sha512-v9BVVpOTLB59C9E7aSnmIF8h7qRsFpx+A2nugVMTszEOMcfjlZMsXRm4LF23I3Z9AJxc8ANpIvzbzONoX9VJlg==} hasBin: true @@ -1684,8 +1723,9 @@ packages: brace-expansion@1.1.12: resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} - brace-expansion@2.0.2: - resolution: {integrity: sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==} + brace-expansion@5.0.4: + resolution: {integrity: sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==} + engines: {node: 18 || 20 || >=22} browserslist@4.28.1: resolution: {integrity: sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==} @@ -2135,24 +2175,28 @@ packages: engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] + libc: [glibc] lightningcss-linux-arm64-musl@1.30.2: resolution: {integrity: sha512-5Vh9dGeblpTxWHpOx8iauV02popZDsCYMPIgiuw97OJ5uaDsL86cnqSFs5LZkG3ghHoX5isLgWzMs+eD1YzrnA==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] + libc: [musl] lightningcss-linux-x64-gnu@1.30.2: resolution: {integrity: sha512-Cfd46gdmj1vQ+lR6VRTTadNHu6ALuw2pKR9lYq4FnhvgBc4zWY1EtZcAc6EffShbb1MFrIPfLDXD6Xprbnni4w==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] + libc: [glibc] lightningcss-linux-x64-musl@1.30.2: resolution: {integrity: sha512-XJaLUUFXb6/QG2lGIW6aIk6jKdtjtcffUT0NKvIqhSBY3hh9Ch+1LCeH80dR9q9LBjG3ewbDjnumefsLsP6aiA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] + libc: [musl] lightningcss-win32-arm64-msvc@1.30.2: resolution: {integrity: sha512-FZn+vaj7zLv//D/192WFFVA0RgHawIcHqLX9xuWiQt7P0PtdFEVaxgF9rjM/IRYHQXNnk61/H/gb2Ei+kUQ4xQ==} @@ -2214,11 +2258,11 @@ packages: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} engines: {node: '>=4'} - minimatch@3.1.2: - resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} + minimatch@3.1.4: + resolution: {integrity: sha512-twmL+S8+7yIsE9wsqgzU3E8/LumN3M3QELrBZ20OdmQ9jB2JvW5oZtBEmft84k/Gs5CG9mqtWc6Y9vW+JEzGxw==} - minimatch@9.0.5: - resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} + minimatch@9.0.7: + resolution: {integrity: sha512-MOwgjc8tfrpn5QQEvjijjmDVtMw2oL88ugTevzxQnzRLm6l3fVEF2gzU0kYeYYKD8C66+IdGX6peJ4MyUlUnPg==} engines: {node: '>=16 || 14 >=14.17'} minipass@7.1.2: @@ -2400,8 +2444,8 @@ packages: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} - rollup@4.53.3: - resolution: {integrity: sha512-w8GmOxZfBmKknvdXU1sdM9NHcoQejwF/4mNgj2JuEEdRaHwwF12K7e9eXn1nLZ07ad+du76mkVsyeb2rKGllsA==} + rollup@4.59.0: + resolution: {integrity: sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true @@ -3031,7 +3075,7 @@ snapshots: dependencies: '@eslint/object-schema': 2.1.7 debug: 4.4.3 - minimatch: 3.1.2 + minimatch: 3.1.4 transitivePeerDependencies: - supports-color @@ -3052,7 +3096,7 @@ snapshots: ignore: 5.3.2 import-fresh: 3.3.1 js-yaml: 4.1.1 - minimatch: 3.1.2 + minimatch: 3.1.4 strip-json-comments: 3.1.1 transitivePeerDependencies: - supports-color @@ -3910,70 +3954,79 @@ snapshots: '@rolldown/pluginutils@1.0.0-beta.27': {} - '@rollup/rollup-android-arm-eabi@4.53.3': + '@rollup/rollup-android-arm-eabi@4.59.0': + optional: true + + '@rollup/rollup-android-arm64@4.59.0': + optional: true + + '@rollup/rollup-darwin-arm64@4.59.0': optional: true - '@rollup/rollup-android-arm64@4.53.3': + '@rollup/rollup-darwin-x64@4.59.0': optional: true - '@rollup/rollup-darwin-arm64@4.53.3': + '@rollup/rollup-freebsd-arm64@4.59.0': optional: true - '@rollup/rollup-darwin-x64@4.53.3': + '@rollup/rollup-freebsd-x64@4.59.0': optional: true - '@rollup/rollup-freebsd-arm64@4.53.3': + '@rollup/rollup-linux-arm-gnueabihf@4.59.0': optional: true - '@rollup/rollup-freebsd-x64@4.53.3': + '@rollup/rollup-linux-arm-musleabihf@4.59.0': optional: true - '@rollup/rollup-linux-arm-gnueabihf@4.53.3': + '@rollup/rollup-linux-arm64-gnu@4.59.0': optional: true - '@rollup/rollup-linux-arm-musleabihf@4.53.3': + '@rollup/rollup-linux-arm64-musl@4.59.0': optional: true - '@rollup/rollup-linux-arm64-gnu@4.53.3': + '@rollup/rollup-linux-loong64-gnu@4.59.0': optional: true - '@rollup/rollup-linux-arm64-musl@4.53.3': + '@rollup/rollup-linux-loong64-musl@4.59.0': optional: true - '@rollup/rollup-linux-loong64-gnu@4.53.3': + '@rollup/rollup-linux-ppc64-gnu@4.59.0': optional: true - '@rollup/rollup-linux-ppc64-gnu@4.53.3': + '@rollup/rollup-linux-ppc64-musl@4.59.0': optional: true - '@rollup/rollup-linux-riscv64-gnu@4.53.3': + '@rollup/rollup-linux-riscv64-gnu@4.59.0': optional: true - '@rollup/rollup-linux-riscv64-musl@4.53.3': + '@rollup/rollup-linux-riscv64-musl@4.59.0': optional: true - '@rollup/rollup-linux-s390x-gnu@4.53.3': + '@rollup/rollup-linux-s390x-gnu@4.59.0': optional: true - '@rollup/rollup-linux-x64-gnu@4.53.3': + '@rollup/rollup-linux-x64-gnu@4.59.0': optional: true - '@rollup/rollup-linux-x64-musl@4.53.3': + '@rollup/rollup-linux-x64-musl@4.59.0': optional: true - '@rollup/rollup-openharmony-arm64@4.53.3': + '@rollup/rollup-openbsd-x64@4.59.0': optional: true - '@rollup/rollup-win32-arm64-msvc@4.53.3': + '@rollup/rollup-openharmony-arm64@4.59.0': optional: true - '@rollup/rollup-win32-ia32-msvc@4.53.3': + '@rollup/rollup-win32-arm64-msvc@4.59.0': optional: true - '@rollup/rollup-win32-x64-gnu@4.53.3': + '@rollup/rollup-win32-ia32-msvc@4.59.0': optional: true - '@rollup/rollup-win32-x64-msvc@4.53.3': + '@rollup/rollup-win32-x64-gnu@4.59.0': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.59.0': optional: true '@tailwindcss/node@4.1.17': @@ -4198,7 +4251,7 @@ snapshots: '@typescript-eslint/types': 8.49.0 '@typescript-eslint/visitor-keys': 8.49.0 debug: 4.4.3 - minimatch: 9.0.5 + minimatch: 9.0.7 semver: 7.7.3 tinyglobby: 0.2.15 ts-api-utils: 2.1.0(typescript@5.8.3) @@ -4355,6 +4408,8 @@ snapshots: balanced-match@1.0.2: {} + balanced-match@4.0.4: {} + baseline-browser-mapping@2.9.6: {} brace-expansion@1.1.12: @@ -4362,9 +4417,9 @@ snapshots: balanced-match: 1.0.2 concat-map: 0.0.1 - brace-expansion@2.0.2: + brace-expansion@5.0.4: dependencies: - balanced-match: 1.0.2 + balanced-match: 4.0.4 browserslist@4.28.1: dependencies: @@ -4557,7 +4612,7 @@ snapshots: is-glob: 4.0.3 json-stable-stringify-without-jsonify: 1.0.1 lodash.merge: 4.6.2 - minimatch: 3.1.2 + minimatch: 3.1.4 natural-compare: 1.4.0 optionator: 0.9.4 optionalDependencies: @@ -4639,7 +4694,7 @@ snapshots: dependencies: foreground-child: 3.3.1 jackspeak: 3.4.3 - minimatch: 9.0.5 + minimatch: 9.0.7 minipass: 7.1.2 package-json-from-dist: 1.0.1 path-scurry: 1.11.1 @@ -4871,13 +4926,13 @@ snapshots: min-indent@1.0.1: {} - minimatch@3.1.2: + minimatch@3.1.4: dependencies: brace-expansion: 1.1.12 - minimatch@9.0.5: + minimatch@9.0.7: dependencies: - brace-expansion: 2.0.2 + brace-expansion: 5.0.4 minipass@7.1.2: {} @@ -5082,32 +5137,35 @@ snapshots: resolve-from@4.0.0: {} - rollup@4.53.3: + rollup@4.59.0: dependencies: '@types/estree': 1.0.8 optionalDependencies: - '@rollup/rollup-android-arm-eabi': 4.53.3 - '@rollup/rollup-android-arm64': 4.53.3 - '@rollup/rollup-darwin-arm64': 4.53.3 - '@rollup/rollup-darwin-x64': 4.53.3 - '@rollup/rollup-freebsd-arm64': 4.53.3 - '@rollup/rollup-freebsd-x64': 4.53.3 - '@rollup/rollup-linux-arm-gnueabihf': 4.53.3 - '@rollup/rollup-linux-arm-musleabihf': 4.53.3 - '@rollup/rollup-linux-arm64-gnu': 4.53.3 - '@rollup/rollup-linux-arm64-musl': 4.53.3 - '@rollup/rollup-linux-loong64-gnu': 4.53.3 - '@rollup/rollup-linux-ppc64-gnu': 4.53.3 - '@rollup/rollup-linux-riscv64-gnu': 4.53.3 - '@rollup/rollup-linux-riscv64-musl': 4.53.3 - '@rollup/rollup-linux-s390x-gnu': 4.53.3 - '@rollup/rollup-linux-x64-gnu': 4.53.3 - '@rollup/rollup-linux-x64-musl': 4.53.3 - '@rollup/rollup-openharmony-arm64': 4.53.3 - '@rollup/rollup-win32-arm64-msvc': 4.53.3 - '@rollup/rollup-win32-ia32-msvc': 4.53.3 - '@rollup/rollup-win32-x64-gnu': 4.53.3 - '@rollup/rollup-win32-x64-msvc': 4.53.3 + '@rollup/rollup-android-arm-eabi': 4.59.0 + '@rollup/rollup-android-arm64': 4.59.0 + '@rollup/rollup-darwin-arm64': 4.59.0 + '@rollup/rollup-darwin-x64': 4.59.0 + '@rollup/rollup-freebsd-arm64': 4.59.0 + '@rollup/rollup-freebsd-x64': 4.59.0 + '@rollup/rollup-linux-arm-gnueabihf': 4.59.0 + '@rollup/rollup-linux-arm-musleabihf': 4.59.0 + '@rollup/rollup-linux-arm64-gnu': 4.59.0 + '@rollup/rollup-linux-arm64-musl': 4.59.0 + '@rollup/rollup-linux-loong64-gnu': 4.59.0 + '@rollup/rollup-linux-loong64-musl': 4.59.0 + '@rollup/rollup-linux-ppc64-gnu': 4.59.0 + '@rollup/rollup-linux-ppc64-musl': 4.59.0 + '@rollup/rollup-linux-riscv64-gnu': 4.59.0 + '@rollup/rollup-linux-riscv64-musl': 4.59.0 + '@rollup/rollup-linux-s390x-gnu': 4.59.0 + '@rollup/rollup-linux-x64-gnu': 4.59.0 + '@rollup/rollup-linux-x64-musl': 4.59.0 + '@rollup/rollup-openbsd-x64': 4.59.0 + '@rollup/rollup-openharmony-arm64': 4.59.0 + '@rollup/rollup-win32-arm64-msvc': 4.59.0 + '@rollup/rollup-win32-ia32-msvc': 4.59.0 + '@rollup/rollup-win32-x64-gnu': 4.59.0 + '@rollup/rollup-win32-x64-msvc': 4.59.0 fsevents: 2.3.3 rrweb-cssom@0.8.0: {} @@ -5192,7 +5250,7 @@ snapshots: dependencies: '@istanbuljs/schema': 0.1.3 glob: 10.5.0 - minimatch: 9.0.5 + minimatch: 9.0.7 tinybench@2.9.0: {} @@ -5308,7 +5366,7 @@ snapshots: fdir: 6.5.0(picomatch@4.0.3) picomatch: 4.0.3 postcss: 8.5.6 - rollup: 4.53.3 + rollup: 4.59.0 tinyglobby: 0.2.15 optionalDependencies: '@types/node': 24.10.3 From de589b49c305959dd532b1da0c1f83510d4049a0 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Tue, 3 Mar 2026 01:01:22 +0530 Subject: [PATCH 20/81] Add config to enable nemotron parse only extraction in nv-ingest (#395) * Add config to enable nemotron parse only extraction in nv-ingest * Refactor nemotron parse only documentation * Remove nemotron parse only references from the previous section --- .../docker-compose-ingestor-server.yaml | 3 +- deploy/helm/nvidia-blueprint-rag/Chart.lock | 6 +- deploy/helm/nvidia-blueprint-rag/values.yaml | 1 + docs/nemotron-parse-extraction.md | 135 +++++++++++++++++- src/nvidia_rag/ingestor_server/nvingest.py | 2 + src/nvidia_rag/utils/configuration.py | 14 ++ 6 files changed, 154 insertions(+), 7 deletions(-) diff --git a/deploy/compose/docker-compose-ingestor-server.yaml b/deploy/compose/docker-compose-ingestor-server.yaml index be15c17dc..fa82c06c8 100644 --- a/deploy/compose/docker-compose-ingestor-server.yaml +++ b/deploy/compose/docker-compose-ingestor-server.yaml @@ -95,7 +95,8 @@ services: APP_NVINGEST_EXTRACTPAGEASIMAGE: ${APP_NVINGEST_EXTRACTPAGEASIMAGE:-False} APP_NVINGEST_STRUCTURED_ELEMENTS_MODALITY: ${APP_NVINGEST_STRUCTURED_ELEMENTS_MODALITY:-""} # Select from "image", "text_image" APP_NVINGEST_IMAGE_ELEMENTS_MODALITY: ${APP_NVINGEST_IMAGE_ELEMENTS_MODALITY:-""} # Select from "image" - APP_NVINGEST_PDFEXTRACTMETHOD: ${APP_NVINGEST_PDFEXTRACTMETHOD:-None} # Select from pdfium, nemoretron_parse, None + APP_NVINGEST_PDFEXTRACTMETHOD: ${APP_NVINGEST_PDFEXTRACTMETHOD:-None} # Select from pdfium, nemotron_parse, None + APP_NVINGEST_EXTRACTTABLESMETHOD: ${APP_NVINGEST_EXTRACTTABLESMETHOD:-yolox} # yolox, nemotron_parse, or None # Extract text by "page" only recommended for documents with pages like .pdf, .docx, etc. APP_NVINGEST_TEXTDEPTH: ${APP_NVINGEST_TEXTDEPTH:-page} # extract by "page" or "document" diff --git a/deploy/helm/nvidia-blueprint-rag/Chart.lock b/deploy/helm/nvidia-blueprint-rag/Chart.lock index 7b479e4aa..723660bfd 100644 --- a/deploy/helm/nvidia-blueprint-rag/Chart.lock +++ b/deploy/helm/nvidia-blueprint-rag/Chart.lock @@ -1,7 +1,7 @@ dependencies: - name: nv-ingest repository: https://helm.ngc.nvidia.com/nvidia/nemo-microservices - version: 26.1.1 + version: 26.1.2 - name: eck-elasticsearch repository: https://helm.elastic.co version: 0.18.0 @@ -14,5 +14,5 @@ dependencies: - name: kube-prometheus-stack repository: https://prometheus-community.github.io/helm-charts version: 76.3.0 -digest: sha256:7f85073bdf19922173b3372d9b5a877d6c2f783b431ce7a2f783308f67806c66 -generated: "2026-02-04T07:29:44.453434343Z" +digest: sha256:a65037bbcb6fa587af3d15b949a32b059cf26d1102a2166d0e77daed29a0f520 +generated: "2026-03-02T16:48:31.702049307+05:30" diff --git a/deploy/helm/nvidia-blueprint-rag/values.yaml b/deploy/helm/nvidia-blueprint-rag/values.yaml index 00c0ecd1d..af3c2166e 100644 --- a/deploy/helm/nvidia-blueprint-rag/values.yaml +++ b/deploy/helm/nvidia-blueprint-rag/values.yaml @@ -359,6 +359,7 @@ ingestor-server: # === NV-Ingest extraction configurations === APP_NVINGEST_PDFEXTRACTMETHOD: "None" # Method used for text extraction from "None", "pdfium", "nemotron_parse" + APP_NVINGEST_EXTRACTTABLESMETHOD: "yolox" # Method for table extraction: "yolox", "nemotron_parse", or None APP_NVINGEST_EXTRACTTEXT: "True" # Enable text extraction APP_NVINGEST_EXTRACTINFOGRAPHICS: "False" # Enable infographic extraction APP_NVINGEST_EXTRACTTABLES: "True" # Enable table extraction diff --git a/docs/nemotron-parse-extraction.md b/docs/nemotron-parse-extraction.md index a23dca7f4..c15cacf3f 100644 --- a/docs/nemotron-parse-extraction.md +++ b/docs/nemotron-parse-extraction.md @@ -62,7 +62,7 @@ When using NVIDIA hosted endpoints, you may encounter rate limiting with larger ## Using Helm -To enable PDF extraction with Nemotron Parse using Helm, you need to enable the Nemotron Parse service and configure the ingestor-server to use it. +To enable PDF extraction with Nemotron Parse using Helm, enable the Nemotron Parse service and configure the ingestor-server to use it. ### Prerequisites - Ensure you have sufficient GPU resources. Nemotron Parse requires a dedicated GPU. @@ -71,7 +71,7 @@ To enable PDF extraction with Nemotron Parse using Helm, you need to enable the To deploy with Nemotron Parse enabled: -Modify [`values.yaml`](../deploy/helm/nvidia-blueprint-rag/values.yaml) to enable Nemotron Parse: +Modify [`values.yaml`](../deploy/helm/nvidia-blueprint-rag/values.yaml) to enable Nemotron Parse and configure the ingestor-server: ```yaml # Enable Nemotron Parse NIM @@ -93,9 +93,136 @@ For detailed HELM deployment instructions, see [Helm Deployment Guide](deploy-he :::{note} **Key Configuration Changes:** - `nv-ingest.nimOperator.nemotron_parse.enabled=true` - Enables Nemotron Parse NIM -- `ingestor-server.envVars.APP_NVINGEST_PDFEXTRACTMETHOD="nemotron_parse"` - Configures ingestor to use Nemotron Parse +- `ingestor-server.envVars.APP_NVINGEST_PDFEXTRACTMETHOD="nemotron_parse"` - Configures ingestor to use Nemotron Parse for PDF extraction ::: +## Experimental: Nemotron-parse-only extraction + +:::{note} +The steps in this section describe a nemotron-parse-only pipeline. For production use, the default pipeline (Nemotron Parse with page-elements and table-structure NIMs) is recommended for better accuracy. +::: + +The **default** Nemotron Parse pipeline uses the **page-elements** and **table-structure** NIMs together with the Nemotron Parse NIM in the extraction pipeline. This combination provides better accuracy for PDF and table extraction. +To **experiment** with a nemotron-parse-only extraction pipeline (using only the Nemotron Parse NIM, without OCR, page-elements, graphic-elements, or table-structure NIMs), use the following steps. + +### Key configuration + +- **PDF extraction method** — Set `APP_NVINGEST_PDFEXTRACTMETHOD` to `nemotron_parse` so the ingestor uses Nemotron Parse for PDF text extraction. +- **Table extraction method** — Set `APP_NVINGEST_EXTRACTTABLESMETHOD` to `nemotron_parse` so the ingestor uses Nemotron Parse for table extraction instead of the default YOLOX-based table NIMs. This is required for a nemotron-parse-only pipeline. +- **nv-ingest health check** — Set `COMPONENTS_TO_READY_CHECK` to an empty string (`""`) in the **nv-ingest** service environment. By default, nv-ingest readiness waits for other ingest NIMs. With only Nemotron Parse running, the readiness probe would otherwise never pass. Emptying this value allows nv-ingest to become ready when only Nemotron Parse is available. + +### Using Docker Compose (nemotron-parse-only) + +#### On-prem models + +1. **Prerequisites**: Follow the [deployment guide](deploy-docker-self-hosted.md) up to and including the step labelled "Start all required NIMs." + +2. Start only the Nemotron Parse service (and any other non-ingest services your setup needs): + ```bash + USERID=$(id -u) docker compose --profile rag --profile nemotron-parse -f deploy/compose/nims.yaml up -d + ``` + You can skip the OCR, page-elements, graphic-elements, or table-structure NIMs if you want a nemotron-parse-only pipeline. + +3. Configure the ingestor-server and nv-ingest for nemotron-parse-only. Set these environment variables: + + **Ingestor-server** (ingestor-server environment): + ```bash + export APP_NVINGEST_PDFEXTRACTMETHOD=nemotron_parse + export APP_NVINGEST_EXTRACTTABLESMETHOD=nemotron_parse + ``` + + **nv-ingest** (nv-ingest service environment, e.g. in the compose file where nv-ingest runs): + ```bash + export COMPONENTS_TO_READY_CHECK="" + ``` + This ensures the nv-ingest readiness probe passes when other ingest NIMs are not running. + +4. Deploy the ingestion-server and rag-server containers following the remaining steps in the deployment guide. + +5. Ingest PDFs using the [ingestion API usage notebook](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/ingestion_api_usage.ipynb). + +#### NVIDIA hosted API endpoints + +1. **Prerequisites**: Follow the [deployment guide](deploy-docker-nvidia-hosted.md) up to and including the step labelled "Start the vector db containers from the repo root." + +2. Export variables for the Nemotron Parse API: + ```bash + export NEMOTRON_PARSE_HTTP_ENDPOINT=https://integrate.api.nvidia.com/v1/chat/completions + export NEMOTRON_PARSE_MODEL_NAME=nvidia/nemotron-parse + export NEMOTRON_PARSE_INFER_PROTOCOL=http + ``` + +3. Configure the ingestor-server and nv-ingest for nemotron-parse-only: + + **Ingestor-server**: + ```bash + export APP_NVINGEST_PDFEXTRACTMETHOD=nemotron_parse + export APP_NVINGEST_EXTRACTTABLESMETHOD=nemotron_parse + ``` + + **nv-ingest** (so readiness passes without other NIMs): + ```bash + export COMPONENTS_TO_READY_CHECK="" + ``` + +4. Deploy the ingestion-server and rag-server containers following the remaining steps in the deployment guide. + +5. Ingest PDFs using the [ingestion API usage notebook](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/ingestion_api_usage.ipynb). + +:::{note} +When using NVIDIA hosted endpoints, you may encounter rate limiting with larger file ingestions (>10 files). +::: + +### Using Helm (nemotron-parse-only) + +To run only Nemotron Parse for PDF and table extraction with Helm: + +1. **Prerequisites**: Ensure you have sufficient GPU resources. Nemotron Parse requires a dedicated GPU. + +2. Edit [`values.yaml`](../deploy/helm/nvidia-blueprint-rag/values.yaml): + + - **Enable Nemotron Parse** and **disable the other ingest NIMs** under `nv-ingest.nimOperator`: + + ```yaml + nv-ingest: + nimOperator: + nemotron_parse: + enabled: true + nemoretriever_ocr_v1: + enabled: false + graphic_elements: + enabled: false + page_elements: + enabled: false + table_structure: + enabled: false + envVars: + COMPONENTS_TO_READY_CHECK: "" + ``` + + - **Configure the ingestor-server** to use Nemotron Parse for both PDF and table extraction: + + ```yaml + ingestor-server: + envVars: + APP_NVINGEST_PDFEXTRACTMETHOD: "nemotron_parse" + APP_NVINGEST_EXTRACTTABLESMETHOD: "nemotron_parse" + ``` + +3. Apply the changes as described in [Change a Deployment](deploy-helm.md#change-a-deployment). + +4. For full Helm deployment steps, see the [Helm Deployment Guide](deploy-helm.md). + +**Summary of nemotron-parse-only Helm settings:** + +| Setting | Purpose | +|---------|---------| +| `nv-ingest.nimOperator.nemotron_parse.enabled: true` | Enable the Nemotron Parse NIM. | +| `nv-ingest.nimOperator..enabled: false` | Disable OCR, page-elements, graphic-elements, and table-structure NIMs. | +| `nv-ingest.envVars.COMPONENTS_TO_READY_CHECK: ""` | nv-ingest health check: readiness passes without other NIMs. | +| `ingestor-server.envVars.APP_NVINGEST_PDFEXTRACTMETHOD: "nemotron_parse"` | Use Nemotron Parse for PDF extraction. | +| `ingestor-server.envVars.APP_NVINGEST_EXTRACTTABLESMETHOD: "nemotron_parse"` | Use Nemotron Parse for table extraction. | + ## Limitations and Requirements When using Nemotron Parse for PDF extraction, consider the following: @@ -115,6 +242,8 @@ The `APP_NVINGEST_PDFEXTRACTMETHOD` environment variable supports the following - `pdfium`: Uses the default PDFium-based extraction - `None`: Uses the default extraction method +**Table extraction method:** The `APP_NVINGEST_EXTRACTTABLESMETHOD` environment variable controls how tables are extracted. Set it to `nemotron_parse` to use Nemotron Parse for table extraction (recommended for a nemotron-parse-only pipeline). The default is `yolox`, which uses the YOLOX-based table NIMs. + :::{note} The Nemotron Parse service requires GPU resources and must run on a dedicated GPU. Make sure you have sufficient GPU resources available before enabling this feature. ::: diff --git a/src/nvidia_rag/ingestor_server/nvingest.py b/src/nvidia_rag/ingestor_server/nvingest.py index 55e218c53..0f02aa1bb 100644 --- a/src/nvidia_rag/ingestor_server/nvingest.py +++ b/src/nvidia_rag/ingestor_server/nvingest.py @@ -141,6 +141,8 @@ def get_nv_ingest_ingestor( "extract_audio_params": {"segment_audio": config.nv_ingest.segment_audio}, "extract_page_as_image": config.nv_ingest.extract_page_as_image, } + if config.nv_ingest.extract_tables_method is not None: + extract_kwargs["extract_tables_method"] = config.nv_ingest.extract_tables_method if remove_extract_method or config.nv_ingest.pdf_extract_method is None: extract_kwargs.pop("extract_method", None) diff --git a/src/nvidia_rag/utils/configuration.py b/src/nvidia_rag/utils/configuration.py index 3eb738ea0..e1c9c1526 100644 --- a/src/nvidia_rag/utils/configuration.py +++ b/src/nvidia_rag/utils/configuration.py @@ -302,6 +302,20 @@ def normalize_pdf_extract_method(cls, v: Any) -> Any: env="APP_NVINGEST_TEXTDEPTH", description="Granularity level for text extraction (page, document)", ) + extract_tables_method: str | None = Field( + default=None, + env="APP_NVINGEST_EXTRACTTABLESMETHOD", + description="Method for table/chart extraction in PDFs (e.g. yolox, nemotron_parse). If None, client default is used.", + ) + + @field_validator("extract_tables_method", mode="before") + @classmethod + def normalize_extract_tables_method(cls, v: Any) -> Any: + """Normalize string 'None'/'none' to Python None.""" + if isinstance(v, str) and v.lower() in ("none", "null", ""): + return None + return v + tokenizer: str = Field( default="intfloat/e5-large-unsupervised", env="APP_NVINGEST_TOKENIZER", From 5cee48f035839bdfac24cf91d3266171895ddebf Mon Sep 17 00:00:00 2001 From: kumar-punit Date: Tue, 3 Mar 2026 15:15:19 +0530 Subject: [PATCH 21/81] Added patch command in rtx6000pro mig block also in documentation (#398) --- docs/mig-deployment.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/mig-deployment.md b/docs/mig-deployment.md index 149c25366..af5230e2c 100644 --- a/docs/mig-deployment.md +++ b/docs/mig-deployment.md @@ -158,6 +158,9 @@ Use [`mig-config-rtx6000.yaml`](../deploy/helm/mig-slicing/mig-config-rtx6000.ya ```bash kubectl apply -n nvidia-gpu-operator -f mig-slicing/mig-config-rtx6000.yaml +kubectl patch clusterpolicies.nvidia.com/cluster-policy \ + --type='json' \ + -p='[{"op":"replace", "path":"/spec/migManager/config/name", "value":"custom-mig-config"}]' kubectl label nodes nvidia.com/mig.config=custom-rtx6000-4x1g24-2x1g24-1x2g48-1x4g96 --overwrite ``` ::: From 60d3ca69daa85f0f0d765428335763d49ecd09be Mon Sep 17 00:00:00 2001 From: Swapnil Masurekar Date: Wed, 4 Mar 2026 11:34:27 +0530 Subject: [PATCH 22/81] Upgraded to GA nemotron-ranking-ms and nemotron-embedding-ms containers (#402) Signed-off-by: Swapnil Masurekar --- deploy/compose/nims.yaml | 4 ++-- deploy/helm/nvidia-blueprint-rag/values.yaml | 8 ++++---- deploy/workbench/compose.yaml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/deploy/compose/nims.yaml b/deploy/compose/nims.yaml index 261a705bc..eb29bfc06 100644 --- a/deploy/compose/nims.yaml +++ b/deploy/compose/nims.yaml @@ -33,7 +33,7 @@ services: nemotron-embedding-ms: container_name: nemotron-embedding-ms - image: nvcr.io/nvstaging/nim/llama-nemotron-embed-1b-v2:1.12.0-rc.20260216115852-6fba925dcb63c190 + image: nvcr.io/nim/nvidia/llama-nemotron-embed-1b-v2:1.13.0 volumes: - ${MODEL_DIRECTORY:-./}:/opt/nim/.cache ports: @@ -93,7 +93,7 @@ services: nemotron-ranking-ms: container_name: nemotron-ranking-ms - image: nvcr.io/nvstaging/nim/llama-nemotron-rerank-1b-v2:1.10.0-rc.20260219005237-7e2370934c6f9b9c + image: nvcr.io/nim/nvidia/llama-nemotron-rerank-1b-v2:1.10.0 volumes: - ${MODEL_DIRECTORY:-./}:/opt/nim/.cache ports: diff --git a/deploy/helm/nvidia-blueprint-rag/values.yaml b/deploy/helm/nvidia-blueprint-rag/values.yaml index af3c2166e..986caacc9 100644 --- a/deploy/helm/nvidia-blueprint-rag/values.yaml +++ b/deploy/helm/nvidia-blueprint-rag/values.yaml @@ -719,8 +719,8 @@ nimOperator: service: name: "nemotron-embedding-ms" image: - repository: nvcr.io/nvstaging/nim/llama-nemotron-embed-1b-v2 - tag: "1.12.0-rc.20260216115852-6fba925dcb63c190" + repository: nvcr.io/nim/nvidia/llama-nemotron-embed-1b-v2 + tag: "1.13.0" pullPolicy: IfNotPresent resources: limits: @@ -798,8 +798,8 @@ nimOperator: service: name: "nemotron-ranking-ms" image: - repository: nvcr.io/nvstaging/nim/llama-nemotron-rerank-1b-v2 - tag: "1.10.0-rc.20260219005237-7e2370934c6f9b9c" + repository: nvcr.io/nim/nvidia/llama-nemotron-rerank-1b-v2 + tag: "1.10.0" pullPolicy: IfNotPresent resources: limits: diff --git a/deploy/workbench/compose.yaml b/deploy/workbench/compose.yaml index aa9dd5713..c6867b9d6 100644 --- a/deploy/workbench/compose.yaml +++ b/deploy/workbench/compose.yaml @@ -30,7 +30,7 @@ services: nemotron-embedding-ms: container_name: nemotron-embedding-ms - image: nvcr.io/nvstaging/nim/llama-nemotron-embed-1b-v2:1.12.0-rc.20260216115852-6fba925dcb63c190 + image: nvcr.io/nim/nvidia/llama-nemotron-embed-1b-v2:1.13.0 volumes: - ${MODEL_DIRECTORY:-/tmp}:/opt/nim/.cache ports: @@ -60,7 +60,7 @@ services: nemotron-ranking-ms: container_name: nemotron-ranking-ms - image: nvcr.io/nvstaging/nim/llama-nemotron-rerank-1b-v2:1.10.0-rc.20260219005237-7e2370934c6f9b9c + image: nvcr.io/nim/nvidia/llama-nemotron-rerank-1b-v2:1.10.0 volumes: - ${MODEL_DIRECTORY:-/tmp}:/opt/nim/.cache ports: From 720aed393cf4dedd13bb04577c2bb0b5b05403c8 Mon Sep 17 00:00:00 2001 From: Shubhadeep Das <149712532+shubhadeepd@users.noreply.github.com> Date: Wed, 4 Mar 2026 11:59:15 +0530 Subject: [PATCH 23/81] Update packages to resolve source code CVEs (#400) * Update langchain-nvidia-ai-endpointsto >=1.1.0 * security: Update langgraph to version 1.10.0 --- examples/rag_react_agent/pyproject.toml | 2 +- examples/rag_react_agent/uv.lock | 372 ++++++++++++++++-------- pyproject.toml | 2 +- uv.lock | 20 +- 4 files changed, 258 insertions(+), 138 deletions(-) diff --git a/examples/rag_react_agent/pyproject.toml b/examples/rag_react_agent/pyproject.toml index fcebbcb3a..c4967a58a 100644 --- a/examples/rag_react_agent/pyproject.toml +++ b/examples/rag_react_agent/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ # Keep package version constraints as open as possible to avoid conflicts with other packages. Always define a minimum # version when adding a new package. If unsure, default to using `~=` instead of `==`. Does not apply to nvidia-nat packages. # Keep sorted!!! - "langgraph>=1.0.7", # Required for react_agent workflow + "langgraph>=1.0.8", # Required for react_agent workflow "langchain_classic", "nvidia-nat>=1.5.0a0,<2.0", # Allow pre-release versions "nvidia-nat-langchain>=1.5.0a0,<2.0", # Allow pre-release versions diff --git a/examples/rag_react_agent/uv.lock b/examples/rag_react_agent/uv.lock index 244c06a23..eee7b812b 100644 --- a/examples/rag_react_agent/uv.lock +++ b/examples/rag_react_agent/uv.lock @@ -324,6 +324,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cd/3a/577b549de0cc09d95f11087ee63c739bba856cd3952697eec4c4bb91350a/bleach-6.3.0-py3-none-any.whl", hash = "sha256:fe10ec77c93ddf3d13a73b035abaac7a9f5e436513864ccdad516693213c65d6", size = 164437, upload-time = "2025-10-27T17:57:37.538Z" }, ] +[[package]] +name = "blinker" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, +] + [[package]] name = "boto3" version = "1.40.61" @@ -597,21 +606,20 @@ name = "datasets" version = "4.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "dill" }, - { name = "filelock" }, - { name = "fsspec", extra = ["http"] }, - { name = "httpx" }, - { name = "huggingface-hub" }, - { name = "multiprocess" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "packaging" }, - { name = "pandas" }, - { name = "pyarrow" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "tqdm" }, - { name = "xxhash" }, + { name = "dill", marker = "python_full_version >= '3.12'" }, + { name = "filelock", marker = "python_full_version >= '3.12'" }, + { name = "fsspec", extra = ["http"], marker = "python_full_version >= '3.12'" }, + { name = "httpx", marker = "python_full_version >= '3.12'" }, + { name = "huggingface-hub", marker = "python_full_version >= '3.12'" }, + { name = "multiprocess", marker = "python_full_version >= '3.12'" }, + { name = "numpy", marker = "python_full_version >= '3.12'" }, + { name = "packaging", marker = "python_full_version >= '3.12'" }, + { name = "pandas", marker = "python_full_version >= '3.12'" }, + { name = "pyarrow", marker = "python_full_version >= '3.12'" }, + { name = "pyyaml", marker = "python_full_version >= '3.12'" }, + { name = "requests", marker = "python_full_version >= '3.12'" }, + { name = "tqdm", marker = "python_full_version >= '3.12'" }, + { name = "xxhash", marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/55/bf/bb927bde63d649296c83e883171ae77074717c1b80fe2868b328bd0dbcbb/datasets-4.5.0.tar.gz", hash = "sha256:00c698ce1c2452e646cc5fad47fef39d3fe78dd650a8a6eb205bb45eb63cd500", size = 588384, upload-time = "2026-01-14T18:27:54.297Z" } wheels = [ @@ -736,6 +744,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/79/1b8fa1bb3568781e84c9200f951c735f3f157429f44be0495da55894d620/filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25", size = 19970, upload-time = "2022-11-02T17:34:01.425Z" }, ] +[[package]] +name = "flask" +version = "3.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "blinker" }, + { name = "click" }, + { name = "itsdangerous" }, + { name = "jinja2" }, + { name = "markupsafe" }, + { name = "werkzeug" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/00/35d85dcce6c57fdc871f3867d465d780f302a175ea360f62533f12b27e2b/flask-3.1.3.tar.gz", hash = "sha256:0ef0e52b8a9cd932855379197dd8f94047b359ca0a78695144304cb45f87c9eb", size = 759004, upload-time = "2026-02-19T05:00:57.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", size = 103424, upload-time = "2026-02-19T05:00:56.027Z" }, +] + [[package]] name = "flatbuffers" version = "25.12.19" @@ -828,7 +853,7 @@ wheels = [ [package.optional-dependencies] http = [ - { name = "aiohttp" }, + { name = "aiohttp", marker = "python_full_version >= '3.12'" }, ] [[package]] @@ -1078,6 +1103,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, ] +[[package]] +name = "itsdangerous" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -1235,18 +1269,17 @@ wheels = [ [[package]] name = "langchain-aws" -version = "1.0.0" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "boto3" }, { name = "langchain-core" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/90/52/7e57fb7fc34c386625f66f0ab31da9cf2788b03ef15ae78ccd4c627b30cf/langchain_aws-1.0.0.tar.gz", hash = "sha256:597342bda0e7384e13590e9ab69c872ddcfbbf07d81ac6bb0f8a67970252212e", size = 214146, upload-time = "2025-10-17T19:06:49.001Z" } +sdist = { url = "https://files.pythonhosted.org/packages/52/1d/bb306951b1c394b7a27effb8eb6c9ee65dd77fcc4be7c20f76e3299a9e1e/langchain_aws-1.1.0.tar.gz", hash = "sha256:1e2f8570328eae4907c3cf7e900dc68d8034ddc865d9dc96823c9f9d8cccb901", size = 393899, upload-time = "2025-11-24T14:35:24.216Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/67/5d/5b3c07780a8eb4b916ffe504893896f87f318924c86dcbeb89562baa2d20/langchain_aws-1.0.0-py3-none-any.whl", hash = "sha256:68f6965b5030d0779b02e731ce1c910a5f4518bfe0e2ae82999a5342bc46dbd5", size = 150400, upload-time = "2025-10-17T19:06:47.926Z" }, + { url = "https://files.pythonhosted.org/packages/26/33/91b8d2a7570657b371382b45054142c54165a51706990a5c1b4cc40c0e9a/langchain_aws-1.1.0-py3-none-any.whl", hash = "sha256:8ec074615b42839e035354063717374c32c63f5028ef5221ba073fd5f3ef5e37", size = 152432, upload-time = "2025-11-24T14:35:23.004Z" }, ] [[package]] @@ -1278,8 +1311,7 @@ dependencies = [ { name = "langchain-classic" }, { name = "langchain-core" }, { name = "langsmith" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, { name = "pydantic-settings" }, { name = "pyyaml" }, { name = "requests" }, @@ -1293,7 +1325,7 @@ wheels = [ [[package]] name = "langchain-core" -version = "1.2.7" +version = "1.2.17" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonpatch" }, @@ -1305,9 +1337,23 @@ dependencies = [ { name = "typing-extensions" }, { name = "uuid-utils" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a2/0e/664d8d81b3493e09cbab72448d2f9d693d1fa5aa2bcc488602203a9b6da0/langchain_core-1.2.7.tar.gz", hash = "sha256:e1460639f96c352b4a41c375f25aeb8d16ffc1769499fb1c20503aad59305ced", size = 837039, upload-time = "2026-01-09T17:44:25.505Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/93/36226f593df52b871fc24d494c274f3a6b2ac76763a2806e7d35611634a1/langchain_core-1.2.17.tar.gz", hash = "sha256:54aa267f3311e347fb2e50951fe08e53761cebfb999ab80e6748d70525bbe872", size = 836130, upload-time = "2026-03-02T22:47:55.846Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/6f/34a9fba14d191a67f7e2ee3dbce3e9b86d2fa7310e2c7f2c713583481bd2/langchain_core-1.2.7-py3-none-any.whl", hash = "sha256:452f4fef7a3d883357b22600788d37e3d8854ef29da345b7ac7099f33c31828b", size = 490232, upload-time = "2026-01-09T17:44:24.236Z" }, + { url = "https://files.pythonhosted.org/packages/be/90/073f33ab383a62908eca7ea699586dfea280e77182176e33199c80ddf22a/langchain_core-1.2.17-py3-none-any.whl", hash = "sha256:bf6bd6ce503874e9c2da1669a69383e967c3de1ea808921d19a9a6bff1a9fbbe", size = 502727, upload-time = "2026-03-02T22:47:54.537Z" }, +] + +[[package]] +name = "langchain-huggingface" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "langchain-core" }, + { name = "tokenizers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/5b/4910551367de5c6ec246616fcc0ddb0bc6f9e5d353d4a22dcb5ab1f87e60/langchain_huggingface-1.2.1.tar.gz", hash = "sha256:33d52a30a56775380c6b4321b78136a410eb079132a80fe7120ddd4b954b4efa", size = 253106, upload-time = "2026-03-02T18:44:39.163Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/90/a1440bfa467a6dd9025ad80f3c239554de28aec49dacfb369fda92871556/langchain_huggingface-1.2.1-py3-none-any.whl", hash = "sha256:0930c216a457d2c8dc7b39a756c39c567f1d88593bfee2c3441f3ae718435f0f", size = 30924, upload-time = "2026-03-02T18:44:37.745Z" }, ] [[package]] @@ -1338,16 +1384,16 @@ wheels = [ [[package]] name = "langchain-nvidia-ai-endpoints" -version = "1.0.3" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, { name = "filetype" }, { name = "langchain-core" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5a/9e/30814da280f7a79b168f83180f6a0396c166f86a566e56bb9877bf562611/langchain_nvidia_ai_endpoints-1.0.3.tar.gz", hash = "sha256:11c48fd24e4a9d4c86c65bcef943400f4e709497c93254c7dc97c43f68c2be89", size = 46526, upload-time = "2026-01-28T22:04:33.93Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/d1/f72ec11097694e24d93268ab031c7ec56ab4bec1c43ef7814c659f3e2493/langchain_nvidia_ai_endpoints-1.1.0.tar.gz", hash = "sha256:048a3e6d7231365fdb9fff7bcff18ce6a516b25500681f51dcb69c39e82512a0", size = 47433, upload-time = "2026-02-25T21:48:16.87Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/67/04/c83f61106a245b74de11c1e075c1cc1e70462ece1dd9fc0584ad992a776d/langchain_nvidia_ai_endpoints-1.0.3-py3-none-any.whl", hash = "sha256:e5f170ad0a335637298bb90fb3df119793821e316355f61ab82f0106913eebbf", size = 50130, upload-time = "2026-01-28T22:04:33.065Z" }, + { url = "https://files.pythonhosted.org/packages/0b/22/5f07957028f7fa8c3d695934af8e7309bfd5ab43f2a7a756d3c3d6ce44f3/langchain_nvidia_ai_endpoints-1.1.0-py3-none-any.whl", hash = "sha256:eb04251b2b21facf9d6f2e6e7fa593b89e4f5023ebe3af1e02813512d1cd9687", size = 51514, upload-time = "2026-02-25T21:48:15.695Z" }, ] [[package]] @@ -1393,7 +1439,7 @@ wheels = [ [[package]] name = "langgraph" -version = "1.0.7" +version = "1.0.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, @@ -1403,9 +1449,9 @@ dependencies = [ { name = "pydantic" }, { name = "xxhash" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/72/5b/f72655717c04e33d3b62f21b166dc063d192b53980e9e3be0e2a117f1c9f/langgraph-1.0.7.tar.gz", hash = "sha256:0cfdfee51e6e8cfe503ecc7367c73933437c505b03fa10a85c710975c8182d9a", size = 497098, upload-time = "2026-01-22T16:57:47.303Z" } +sdist = { url = "https://files.pythonhosted.org/packages/55/92/14df6fefba28c10caf1cb05aa5b8c7bf005838fe32a86d903b6c7cc4018d/langgraph-1.0.10.tar.gz", hash = "sha256:73bd10ee14a8020f31ef07e9cd4c1a70c35cc07b9c2b9cd637509a10d9d51e29", size = 511644, upload-time = "2026-02-27T21:04:38.743Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/0e/fe80144e3e4048e5d19ccdb91ac547c1a7dc3da8dbd1443e210048194c14/langgraph-1.0.7-py3-none-any.whl", hash = "sha256:9d68e8f8dd8f3de2fec45f9a06de05766d9b075b78fb03171779893b7a52c4d2", size = 157353, upload-time = "2026-01-22T16:57:45.997Z" }, + { url = "https://files.pythonhosted.org/packages/5d/60/260e0c04620a37ba8916b712766c341cc5fc685dabc6948c899494bbc2ae/langgraph-1.0.10-py3-none-any.whl", hash = "sha256:7c298bef4f6ea292fcf9824d6088fe41a6727e2904ad6066f240c4095af12247", size = 160920, upload-time = "2026-02-27T21:04:35.932Z" }, ] [[package]] @@ -1423,15 +1469,15 @@ wheels = [ [[package]] name = "langgraph-prebuilt" -version = "1.0.7" +version = "1.0.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "langgraph-checkpoint" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a7/59/711aecd1a50999456850dc328f3cad72b4372d8218838d8d5326f80cb76f/langgraph_prebuilt-1.0.7.tar.gz", hash = "sha256:38e097e06de810de4d0e028ffc0e432bb56d1fb417620fb1dfdc76c5e03e4bf9", size = 163692, upload-time = "2026-01-22T16:45:22.801Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/06/dd61a5c2dce009d1b03b1d56f2a85b3127659fdddf5b3be5d8f1d60820fb/langgraph_prebuilt-1.0.8.tar.gz", hash = "sha256:0cd3cf5473ced8a6cd687cc5294e08d3de57529d8dd14fdc6ae4899549efcf69", size = 164442, upload-time = "2026-02-19T18:14:39.083Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/47/49/5e37abb3f38a17a3487634abc2a5da87c208cc1d14577eb8d7184b25c886/langgraph_prebuilt-1.0.7-py3-none-any.whl", hash = "sha256:e14923516504405bb5edc3977085bc9622c35476b50c1808544490e13871fe7c", size = 35324, upload-time = "2026-01-22T16:45:21.784Z" }, + { url = "https://files.pythonhosted.org/packages/dc/41/ec966424ad3f2ed3996d24079d3342c8cd6c0bd0653c12b2a917a685ec6c/langgraph_prebuilt-1.0.8-py3-none-any.whl", hash = "sha256:d16a731e591ba4470f3e313a319c7eee7dbc40895bcf15c821f985a3522a7ce0", size = 35648, upload-time = "2026-02-19T18:14:37.611Z" }, ] [[package]] @@ -1592,20 +1638,20 @@ name = "mcp" version = "1.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "httpx" }, - { name = "httpx-sse" }, - { name = "jsonschema" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "pyjwt", extra = ["crypto"] }, - { name = "python-multipart" }, - { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "sse-starlette" }, - { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, - { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, + { name = "anyio", marker = "python_full_version >= '3.12'" }, + { name = "httpx", marker = "python_full_version >= '3.12'" }, + { name = "httpx-sse", marker = "python_full_version >= '3.12'" }, + { name = "jsonschema", marker = "python_full_version >= '3.12'" }, + { name = "pydantic", marker = "python_full_version >= '3.12'" }, + { name = "pydantic-settings", marker = "python_full_version >= '3.12'" }, + { name = "pyjwt", extra = ["crypto"], marker = "python_full_version >= '3.12'" }, + { name = "python-multipart", marker = "python_full_version >= '3.12'" }, + { name = "pywin32", marker = "python_full_version >= '3.12' and sys_platform == 'win32'" }, + { name = "sse-starlette", marker = "python_full_version >= '3.12'" }, + { name = "starlette", marker = "python_full_version >= '3.12'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.12'" }, + { name = "typing-inspection", marker = "python_full_version >= '3.12'" }, + { name = "uvicorn", marker = "python_full_version >= '3.12' and sys_platform != 'emscripten'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fc/6d/62e76bbb8144d6ed86e202b5edd8a4cb631e7c8130f3f4893c3f90262b10/mcp-1.26.0.tar.gz", hash = "sha256:db6e2ef491eecc1a0d93711a76f28dec2e05999f93afd48795da1c1137142c66", size = 608005, upload-time = "2026-01-24T19:40:32.468Z" } wheels = [ @@ -1746,7 +1792,7 @@ name = "multiprocess" version = "0.70.18" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "dill" }, + { name = "dill", marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/72/fd/2ae3826f5be24c6ed87266bc4e59c46ea5b059a103f3d7e7eb76a52aeecb/multiprocess-0.70.18.tar.gz", hash = "sha256:f9597128e6b3e67b23956da07cf3d2e5cba79e2f4e0fba8d7903636663ec6d0d", size = 1798503, upload-time = "2025-04-17T03:11:27.742Z" } wheels = [ @@ -1770,6 +1816,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "narwhals" +version = "2.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/75/59/81d0f4cad21484083466f278e6b392addd9f4205b48d45b5c8771670ebf8/narwhals-2.17.0.tar.gz", hash = "sha256:ebd5bc95bcfa2f8e89a8ac09e2765a63055162837208e67b42d6eeb6651d5e67", size = 620306, upload-time = "2026-02-23T09:44:34.142Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/27/20770bd6bf8fbe1e16f848ba21da9df061f38d2e6483952c29d2bb5d1d8b/narwhals-2.17.0-py3-none-any.whl", hash = "sha256:2ac5307b7c2b275a7d66eeda906b8605e3d7a760951e188dcfff86e8ebe083dd", size = 444897, upload-time = "2026-02-23T09:44:32.006Z" }, +] + [[package]] name = "nest-asyncio" version = "1.6.0" @@ -1797,41 +1852,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, ] -[[package]] -name = "numpy" -version = "1.26.4" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.12'", -] -sdist = { url = "https://files.pythonhosted.org/packages/65/6e/09db70a523a96d25e115e71cc56a6f9031e7b8cd166c1ac8438307c14058/numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010", size = 15786129, upload-time = "2024-02-06T00:26:44.495Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/11/57/baae43d14fe163fa0e4c47f307b6b2511ab8d7d30177c491960504252053/numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71", size = 20630554, upload-time = "2024-02-05T23:51:50.149Z" }, - { url = "https://files.pythonhosted.org/packages/1a/2e/151484f49fd03944c4a3ad9c418ed193cfd02724e138ac8a9505d056c582/numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef", size = 13997127, upload-time = "2024-02-05T23:52:15.314Z" }, - { url = "https://files.pythonhosted.org/packages/79/ae/7e5b85136806f9dadf4878bf73cf223fe5c2636818ba3ab1c585d0403164/numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e", size = 14222994, upload-time = "2024-02-05T23:52:47.569Z" }, - { url = "https://files.pythonhosted.org/packages/3a/d0/edc009c27b406c4f9cbc79274d6e46d634d139075492ad055e3d68445925/numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5", size = 18252005, upload-time = "2024-02-05T23:53:15.637Z" }, - { url = "https://files.pythonhosted.org/packages/09/bf/2b1aaf8f525f2923ff6cfcf134ae5e750e279ac65ebf386c75a0cf6da06a/numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a", size = 13885297, upload-time = "2024-02-05T23:53:42.16Z" }, - { url = "https://files.pythonhosted.org/packages/df/a0/4e0f14d847cfc2a633a1c8621d00724f3206cfeddeb66d35698c4e2cf3d2/numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a", size = 18093567, upload-time = "2024-02-05T23:54:11.696Z" }, - { url = "https://files.pythonhosted.org/packages/d2/b7/a734c733286e10a7f1a8ad1ae8c90f2d33bf604a96548e0a4a3a6739b468/numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20", size = 5968812, upload-time = "2024-02-05T23:54:26.453Z" }, - { url = "https://files.pythonhosted.org/packages/3f/6b/5610004206cf7f8e7ad91c5a85a8c71b2f2f8051a0c0c4d5916b76d6cbb2/numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2", size = 15811913, upload-time = "2024-02-05T23:54:53.933Z" }, - { url = "https://files.pythonhosted.org/packages/95/12/8f2020a8e8b8383ac0177dc9570aad031a3beb12e38847f7129bacd96228/numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218", size = 20335901, upload-time = "2024-02-05T23:55:32.801Z" }, - { url = "https://files.pythonhosted.org/packages/75/5b/ca6c8bd14007e5ca171c7c03102d17b4f4e0ceb53957e8c44343a9546dcc/numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b", size = 13685868, upload-time = "2024-02-05T23:55:56.28Z" }, - { url = "https://files.pythonhosted.org/packages/79/f8/97f10e6755e2a7d027ca783f63044d5b1bc1ae7acb12afe6a9b4286eac17/numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b", size = 13925109, upload-time = "2024-02-05T23:56:20.368Z" }, - { url = "https://files.pythonhosted.org/packages/0f/50/de23fde84e45f5c4fda2488c759b69990fd4512387a8632860f3ac9cd225/numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed", size = 17950613, upload-time = "2024-02-05T23:56:56.054Z" }, - { url = "https://files.pythonhosted.org/packages/4c/0c/9c603826b6465e82591e05ca230dfc13376da512b25ccd0894709b054ed0/numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a", size = 13572172, upload-time = "2024-02-05T23:57:21.56Z" }, - { url = "https://files.pythonhosted.org/packages/76/8c/2ba3902e1a0fc1c74962ea9bb33a534bb05984ad7ff9515bf8d07527cadd/numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0", size = 17786643, upload-time = "2024-02-05T23:57:56.585Z" }, - { url = "https://files.pythonhosted.org/packages/28/4a/46d9e65106879492374999e76eb85f87b15328e06bd1550668f79f7b18c6/numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110", size = 5677803, upload-time = "2024-02-05T23:58:08.963Z" }, - { url = "https://files.pythonhosted.org/packages/16/2e/86f24451c2d530c88daf997cb8d6ac622c1d40d19f5a031ed68a4b73a374/numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818", size = 15517754, upload-time = "2024-02-05T23:58:36.364Z" }, -] - [[package]] name = "numpy" version = "2.4.1" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.13'", - "python_full_version == '3.12.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/24/62/ae72ff66c0f1fd959925b4c11f8c2dea61f47f6acaea75a08512cdfe3fed/numpy-2.4.1.tar.gz", hash = "sha256:a1ceafc5042451a858231588a104093474c6a5c57dcc724841f5c888d237d690", size = 20721320, upload-time = "2026-01-10T06:44:59.619Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a5/34/2b1bc18424f3ad9af577f6ce23600319968a70575bd7db31ce66731bbef9/numpy-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0cce2a669e3c8ba02ee563c7835f92c153cf02edff1ae05e1823f1dde21b16a5", size = 16944563, upload-time = "2026-01-10T06:42:14.615Z" }, @@ -1890,62 +1914,124 @@ wheels = [ name = "nvidia-nat" version = "1.5.0a20260112" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", +] +dependencies = [ + { name = "aioboto3", marker = "python_full_version >= '3.12'" }, + { name = "authlib", marker = "python_full_version >= '3.12'" }, + { name = "click", marker = "python_full_version >= '3.12'" }, + { name = "colorama", marker = "python_full_version >= '3.12'" }, + { name = "datasets", marker = "python_full_version >= '3.12'" }, + { name = "expandvars", marker = "python_full_version >= '3.12'" }, + { name = "fastapi", marker = "python_full_version >= '3.12'" }, + { name = "httpx", marker = "python_full_version >= '3.12'" }, + { name = "jinja2", marker = "python_full_version >= '3.12'" }, + { name = "jsonpath-ng", marker = "python_full_version >= '3.12'" }, + { name = "mcp", marker = "python_full_version >= '3.12'" }, + { name = "nest-asyncio2", marker = "python_full_version >= '3.12'" }, + { name = "networkx", marker = "python_full_version >= '3.12'" }, + { name = "numpy", marker = "python_full_version >= '3.12'" }, + { name = "openinference-semantic-conventions", marker = "python_full_version >= '3.12'" }, + { name = "openpyxl", marker = "python_full_version >= '3.12'" }, + { name = "optuna", marker = "python_full_version >= '3.12'" }, + { name = "pip", marker = "python_full_version >= '3.12'" }, + { name = "pkce", marker = "python_full_version >= '3.12'" }, + { name = "pkginfo", marker = "python_full_version >= '3.12'" }, + { name = "platformdirs", marker = "python_full_version >= '3.12'" }, + { name = "pydantic", marker = "python_full_version >= '3.12'" }, + { name = "pymilvus", marker = "python_full_version >= '3.12'" }, + { name = "python-dotenv", marker = "python_full_version >= '3.12'" }, + { name = "pyyaml", marker = "python_full_version >= '3.12'" }, + { name = "ragas", marker = "python_full_version >= '3.12'" }, + { name = "rich", marker = "python_full_version >= '3.12'" }, + { name = "tabulate", marker = "python_full_version >= '3.12'" }, + { name = "uvicorn", extra = ["standard"], marker = "python_full_version >= '3.12'" }, + { name = "wikipedia", marker = "python_full_version >= '3.12'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/e0/c7426ed15d1eb528eb0c9135efb66da033b0a56b63f42d4099b2fe05fd24/nvidia_nat-1.5.0a20260112-py3-none-any.whl", hash = "sha256:3d05c948efe0e3ab58e3d7a58ab90510d1a1128eb678810e1ef62efc5dfc9681", size = 950027, upload-time = "2026-01-12T10:46:15.705Z" }, +] + +[[package]] +name = "nvidia-nat" +version = "1.5.0a20260223" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.12'", +] +dependencies = [ + { name = "nvidia-nat-core", marker = "python_full_version < '3.12'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/7e/6e984de1473e8264d5cf6598d14f1c01f6dabf22f2fedda5f8e97140ae05/nvidia_nat-1.5.0a20260223-py3-none-any.whl", hash = "sha256:137461b310af90ed12e0496bac90ddb62297b00287707c80df48208437e2502a", size = 52704, upload-time = "2026-02-23T10:04:57.955Z" }, +] + +[[package]] +name = "nvidia-nat-core" +version = "1.5.0a20260223" +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aioboto3" }, { name = "authlib" }, { name = "click" }, { name = "colorama" }, - { name = "datasets" }, { name = "expandvars" }, { name = "fastapi" }, + { name = "flask" }, { name = "httpx" }, { name = "jinja2" }, { name = "jsonpath-ng" }, - { name = "mcp" }, { name = "nest-asyncio2" }, { name = "networkx" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, { name = "openinference-semantic-conventions" }, - { name = "openpyxl" }, { name = "optuna" }, + { name = "pandas" }, { name = "pip" }, { name = "pkce" }, { name = "pkginfo" }, { name = "platformdirs" }, + { name = "plotly" }, { name = "pydantic" }, + { name = "pyjwt" }, { name = "pymilvus" }, { name = "python-dotenv" }, + { name = "python-multipart" }, { name = "pyyaml" }, - { name = "ragas" }, { name = "rich" }, { name = "tabulate" }, + { name = "urllib3" }, { name = "uvicorn", extra = ["standard"] }, { name = "wikipedia" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/e0/c7426ed15d1eb528eb0c9135efb66da033b0a56b63f42d4099b2fe05fd24/nvidia_nat-1.5.0a20260112-py3-none-any.whl", hash = "sha256:3d05c948efe0e3ab58e3d7a58ab90510d1a1128eb678810e1ef62efc5dfc9681", size = 950027, upload-time = "2026-01-12T10:46:15.705Z" }, + { url = "https://files.pythonhosted.org/packages/b1/23/b043caadf08a72e4eb2c95bb65fed5083e7bf40af48ea92305fabf3b2820/nvidia_nat_core-1.5.0a20260223-py3-none-any.whl", hash = "sha256:5262cae48d66efbd53f98134e7820759121a1b4398b339e1d14d307ed2195a21", size = 762259, upload-time = "2026-02-23T10:01:44.692Z" }, ] [[package]] name = "nvidia-nat-langchain" -version = "1.5.0a20260112" +version = "1.5.0a20260223" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain" }, { name = "langchain-aws" }, { name = "langchain-classic" }, + { name = "langchain-community" }, { name = "langchain-core" }, + { name = "langchain-huggingface" }, { name = "langchain-litellm" }, { name = "langchain-milvus" }, { name = "langchain-nvidia-ai-endpoints" }, { name = "langchain-openai" }, { name = "langchain-tavily" }, { name = "langgraph" }, - { name = "nvidia-nat" }, + { name = "nvidia-nat-core" }, + { name = "openevals" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/84/2a/7a2cd2e7444ef03bdebb6c9637e63a9eee33da84e7c23baceb18f83f2250/nvidia_nat_langchain-1.5.0a20260112-py3-none-any.whl", hash = "sha256:cba64b0192d589f325cbbc2de60da8eb514efc27b49157b3eafca204ab989a55", size = 60925, upload-time = "2026-01-12T10:43:35.414Z" }, + { url = "https://files.pythonhosted.org/packages/98/65/e565dc570ecfdf4c4ca34d0d873794d33fdf11a8889a9b2b1a78ad15b589/nvidia_nat_langchain-1.5.0a20260223-py3-none-any.whl", hash = "sha256:87c70294c1f38fcd09252a79dd5e9038aee73326678dd9d7519b8064b914d7e4", size = 160480, upload-time = "2026-02-23T10:06:08.62Z" }, ] [[package]] @@ -2016,16 +2102,16 @@ requires-dist = [ { name = "langchain-elasticsearch", marker = "extra == 'all'", specifier = ">=0.3" }, { name = "langchain-elasticsearch", marker = "extra == 'elasticsearch'", specifier = ">=0.3" }, { name = "langchain-milvus", specifier = ">=0.3.0" }, - { name = "langchain-nvidia-ai-endpoints", specifier = ">=1.0.3" }, + { name = "langchain-nvidia-ai-endpoints", specifier = ">=1.1.0" }, { name = "langchain-openai", marker = "extra == 'all'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'ingest'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'rag'", specifier = ">=0.2" }, { name = "lark", specifier = ">=1.2.2" }, { name = "minio", specifier = ">=7.2,<8.0" }, - { name = "nv-ingest-api", marker = "extra == 'all'", specifier = "==26.1.1" }, - { name = "nv-ingest-api", marker = "extra == 'ingest'", specifier = "==26.1.1" }, - { name = "nv-ingest-client", marker = "extra == 'all'", specifier = "==26.1.1" }, - { name = "nv-ingest-client", marker = "extra == 'ingest'", specifier = "==26.1.1" }, + { name = "nv-ingest-api", marker = "extra == 'all'", specifier = "==26.1.2" }, + { name = "nv-ingest-api", marker = "extra == 'ingest'", specifier = "==26.1.2" }, + { name = "nv-ingest-client", marker = "extra == 'all'", specifier = "==26.1.2" }, + { name = "nv-ingest-client", marker = "extra == 'ingest'", specifier = "==26.1.2" }, { name = "opentelemetry-api", marker = "extra == 'all'", specifier = ">=1.29,<2.0" }, { name = "opentelemetry-api", marker = "extra == 'ingest'", specifier = ">=1.29,<2.0" }, { name = "opentelemetry-api", marker = "extra == 'rag'", specifier = ">=1.29,<2.0" }, @@ -2094,8 +2180,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "coloredlogs" }, { name = "flatbuffers" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, { name = "packaging" }, { name = "protobuf" }, { name = "sympy" }, @@ -2129,6 +2214,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/df/c306f7375d42bafb379934c2df4c2fa3964656c8c782bac75ee10c102818/openai-2.15.0-py3-none-any.whl", hash = "sha256:6ae23b932cd7230f7244e52954daa6602716d6b9bf235401a107af731baea6c3", size = 1067879, upload-time = "2026-01-09T22:10:06.446Z" }, ] +[[package]] +name = "openevals" +version = "0.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "langchain" }, + { name = "langchain-openai" }, + { name = "langsmith" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/37/31e23ef661fa4c3c6a3c979afd884b30205512b4dde680b36d5909550500/openevals-0.1.3.tar.gz", hash = "sha256:9b00df1a7738464676aa887d4d950b77d3ef7024f6e8a54be3a83c82f485ea65", size = 100828, upload-time = "2025-12-18T04:09:03.034Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/68/162b0d273ffef5b0ad557ebccb790725bf94d78969702324dd5726828cf0/openevals-0.1.3-py3-none-any.whl", hash = "sha256:aed448df0cfdded732e24cda026eda065435a71ffb8c406a3ce73e590156d9f9", size = 67802, upload-time = "2025-12-18T04:09:01.59Z" }, +] + [[package]] name = "openinference-semantic-conventions" version = "0.1.25" @@ -2143,7 +2243,7 @@ name = "openpyxl" version = "3.1.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "et-xmlfile" }, + { name = "et-xmlfile", marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } wheels = [ @@ -2392,8 +2492,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "alembic" }, { name = "colorlog" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, { name = "packaging" }, { name = "pyyaml" }, { name = "sqlalchemy" }, @@ -2506,8 +2605,7 @@ name = "pandas" version = "2.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, { name = "python-dateutil" }, { name = "pytz" }, { name = "tzdata" }, @@ -2668,6 +2766,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/28/3bfe2fa5a7b9c46fe7e13c97bda14c895fb10fa2ebf1d0abb90e0cea7ee1/platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31", size = 18731, upload-time = "2025-12-05T13:52:56.823Z" }, ] +[[package]] +name = "plotly" +version = "6.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "narwhals" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/24/fb/41efe84970cfddefd4ccf025e2cbfafe780004555f583e93dba3dac2cdef/plotly-6.6.0.tar.gz", hash = "sha256:b897f15f3b02028d69f755f236be890ba950d0a42d7dfc619b44e2d8cea8748c", size = 7027956, upload-time = "2026-03-02T21:10:25.321Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/d2/c6e44dba74f17c6216ce1b56044a9b93a929f1c2d5bdaff892512b260f5e/plotly-6.6.0-py3-none-any.whl", hash = "sha256:8d6daf0f87412e0c0bfe72e809d615217ab57cc715899a1e5145135a7800d1d0", size = 9910315, upload-time = "2026-03-02T21:10:18.131Z" }, +] + [[package]] name = "ply" version = "3.11" @@ -2963,7 +3074,7 @@ wheels = [ [package.optional-dependencies] crypto = [ - { name = "cryptography" }, + { name = "cryptography", marker = "python_full_version >= '3.12'" }, ] [[package]] @@ -2994,8 +3105,7 @@ name = "pymilvus-model" version = "0.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, { name = "onnxruntime" }, { name = "protobuf" }, { name = "scipy" }, @@ -3142,7 +3252,8 @@ source = { editable = "." } dependencies = [ { name = "langchain-classic" }, { name = "langgraph" }, - { name = "nvidia-nat" }, + { name = "nvidia-nat", version = "1.5.0a20260112", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "nvidia-nat", version = "1.5.0a20260223", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "nvidia-nat-langchain" }, { name = "nvidia-rag", extra = ["rag"] }, { name = "transformers" }, @@ -3151,7 +3262,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "langchain-classic" }, - { name = "langgraph", specifier = ">=1.0.7" }, + { name = "langgraph", specifier = ">=1.0.8" }, { name = "nvidia-nat", specifier = ">=1.5.0a0,<2.0" }, { name = "nvidia-nat-langchain", specifier = ">=1.5.0a0,<2.0" }, { name = "nvidia-rag", extras = ["rag"], editable = "../../" }, @@ -3163,19 +3274,18 @@ name = "ragas" version = "0.2.15" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "appdirs" }, - { name = "datasets" }, - { name = "diskcache" }, - { name = "langchain" }, - { name = "langchain-community" }, - { name = "langchain-core" }, - { name = "langchain-openai" }, - { name = "nest-asyncio" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "openai" }, - { name = "pydantic" }, - { name = "tiktoken" }, + { name = "appdirs", marker = "python_full_version >= '3.12'" }, + { name = "datasets", marker = "python_full_version >= '3.12'" }, + { name = "diskcache", marker = "python_full_version >= '3.12'" }, + { name = "langchain", marker = "python_full_version >= '3.12'" }, + { name = "langchain-community", marker = "python_full_version >= '3.12'" }, + { name = "langchain-core", marker = "python_full_version >= '3.12'" }, + { name = "langchain-openai", marker = "python_full_version >= '3.12'" }, + { name = "nest-asyncio", marker = "python_full_version >= '3.12'" }, + { name = "numpy", marker = "python_full_version >= '3.12'" }, + { name = "openai", marker = "python_full_version >= '3.12'" }, + { name = "pydantic", marker = "python_full_version >= '3.12'" }, + { name = "tiktoken", marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6c/0f/04fddfa94744b1c3d8901aed8832a6b4193cc8e4886881f1bb88ff055350/ragas-0.2.15.tar.gz", hash = "sha256:2d0cd77b315a9c9c02ceb0a19ca8a48e82e1d02416587a2944ea51e6e327cd7b", size = 40867766, upload-time = "2025-04-24T16:39:28.734Z" } wheels = [ @@ -3438,8 +3548,7 @@ name = "scipy" version = "1.17.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/56/3e/9cca699f3486ce6bc12ff46dc2031f1ec8eb9ccc9a320fdaf925f1417426/scipy-1.17.0.tar.gz", hash = "sha256:2591060c8e648d8b96439e111ac41fd8342fdeff1876be2e19dea3fe8930454e", size = 30396830, upload-time = "2026-01-10T21:34:23.009Z" } wheels = [ @@ -3569,7 +3678,7 @@ name = "sse-starlette" version = "3.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, + { name = "anyio", marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/db/3c/fa6517610dc641262b77cc7bf994ecd17465812c1b0585fe33e11be758ab/sse_starlette-3.0.3.tar.gz", hash = "sha256:88cfb08747e16200ea990c8ca876b03910a23b547ab3bd764c0d8eb81019b971", size = 21943, upload-time = "2025-10-30T18:44:20.117Z" } wheels = [ @@ -3703,8 +3812,7 @@ version = "5.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, - { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy" }, { name = "packaging" }, { name = "pyyaml" }, { name = "regex" }, @@ -3976,6 +4084,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598, upload-time = "2026-01-10T09:23:45.395Z" }, ] +[[package]] +name = "werkzeug" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/f1/ee81806690a87dab5f5653c1f146c92bc066d7f4cebc603ef88eb9e13957/werkzeug-3.1.6.tar.gz", hash = "sha256:210c6bede5a420a913956b4791a7f4d6843a43b6fcee4dfa08a65e93007d0d25", size = 864736, upload-time = "2026-02-19T15:17:18.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/ec/d58832f89ede95652fd01f4f24236af7d32b70cab2196dfcc2d2fd13c5c2/werkzeug-3.1.6-py3-none-any.whl", hash = "sha256:7ddf3357bb9564e407607f988f683d72038551200c704012bb9a4c523d42f131", size = 225166, upload-time = "2026-02-19T15:17:17.475Z" }, +] + [[package]] name = "wikipedia" version = "1.4.0" diff --git a/pyproject.toml b/pyproject.toml index 6baa185b2..4b1faf01f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "langchain>=1.2.7", "langchain-community>=0.4", "langchain-milvus>=0.3.0", - "langchain-nvidia-ai-endpoints>=1.0.3", + "langchain-nvidia-ai-endpoints>=1.1.0", "minio>=7.2,<8.0", "pdfplumber>=0.11.9", "pydantic>=2.11,<3.0", diff --git a/uv.lock b/uv.lock index 9e77bddac..3c72857d4 100644 --- a/uv.lock +++ b/uv.lock @@ -1309,16 +1309,16 @@ wheels = [ [[package]] name = "langchain-nvidia-ai-endpoints" -version = "1.0.3" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, { name = "filetype" }, { name = "langchain-core" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5a/9e/30814da280f7a79b168f83180f6a0396c166f86a566e56bb9877bf562611/langchain_nvidia_ai_endpoints-1.0.3.tar.gz", hash = "sha256:11c48fd24e4a9d4c86c65bcef943400f4e709497c93254c7dc97c43f68c2be89", size = 46526, upload-time = "2026-01-28T22:04:33.93Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/d1/f72ec11097694e24d93268ab031c7ec56ab4bec1c43ef7814c659f3e2493/langchain_nvidia_ai_endpoints-1.1.0.tar.gz", hash = "sha256:048a3e6d7231365fdb9fff7bcff18ce6a516b25500681f51dcb69c39e82512a0", size = 47433, upload-time = "2026-02-25T21:48:16.87Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/67/04/c83f61106a245b74de11c1e075c1cc1e70462ece1dd9fc0584ad992a776d/langchain_nvidia_ai_endpoints-1.0.3-py3-none-any.whl", hash = "sha256:e5f170ad0a335637298bb90fb3df119793821e316355f61ab82f0106913eebbf", size = 50130, upload-time = "2026-01-28T22:04:33.065Z" }, + { url = "https://files.pythonhosted.org/packages/0b/22/5f07957028f7fa8c3d695934af8e7309bfd5ab43f2a7a756d3c3d6ce44f3/langchain_nvidia_ai_endpoints-1.1.0-py3-none-any.whl", hash = "sha256:eb04251b2b21facf9d6f2e6e7fa593b89e4f5023ebe3af1e02813512d1cd9687", size = 51514, upload-time = "2026-02-25T21:48:15.695Z" }, ] [[package]] @@ -1349,7 +1349,7 @@ wheels = [ [[package]] name = "langgraph" -version = "1.0.7" +version = "1.0.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, @@ -1359,9 +1359,9 @@ dependencies = [ { name = "pydantic" }, { name = "xxhash" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/72/5b/f72655717c04e33d3b62f21b166dc063d192b53980e9e3be0e2a117f1c9f/langgraph-1.0.7.tar.gz", hash = "sha256:0cfdfee51e6e8cfe503ecc7367c73933437c505b03fa10a85c710975c8182d9a", size = 497098, upload-time = "2026-01-22T16:57:47.303Z" } +sdist = { url = "https://files.pythonhosted.org/packages/55/92/14df6fefba28c10caf1cb05aa5b8c7bf005838fe32a86d903b6c7cc4018d/langgraph-1.0.10.tar.gz", hash = "sha256:73bd10ee14a8020f31ef07e9cd4c1a70c35cc07b9c2b9cd637509a10d9d51e29", size = 511644, upload-time = "2026-02-27T21:04:38.743Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/0e/fe80144e3e4048e5d19ccdb91ac547c1a7dc3da8dbd1443e210048194c14/langgraph-1.0.7-py3-none-any.whl", hash = "sha256:9d68e8f8dd8f3de2fec45f9a06de05766d9b075b78fb03171779893b7a52c4d2", size = 157353, upload-time = "2026-01-22T16:57:45.997Z" }, + { url = "https://files.pythonhosted.org/packages/5d/60/260e0c04620a37ba8916b712766c341cc5fc685dabc6948c899494bbc2ae/langgraph-1.0.10-py3-none-any.whl", hash = "sha256:7c298bef4f6ea292fcf9824d6088fe41a6727e2904ad6066f240c4095af12247", size = 160920, upload-time = "2026-02-27T21:04:35.932Z" }, ] [[package]] @@ -1379,15 +1379,15 @@ wheels = [ [[package]] name = "langgraph-prebuilt" -version = "1.0.7" +version = "1.0.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "langgraph-checkpoint" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a7/59/711aecd1a50999456850dc328f3cad72b4372d8218838d8d5326f80cb76f/langgraph_prebuilt-1.0.7.tar.gz", hash = "sha256:38e097e06de810de4d0e028ffc0e432bb56d1fb417620fb1dfdc76c5e03e4bf9", size = 163692, upload-time = "2026-01-22T16:45:22.801Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/06/dd61a5c2dce009d1b03b1d56f2a85b3127659fdddf5b3be5d8f1d60820fb/langgraph_prebuilt-1.0.8.tar.gz", hash = "sha256:0cd3cf5473ced8a6cd687cc5294e08d3de57529d8dd14fdc6ae4899549efcf69", size = 164442, upload-time = "2026-02-19T18:14:39.083Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/47/49/5e37abb3f38a17a3487634abc2a5da87c208cc1d14577eb8d7184b25c886/langgraph_prebuilt-1.0.7-py3-none-any.whl", hash = "sha256:e14923516504405bb5edc3977085bc9622c35476b50c1808544490e13871fe7c", size = 35324, upload-time = "2026-01-22T16:45:21.784Z" }, + { url = "https://files.pythonhosted.org/packages/dc/41/ec966424ad3f2ed3996d24079d3342c8cd6c0bd0653c12b2a917a685ec6c/langgraph_prebuilt-1.0.8-py3-none-any.whl", hash = "sha256:d16a731e591ba4470f3e313a319c7eee7dbc40895bcf15c821f985a3522a7ce0", size = 35648, upload-time = "2026-02-19T18:14:37.611Z" }, ] [[package]] @@ -1918,7 +1918,7 @@ requires-dist = [ { name = "langchain-elasticsearch", marker = "extra == 'all'", specifier = ">=0.3" }, { name = "langchain-elasticsearch", marker = "extra == 'elasticsearch'", specifier = ">=0.3" }, { name = "langchain-milvus", specifier = ">=0.3.0" }, - { name = "langchain-nvidia-ai-endpoints", specifier = ">=1.0.3" }, + { name = "langchain-nvidia-ai-endpoints", specifier = ">=1.1.0" }, { name = "langchain-openai", marker = "extra == 'all'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'ingest'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'rag'", specifier = ">=0.2" }, From a85a69ee15aa60efe2ae4f19504655846774c2e1 Mon Sep 17 00:00:00 2001 From: Minh Nguyen Date: Wed, 4 Mar 2026 15:41:48 +0700 Subject: [PATCH 24/81] Update NIM wait times and patch VSS embed/rerank models (#397) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update NIM wait times and patch VSS embed/rerank models Adjust expected NIM model loading wait from 2-5 min to ~10 min for RTX PRO 6000 hardware. Add explicit patching of VSS config.yaml to align embedding and reranker model names with RAG stack defaults. Signed-off-by: Minh Nguyen Made-with: Cursor * Update VSS prompts to match default format and use seconds-based queries - Align consumer VSS prompts with VSS config.yaml defaults (sports-adapted): caption, caption_summarization, summary_aggregation with proper dedup/merge logic - Extract RAG embed/rerank model names dynamically from compose file - Add parse_compose_default helper to avoid hardcoded model names - Change time-range query from MM:SS to seconds format for VSS compatibility Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen Made-with: Cursor * Clean up notebook: rename variable, remove config overrides - Rename _rag_compose to _rag_compose_path for clarity - Remove hardcoded max_tokens and batch_size patches from VSS config - Simplify time-range query cell comments Signed-off-by: Minh Nguyen Made-with: Cursor * Rename helper function and revert prompts to MM:SS format - Rename parse_compose_default to extract_rag_default_compose_var - Revert VSS prompts to MM:SS timestamp conversion style - Revert time-range query to MM:SS format Signed-off-by: Minh Nguyen Made-with: Cursor * Add GPU assignment table and update NIM container names Update notebook to reflect renamed NIM containers (nemoretriever-* → nemotron-*) and add default GPU assignment table for RTX PRO 6000 / H100 hardware. Signed-off-by: Minh Nguyen Made-with: Cursor --------- Signed-off-by: Minh Nguyen --- notebooks/rag_event_ingest.ipynb | 59 ++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/notebooks/rag_event_ingest.ipynb b/notebooks/rag_event_ingest.ipynb index 372a6cfca..6609a6e1d 100644 --- a/notebooks/rag_event_ingest.ipynb +++ b/notebooks/rag_event_ingest.ipynb @@ -30,6 +30,16 @@ "### Hardware\n", "- **GPU**: 4x RTX PRO 6000 Blackwell or 4x H100\n", "\n", + "#### Default GPU Assignment\n", + "\n", + "| GPU | Service |\n", + "|-----|---------|\n", + "| 0 | RAG NIMs (Embedding, Reranker) |\n", + "| 1 | RAG LLM NIM (Llama-3.3-Nemotron-Super-49B) |\n", + "| 2 | VSS LLM NIM (Llama-3.1-8B-Instruct) |\n", + "| 3 | VSS VLM / via-server (Cosmos-Reason2-8B) |\n", + "\n", + "\n", "### Software (pre-installed required)\n", "- Ubuntu 22.04 or later\n", "- Docker 24.0+ with Docker Compose v2\n", @@ -369,6 +379,18 @@ " print(f\"[ERROR] {e}\")\n", " return None\n", "\n", + "def extract_rag_default_compose_var(file_path, var_name):\n", + " \"\"\"Extract default value from ${VAR:-default} pattern in a compose file.\"\"\"\n", + " pattern = re.compile(r'\\$\\{' + var_name + r':-\"?([^\"}\\s]+)\"?\\}')\n", + " with open(file_path) as f:\n", + " for line in f:\n", + " if line.lstrip().startswith('#'):\n", + " continue\n", + " m = pattern.search(line)\n", + " if m:\n", + " return m.group(1)\n", + " return None\n", + "\n", "print(f\"[OK] Helpers loaded | Host IP: {get_host_ip()}\")\n" ] }, @@ -429,14 +451,14 @@ "ip = get_host_ip()\n", "print(f\"\\nRAG deployed: http://{ip}:8081 (server) | http://{ip}:8082 (ingestor) | http://{ip}:8090 (UI)\")\n", "print(f\"COLLECTION_NAME: {MINIO_COLLECTION}\")\n", - "print(\"Wait 2-5 minutes for NIMs to load models, then run the status check cell.\")\n" + "print(\"Wait ~10 minutes for NIMs to load models, then run the status check cell.\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Verify RAG services are healthy. Wait 2-5 minutes for NIMs to load models.\n", + "Verify RAG services are healthy. Wait ~10 minutes for NIMs to load models.\n", "\n", "The deployment status should be:\n", "```\n", @@ -448,8 +470,8 @@ "milvus-etcd Up 2 minutes (healthy)\n", "milvus-minio Up 2 minutes (healthy)\n", "nim-llm-ms Up 2 minutes (healthy)\n", - "nemoretriever-embedding-ms Up 2 minutes (healthy)\n", - "nemoretriever-ranking-ms Up 2 minutes (healthy)\n", + "nemotron-embedding-ms Up 2 minutes (healthy)\n", + "nemotron-ranking-ms Up 2 minutes (healthy)\n", "```\n", "\n" ] @@ -461,7 +483,7 @@ "outputs": [], "source": [ "# Check service status and print access URLs\n", - "print(\"Wait 2-5 minutes for services to become healthy.\")\n", + "print(\"Wait ~10 minutes for services to become healthy.\")\n", "print(\"Run this cell again after waiting.\\n\")\n", "\n", "ip = get_host_ip()\n", @@ -476,7 +498,7 @@ " except requests.Timeout:\n", " s = \"[TIMEOUT]\"\n", " print(f\" {s} {name}: http://{ip}:{port}\")\n", - "run_command(\"docker ps --format 'table {{.Names}}\\t{{.Status}}' | grep -E '(rag|milvus|ingestor|nim|nemoretriever|rerankqa|NAMES)'\")\n" + "run_command(\"docker ps --format 'table {{.Names}}\\t{{.Status}}' | grep -E '(rag|milvus|ingestor|nim|nemotron|NAMES)'\")\n" ] }, { @@ -500,8 +522,13 @@ "VSS_GPU_DEVICE = 2 # GPU for VSS LLM NIM\n", "VSS_VLM_GPU_DEVICE = 3 # GPU for VLM (via-server with Cosmos-Reason2)\n", "\n", + "_rag_compose_path = os.path.join(RAG_REPO_DIR, \"deploy/compose/docker-compose-rag-server.yaml\")\n", + "RAG_EMBED_MODEL = extract_rag_default_compose_var(_rag_compose_path, \"APP_EMBEDDINGS_MODELNAME\")\n", + "RAG_RERANK_MODEL = extract_rag_default_compose_var(_rag_compose_path, \"APP_RANKING_MODELNAME\")\n", + "print(f\"RAG models from compose: embed={RAG_EMBED_MODEL}, rerank={RAG_RERANK_MODEL}\")\n", + "\n", "# Only deploy LLM for VSS; embedding and reranker are shared from RAG stack\n", - "# RAG NIMs on nvidia-rag network: nemoretriever-embedding-ms:8000, nemoretriever-ranking-ms:8000\n", + "# RAG NIMs on nvidia-rag network: nemotron-embedding-ms:8000, nemotron-ranking-ms:8000\n", "NIM_IMAGES = {\n", " \"vss-llm\": (\"nvcr.io/nim/meta/llama-3.1-8b-instruct:1.12.0\", VSS_LLM_PORT),\n", "}\n", @@ -572,6 +599,9 @@ " cfg = re.sub(r\"http://[^:]+:8007/v1\", f\"http://host.docker.internal:{VSS_LLM_PORT}/v1\", cfg)\n", " cfg = re.sub(r\"http://[^:]+:8006/v1\", \"http://host.docker.internal:9080/v1\", cfg)\n", " cfg = re.sub(r\"http://[^:]+:8005/v1\", \"http://host.docker.internal:1976/v1\", cfg)\n", + " cfg = re.sub(r\"model:\\s*nvidia/[\\w.-]+-embed[\\w.-]+\", f\"model: {RAG_EMBED_MODEL}\", cfg)\n", + " cfg = re.sub(r\"model:\\s*nvidia/[\\w.-]+-re?rank[\\w.-]+\", f\"model: {RAG_RERANK_MODEL}\", cfg)\n", + "\n", " open(config_file, \"w\").write(cfg)\n", "\n", "cmd = f\"cd {vss_deploy_dir} && set -a && source .env && set +a && docker compose up -d\"\n", @@ -579,18 +609,27 @@ "\n", "ip = get_host_ip()\n", "print(f\"\\nVSS deployed: http://{ip}:{VSS_UI_PORT} (UI) | http://{ip}:{VSS_API_PORT} (API)\")\n", - "print(\"Embedding/Reranker: shared from RAG stack (nemoretriever-embedding-ms, nemoretriever-ranking-ms)\")\n", - "print(\"Wait 2-5 minutes for NIMs to load models, then run the status check cell.\")\n" + "print(\"Embedding/Reranker: shared from RAG stack (nemotron-embedding-ms, nemotron-ranking-ms)\")\n", + "print(\"Wait ~10 minutes for NIMs to load models, then run the status check cell.\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Verify VSS services are healthy. Wait 2-5 minutes for NIMs to load models.\n", + "Verify VSS services are healthy. Wait ~10 minutes for NIMs to load models.\n", "\n", "> **Note**: Embedding and reranker NIMs are shared from the RAG stack (`nemoretriever-embedding-ms`, `nemoretriever-ranking-ms`) — no separate VSS containers needed.\n", "\n", + "The health check output should be:\n", + "```\n", + " [OK] VSS UI: http://:9110\n", + " [OK] VSS API: http://:8110\n", + " [OK] VSS LLM NIM: http://:8107\n", + " [OK] Embedding (shared from RAG): http://:9080\n", + " [OK] Reranker (shared from RAG): http://:1976\n", + "```\n", + "\n", "The deployment status should be:\n", "```\n", "NAMES STATUS\n", From 8fb149b1d72d4dbdea876856604972cc7a5bac17 Mon Sep 17 00:00:00 2001 From: Shubhadeep Das <149712532+shubhadeepd@users.noreply.github.com> Date: Wed, 4 Mar 2026 22:45:44 +0530 Subject: [PATCH 25/81] docs: Add release note for v2.5.0 release (#401) Co-authored-by: Kurt Heiss --- docs/release-notes.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/release-notes.md b/docs/release-notes.md index c0a359206..2a3550fad 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -8,6 +8,38 @@ This documentation contains the release notes for [NVIDIA RAG Blueprint](readme. +## Release 2.5.0 (2026-03-12) + +This release introduces support for the Nemotron-super-3 model, updates NIMs to the latest versions, upgrades NV-Ingest, and adds continuous ingestion along with RTX 6000 MIG support. + +### Highlights + +This release includes the following key updates: + +- **Nemotron-super-3 model support.** You can now integrate the Nemotron-super-3 model by following the steps outlined in [Change the Inference or Embedding Model](change-model.md). +- **NIMs updated to latest versions.** + The following model updates are included: + - `nvidia/llama-3.2-nv-embedqa-1b-v2` → `nvidia/llama-nemotron-embed-1b-v2` + - `nvidia/llama-3.2-nv-rerankqa-1b-v2` → `nvidia/llama-nemotron-rerank-1b-v2` + - `nemoretriever-page-elements-v3` → `nemotron-page-elements-v3` + - `nemoretriever-graphic-elements-v1` → `nemotron-graphic-elements-v1` + - `nemoretriever-table-structure-v1` → `nemotron-table-structure-v1` + - `nvidia/llama-3.2-nemoretriever-1b-vlm-embed-v1` → `nvidia/llama-nemotron-embed-vl-1b-v2` + - `nemoretriever-ocr-v1` to `nemotron-ocr-v1` +- Updated NVIngest to [version 26.1.2](https://github.com/NVIDIA/NeMo-Retriever/releases/tag/26.1.2). +- Added an example demonstrating the continuous ingestion pipeline. For more information, see [rag_event_ingest.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/rag_event_ingest.ipynb). +- **Added MIG support for RTX 6000.** For details, refer to [MIG Deployment](mig-deployment.md) and use `values-mig-rtx6000.yaml` and `mig-config-rtx6000.yaml`. +- Added documentation for the experimental Nemotron-parse-only ingestion pipeline. This configuration allows you to perform extraction using only Nemotron Parse through NV-Ingest, without relying on OCR, page-elements, graphic-elements, or table-structure NIMs. For more information, refer to [nemotron-parse-extraction.md](nemotron-parse-extraction.md#experimental-nemotron-parse-only-extraction). +- Several bug fixes, including frontend CVE resolutions, improved multimodal content concatenation for VLM embeddings, enhanced VDB serialization for high-concurrency parallel ingestion, and updates to observability and NeMo Guardrails configurations. + +### Fixed Known Issues + +The following known issues have been resolved in this release: + +- Addressed frontend CVEs. + +- Resolved VDB indexing issues during high-concurrency batch parallel ingestion by implementing VDB serialization. + ## Release 2.4.0 (2026-02-20) This release adds new features to the RAG pipeline for supporting agent workflows and enhances generations with VLMs augmenting multimodal input. From 9cbc5634b2b06b682c83d7ea0fc8a159d0948e6f Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Wed, 4 Mar 2026 14:42:46 -0600 Subject: [PATCH 26/81] workging code and GA pipelines --- .github/ISSUE_TEMPLATE/bug_report_form.yml | 81 -- .../documentation_request_correction.yml | 70 -- .../documentation_request_new.yml | 63 -- .../ISSUE_TEMPLATE/feature_request_form.yml | 109 -- .github/PULL_REQUEST_TEMPLATE.md | 10 - .github/workflows/build-push-main.yml | 6 +- .github/workflows/build-push-pr.yml | 4 +- .github/workflows/ci-pipeline.yml | 966 ------------------ .github/workflows/publish-artifacts.yml | 276 ----- src/nvidia_rag/rag_server/Dockerfile | 2 +- .../rag_server/response_generator.py | 2 +- .../utils/vdb/oracle/oracle_queries.py | 17 +- src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py | 115 ++- 13 files changed, 122 insertions(+), 1599 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/bug_report_form.yml delete mode 100644 .github/ISSUE_TEMPLATE/documentation_request_correction.yml delete mode 100644 .github/ISSUE_TEMPLATE/documentation_request_new.yml delete mode 100644 .github/ISSUE_TEMPLATE/feature_request_form.yml delete mode 100644 .github/PULL_REQUEST_TEMPLATE.md delete mode 100644 .github/workflows/ci-pipeline.yml delete mode 100644 .github/workflows/publish-artifacts.yml diff --git a/.github/ISSUE_TEMPLATE/bug_report_form.yml b/.github/ISSUE_TEMPLATE/bug_report_form.yml deleted file mode 100644 index 99eb5fd2f..000000000 --- a/.github/ISSUE_TEMPLATE/bug_report_form.yml +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Bug Report -description: File a bug report -title: "[BUG]: " -labels: ["bug"] - -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to fill out this bug report! - - - type: input - id: version - attributes: - label: Version - description: What version of rag are you running? - placeholder: "example: 1.0.0" - validations: - required: true - - - type: textarea - id: description - attributes: - label: Describe the bug. - description: Also tell us, what did you expect to happen? - placeholder: XYZ occured, I expected QRS results - validations: - required: true - - - type: textarea - id: mvr - attributes: - label: Minimum reproducible example - description: Please supply a [minimum reproducible code example](https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) here - render: shell - - - type: textarea - id: logs - attributes: - label: Relevant log output - description: Please paste relevant error and log output here - render: shell - - - type: textarea - id: env-printout - attributes: - label: Full env printout - description: Please run and paste the output of the `print_env.sh` script here, to gather any other relevant environment details - render: shell - - - type: textarea - id: misc - attributes: - label: Other/Misc. - description: Please enter any other helpful information here. - - - type: checkboxes - id: terms - attributes: - label: Code of Conduct - description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/CODE_OF_CONDUCT.md) - options: - - label: I agree to follow __THIS PROJECT__'s Code of Conduct - required: true - - label: I have searched the [open bugs](https://github.com/NVIDIA-AI-Blueprints/rag/issues?q=is%3Aopen+is%3Aissue+label%3Abug) and have found no duplicates for this bug report - required: true diff --git a/.github/ISSUE_TEMPLATE/documentation_request_correction.yml b/.github/ISSUE_TEMPLATE/documentation_request_correction.yml deleted file mode 100644 index f87a7b9f3..000000000 --- a/.github/ISSUE_TEMPLATE/documentation_request_correction.yml +++ /dev/null @@ -1,70 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Documentation - Correction/Update Request -description: Request corrections or updates to existing documentation -title: "[DOC]: " -labels: ["doc"] - -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to improve our documentation! - - - type: dropdown - id: criticality - attributes: - label: How would you describe the priority of this documentation request - options: - - Critical (currently preventing usage) - - High - - Medium - - Low (would be nice) - validations: - required: true - - - type: input - id: correction_location - attributes: - label: Please provide a link or source to the relevant docs - placeholder: "ex: https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/README.md" - validations: - required: true - - - type: textarea - id: problem - attributes: - label: Describe the problems in the documentation - placeholder: The documents say to use foo.func(args) however an AttributeError is thrown - validations: - required: true - - - type: textarea - id: correction - attributes: - label: (Optional) Propose a correction - placeholder: foo.func() was deprecated, replace documentation with foo.new_func() - - - type: checkboxes - id: terms - attributes: - label: Code of Conduct - description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/CODE_OF_CONDUCT.md) - options: - - label: I agree to follow __PROJECT__'s Code of Conduct - required: true - - label: I have searched the [open documentation issues](https://github.com/NVIDIA-AI-Blueprints/rag/issues?q=is%3Aopen+is%3Aissue+label%3Adoc) and have found no duplicates for this bug report - required: true diff --git a/.github/ISSUE_TEMPLATE/documentation_request_new.yml b/.github/ISSUE_TEMPLATE/documentation_request_new.yml deleted file mode 100644 index 0d1b543f2..000000000 --- a/.github/ISSUE_TEMPLATE/documentation_request_new.yml +++ /dev/null @@ -1,63 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Documentation - New Documentation Request -description: Request additions to rag documentation -title: "[DOC]: " -labels: ["doc"] - -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to improve our documentation! - - - type: dropdown - id: criticality - attributes: - label: How would you describe the priority of this documentation request - options: - - Critical (currently preventing usage) - - High - - Medium - - Low (would be nice) - validations: - required: true - - - type: textarea - id: problem - attributes: - label: Describe the future/missing documentation - placeholder: A code snippet mentions function foo(args) but I cannot find any documentation on it. - validations: - required: true - - - type: textarea - id: search_locs - attributes: - label: Where have you looked? - placeholder: | - https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/README.md - - - type: checkboxes - id: terms - attributes: - label: Code of Conduct - description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/CODE_OF_CONDUCT.md) - options: - - label: I agree to follow rag's Code of Conduct - required: true - - label: I have searched the [open documentation issues](https://github.com/NVIDIA-AI-Blueprints/rag/issues?q=is%3Aopen+is%3Aissue+label%3Adoc) and have found no duplicates for this bug report - required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request_form.yml b/.github/ISSUE_TEMPLATE/feature_request_form.yml deleted file mode 100644 index b3ba5787f..000000000 --- a/.github/ISSUE_TEMPLATE/feature_request_form.yml +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Feature Request Form -description: Request new or improved functionality or changes to existing functionality -title: "[FEA]: " -labels: ["feature request"] - -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to fill out this feature request! - - - type: dropdown - id: new_or_improvement - attributes: - label: Is this a new feature, an improvement, or a change to existing functionality? - options: - - New Feature - - Improvement - - Change - validations: - required: true - - - type: dropdown - id: criticality - attributes: - label: How would you describe the priority of this feature request - options: - - Critical (currently preventing usage) - - High - - Medium - - Low (would be nice) - validations: - required: true - - - type: textarea - id: problem - attributes: - label: Please provide a clear description of problem this feature solves - description: Real usage examples are especially helpful, non-code. - validations: - required: true - - - type: textarea - id: Feature_Description - attributes: - label: Feature Description - description: Please provide clear description of the feature you request (refer to [User Story format](https://www.atlassian.com/agile/project-management/user-stories#:~:text=User%20story%20template%20and%20examples) and [EARS format](https://ieeexplore.ieee.org/document/5328509)) - placeholder: > - For new feature request, please use one of the following format to describe the feature - 1. From End-user perspective, use the following user story format - As a , I , . - 2. From System perspective, use the following EARS format - shall - For changing or improving existing feature, it's recommended to provide the previoius Feature Request ID. - validations: - required: true - - - type: textarea - id: solution - attributes: - label: Describe your ideal solution - description: Please describe the functionality you would like added. - placeholder: > - A new function that takes in the information in this form, and triages the issue - - def feature_request(form_info): - parse(form_info) - return triage_outcome - validations: - required: true - - - type: textarea - id: alternatives - attributes: - label: Describe any alternatives you have considered - description: List any other libraries, or approaches you have looked at or tried. - placeholder: I have looked at library xyz and qrs, but they do not offer GPU accleration - - - type: textarea - id: misc - attributes: - label: Additional context - description: Add any other context, code examples, or references to existing implementations about the feature request here. If applicable, please list the modules affected. - - - type: checkboxes - id: terms - attributes: - label: Code of Conduct - description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/CODE_OF_CONDUCT.md) - options: - - label: I agree to follow rag's Code of Conduct - required: true - - label: I have searched the [open feature requests](https://github.com/NVIDIA-AI-Blueprints/rag/issues?q=is%3Aopen+is%3Aissue+label%3A%22feature+request%22%2Cimprovement%2Cenhancement) and have found no duplicates for this feature request - required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 3c5b84e91..000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,10 +0,0 @@ -## Description - - - -## Checklist -- [ ] I am familiar with the [Contributing Guidelines](../CONTRIBUTING.md). -- [ ] All commits are signed-off (`git commit -s`) and GPG signed (`git commit -S`). -- [ ] New or existing tests cover these changes. -- [ ] The documentation is up to date with these changes. -- [ ] If adjusting docker-compose.yaml environment variables have you ensured those are mimicked in the Helm values.yaml file. \ No newline at end of file diff --git a/.github/workflows/build-push-main.yml b/.github/workflows/build-push-main.yml index 645cab4c6..d3b671b2f 100644 --- a/.github/workflows/build-push-main.yml +++ b/.github/workflows/build-push-main.yml @@ -6,7 +6,7 @@ on: - oracle/dev jobs: - build-and-push: + build-and-push-rag: runs-on: ubuntu-latest env: # OCI AUTH @@ -38,7 +38,7 @@ jobs: - name: Build Docker Image run: | - docker build -t "${{ env.RAG_IMAGE_NAME }}":latest -f ./Dockerfile . + docker build -t "${{ env.RAG_IMAGE_NAME }}":latest -f src/nvidia_rag/rag_server/Dockerfile . - name: Tag and Push Docker Image env: @@ -79,7 +79,7 @@ jobs: - name: Build Docker Image run: | - docker build -t "${{ env.INGEST_IMAGE_NAME }}":latest -f ./Dockerfile . + docker build -t "${{ env.INGEST_IMAGE_NAME }}":latest -f src/nvidia_rag/rag_server/Dockerfile . - name: Tag and Push Docker Image env: diff --git a/.github/workflows/build-push-pr.yml b/.github/workflows/build-push-pr.yml index 0840e6e59..e858077c7 100644 --- a/.github/workflows/build-push-pr.yml +++ b/.github/workflows/build-push-pr.yml @@ -38,7 +38,7 @@ jobs: - name: Build RAG Docker Image run: | - docker build -t "${{ env.RAG_IMAGE_NAME }}":pr-${{ steps.short-sha.outputs.value }} -f ./Dockerfile . + docker build -t "${{ env.RAG_IMAGE_NAME }}":pr-${{ steps.short-sha.outputs.value }} -f src/nvidia_rag/rag_server/Dockerfile . - name: Tag and Push RAG Docker Image env: @@ -79,7 +79,7 @@ jobs: - name: Build Ingest Docker Image run: | - docker build -t "${{ env.INGEST_IMAGE_NAME }}":pr-${{ steps.short-sha.outputs.value }} -f ./Dockerfile . + docker build -t "${{ env.INGEST_IMAGE_NAME }}":pr-${{ steps.short-sha.outputs.value }} -f src/nvidia_rag/ingest_server/Dockerfile . - name: Tag and Push Ingest Docker Image env: diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml deleted file mode 100644 index 6bea384e4..000000000 --- a/.github/workflows/ci-pipeline.yml +++ /dev/null @@ -1,966 +0,0 @@ -name: CI Pipeline - -# Workflow control - mirroring GitLab workflow rules -on: - # Run on PRs (including forks) - safe jobs without secrets run for all, secret jobs only for same-repo PRs - pull_request: - # Run on push only for default branch (e.g. after merge); avoids duplicate runs when pushing to a PR branch - push: - branches: - - develop - - main - - 'release-**' - workflow_dispatch: - schedule: - # Run nightly at 7.30 PM UTC or 1 AM IST - - cron: '30 19 * * *' - -env: - # Common environment variables - MILVUS_VERSION: v2.6.5 - NV_INGEST_MAX_UTIL: 8 - APP_VECTORSTORE_ENABLEGPUSEARCH: False - APP_VECTORSTORE_ENABLEGPUINDEX: False - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - # ============================================================================ - # TEST STAGE JOBS - # ============================================================================ - - helm-blueprint-compliance: - name: Helm Blueprint Compliance - runs-on: ubuntu-latest - # Only run if push to develop OR PR from same repo (not fork) - needs secrets - if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || github.event.pull_request.head.repo.full_name == github.repository - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Helm - uses: azure/setup-helm@v4 - with: - version: 'latest' - - - name: Add Helm repositories - env: - NGC_API_KEY: ${{ secrets.NGC_API_KEY }} - CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - run: | - helm repo add nvidia-nim https://helm.ngc.nvidia.com/nim/nvidia/ --username='$oauthtoken' --password=$NGC_API_KEY - helm repo add nim https://helm.ngc.nvidia.com/nim/ --username='$oauthtoken' --password=$NGC_API_KEY - helm repo add nemo-microservices https://helm.ngc.nvidia.com/nvidia/nemo-microservices --username='$oauthtoken' --password=$NGC_API_KEY - helm repo add baidu-nim https://helm.ngc.nvidia.com/nim/baidu --username='$oauthtoken' --password=$NGC_API_KEY - helm repo add nvstaging-nim https://helm.ngc.nvidia.com/nvstaging/blueprint --username='$oauthtoken' --password=$CI_NVSTAGING_BLUEPRINT_KEY - helm repo update - - - name: Run Helm Blueprint Compliance - run: | - # Add your helm blueprint compliance checks here - echo "Running Helm Blueprint Compliance checks..." - # The actual compliance command would depend on the blueprint compliance tool - - lint: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@v4 - - - uses: actions/setup-python@v3 - - uses: pre-commit/action@v3.0.1 - - unit-tests: - name: Unit Tests - runs-on: ubuntu-latest - container: - image: python:3.12-slim - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install system dependencies - run: | - apt-get update && apt-get install -y gcc && rm -rf /var/lib/apt/lists/* - - - name: Install package with dependencies - run: | - pip install -e .[all] - pip install --no-cache-dir -r tests/unit/requirements-test.txt - - - name: Run unit tests with coverage - run: | - python -m pytest -v -s --cov=src --cov-report=term-missing tests/unit - - frontend-unit-tests: - name: Frontend Unit Tests - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - - - name: Sanitize branch name - id: sanitize - run: | - SANITIZED_REF="${GITHUB_REF_NAME//\//-}" - echo "ref_name=$SANITIZED_REF" >> $GITHUB_OUTPUT - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - with: - version: 10 - - - name: Cache frontend dependencies - uses: actions/cache@v4 - with: - path: | - frontend/node_modules - frontend/.pnpm-store - key: frontend-deps-${{ github.ref_name }} - restore-keys: | - frontend-deps- - - - name: Configure pnpm and install dependencies - working-directory: frontend - run: | - pnpm config set store-dir .pnpm-store - pnpm install --frozen-lockfile || pnpm install - - - name: Run frontend unit tests with coverage - working-directory: frontend - run: | - pnpm test:coverage - - - name: Upload coverage artifacts - uses: actions/upload-artifact@v4 - if: always() - with: - name: frontend-coverage-${{ steps.sanitize.outputs.ref_name }}-${{ github.sha }} - path: frontend/coverage/ - retention-days: 7 - - check-markdown-links: - name: Check Markdown Links - runs-on: ubuntu-latest - container: - image: python:3.12-slim - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install required packages - run: | - pip install --no-cache-dir requests - - - name: Run markdown link checker - run: | - python ci/check_markdown_links.py --root . --no-external - - # ============================================================================ - # INTEGRATION TESTS STAGE - # ============================================================================ - - integration-tests: - name: Integration Tests - runs-on: arc-runners-org-nvidia-ai-bp-2-gpu - # Only run if push to develop OR PR from same repo (not fork) - needs secrets - if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || github.event.pull_request.head.repo.full_name == github.repository - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install NGC CLI - env: - NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - run: | - echo "Installing NGC CLI..." - wget --content-disposition https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/4.9.10/files/ngccli_linux.zip -O ngccli_linux.zip - unzip -o ngccli_linux.zip - chmod u+x ngc-cli/ngc - - # Add NGC CLI to PATH for subsequent steps - echo "$(pwd)/ngc-cli" >> $GITHUB_PATH - - echo "NGC CLI installed successfully" - - - name: Download test data files - env: - NGC_API_KEY: ${{ secrets.CI_NV_RAG_BLUEPRINT_KEY }} - run: | - echo "Downloading test data files to tests/data..." - # Create tests/data directory if it doesn't exist - mkdir -p tests/data - - # Download integration test dataset from NGC - echo "Downloading integration_test_dataset:2.4.0 from NGC..." - ngc registry resource download-version "0648981100760671/integration_test_dataset:2.4.0" --dest ./tests/data --org 0648981100760671 - - # Move files from subdirectory to tests/data root - echo "Moving files to tests/data root directory..." - if [ -d "tests/data/integration_test_dataset_v2.4.0" ]; then - mv tests/data/integration_test_dataset_v2.4.0/* tests/data/ - rmdir tests/data/integration_test_dataset_v2.4.0 - echo "Files moved successfully" - else - echo "Warning: integration_test_dataset_v2.4.0 directory not found" - fi - - # Verify downloads - echo "Files in tests/data:" - ls -lh tests/data/ - - echo "Test data download completed" - - - name: Docker info - run: docker info - - - name: Clean up existing containers - run: | - echo "Cleaning up existing containers..." - docker ps -a - docker stop $(docker ps -aq) || true - docker rm $(docker ps -aq) || true - - - name: Load common environment variables - run: | - echo "Loading common environment variables..." - export TAG=$(echo ${GITHUB_REF_NAME} | sed 's/[^a-zA-Z0-9]/-/g')-${GITHUB_SHA::7} - export NGC_API_KEY=${{ secrets.NGC_API_KEY }} - export DOCKER_VOLUME_DIRECTORY=/tmp/milvus-${MILVUS_VERSION} - export INGESTOR_SERVER_EXTERNAL_VOLUME_MOUNT=/tmp/ingestor-server-data - echo "TAG=$TAG" >> $GITHUB_ENV - echo "NGC_API_KEY=$NGC_API_KEY" >> $GITHUB_ENV - echo "DOCKER_VOLUME_DIRECTORY=$DOCKER_VOLUME_DIRECTORY" >> $GITHUB_ENV - echo "INGESTOR_SERVER_EXTERNAL_VOLUME_MOUNT=$INGESTOR_SERVER_EXTERNAL_VOLUME_MOUNT" >> $GITHUB_ENV - - # Load nvdev.env and export all variables to GITHUB_ENV - if [ -f ./deploy/compose/nvdev.env ]; then - set -a - source ./deploy/compose/nvdev.env - set +a - # Export all variables from nvdev.env to GITHUB_ENV - grep -E '^export ' ./deploy/compose/nvdev.env | sed 's/export //' | while IFS='=' read -r key value; do - # Evaluate the value to expand any variable references - eval "resolved_value=\"$value\"" - echo "$key=$resolved_value" >> $GITHUB_ENV - done - fi - - - name: Docker login - env: - CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - run: | - echo "$CI_NVSTAGING_BLUEPRINT_KEY" | docker login nvcr.io -u '$oauthtoken' --password-stdin - - - name: Start services - run: | - echo "Starting vector database services..." - docker compose -f tests/integration/vectordb.yaml up -d - echo "Starting RAG server..." - docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build - echo "Starting ingestor server..." - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build - echo "Waiting for services to be ready..." - sleep 60 - echo "Checking service status..." - docker ps - - - name: Print logs for running containers - run: | - echo "=== LOGS FOR RUNNING CONTAINERS ===" - docker logs --tail 50 milvus-standalone || echo "No logs for milvus-standalone" - docker logs --tail 50 milvus-etcd || echo "No logs for milvus-etcd" - docker logs --tail 50 milvus-minio || echo "No logs for milvus-minio" - docker logs rag-server || echo "No logs for rag-server" - docker logs ingestor-server || echo "No logs for ingestor-server" - echo "Deploy stage completed successfully" - - # ======================================================================== - # BASIC TESTS - # ======================================================================== - - - name: Setup Python environment for tests - run: | - # Ensure required Python tooling is installed before use - if command -v apt-get >/dev/null 2>&1; then - sudo apt-get update - sudo apt-get install -y python3 python3-venv python3-pip fuse lsof - - fi - - python3 --version || echo "Python3 not found" - pip3 --version || echo "pip3 not found" - - # Install uv (Python package and environment manager) - curl -LsSf https://astral.sh/uv/install.sh | sh - export PATH="$HOME/.local/bin:$PATH" - - # Create a fresh virtual environment using uv - rm -rf venv || echo "No existing venv to clean up" - uv venv venv - source venv/bin/activate - uv pip install -e .[all] - uv pip install -r tests/integration/requirements.txt - - - name: Run basic integration tests - id: basic-tests - continue-on-error: true - run: | - source venv/bin/activate - echo "Running basic integration tests..." - python -m tests.integration.main --timeout 1200 - echo "Basic integration tests completed" - - - name: Collect logs after basic tests - if: always() - run: | - echo "Collecting container logs..." - mkdir -p logs/basic-tests - docker logs rag-server > logs/basic-tests/rag-server.log 2>&1 || true - docker logs ingestor-server > logs/basic-tests/ingestor-server.log 2>&1 || true - docker logs compose-nv-ingest-ms-runtime-1 > logs/basic-tests/nvingest.log 2>&1 || true - cp tests/integration/integration_test.log logs/basic-tests/ 2>/dev/null || true - - # ======================================================================== - # QUERY REWRITER TESTS - # ======================================================================== - - - name: Configure environment for query rewriter tests - run: | - echo "ENABLE_QUERYREWRITER=True" >> $GITHUB_ENV - echo "CONVERSATION_HISTORY=5" >> $GITHUB_ENV - - - name: Restart services for query rewriter tests - env: - CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - run: | - echo "Restarting services with query rewriter configuration..." - docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true - sleep 5 - docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build - sleep 30 - docker ps - - - name: Run query rewriter integration tests - id: query-rewriter-tests - continue-on-error: true - run: | - source venv/bin/activate - echo "Running query rewriter integration tests..." - python -m tests.integration.main --sequence query_rewriter - echo "Query rewriter integration tests completed" - - - name: Collect logs after query rewriter tests - if: always() - run: | - mkdir -p logs/query-rewriter - docker logs rag-server > logs/query-rewriter/rag-server.log 2>&1 || true - docker logs ingestor-server > logs/query-rewriter/ingestor-server.log 2>&1 || true - docker logs compose-nv-ingest-ms-runtime-1 > logs/query-rewriter/nvingest.log 2>&1 || true - cp tests/integration/integration_test.log logs/query-rewriter/ 2>/dev/null || true - - # ======================================================================== - # REFLECTION TESTS - # ======================================================================== - - - name: Configure environment for reflection tests - run: | - echo "ENABLE_REFLECTION=True" >> $GITHUB_ENV - echo "RESPONSE_GROUNDEDNESS_THRESHOLD=3" >> $GITHUB_ENV - # Unset query rewriter settings - echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV - echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV - - - name: Restart services for reflection tests - env: - CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - run: | - echo "Restarting services with reflection configuration..." - docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true - sleep 5 - docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build - sleep 30 - docker ps - - - name: Run reflection integration tests - id: reflection-tests - continue-on-error: true - run: | - source venv/bin/activate - echo "Running reflection integration tests..." - python -m tests.integration.main --sequence reflection - echo "Reflection integration tests completed" - - - name: Collect logs after reflection tests - if: always() - run: | - mkdir -p logs/reflection - docker logs rag-server > logs/reflection/rag-server.log 2>&1 || true - docker logs ingestor-server > logs/reflection/ingestor-server.log 2>&1 || true - docker logs compose-nv-ingest-ms-runtime-1 > logs/reflection/nvingest.log 2>&1 || true - cp tests/integration/integration_test.log logs/reflection/ 2>/dev/null || true - - # ======================================================================== - # NEMO GUARDRAILS TESTS - # ======================================================================== - - - name: Configure environment for guardrails tests - run: | - echo "ENABLE_GUARDRAILS=True" >> $GITHUB_ENV - # Unset reflection and query rewriter settings - echo "ENABLE_REFLECTION=False" >> $GITHUB_ENV - echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV - echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV - - - name: Restart services for guardrails tests - env: - CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - run: | - echo "Restarting services with guardrails configuration..." - docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true - sleep 5 - docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build - sleep 30 - docker ps - - - name: Start NemoGuardrails microservice - run: | - echo "Starting NemoGuardrails microservice..." - docker compose -f deploy/compose/docker-compose-nemo-guardrails.yaml up -d --no-deps nemo-guardrails-microservice - sleep 30 - - - name: Run NemoGuardrails integration tests - id: guardrails-tests - continue-on-error: true - run: | - source venv/bin/activate - echo "Running NemoGuardrails integration tests..." - python -m tests.integration.main --sequence nemo_guardrails - echo "NemoGuardrails integration tests completed" - - - name: Collect logs after guardrails tests - if: always() - run: | - mkdir -p logs/nemo-guardrails - docker logs rag-server > logs/nemo-guardrails/rag-server.log 2>&1 || true - docker logs ingestor-server > logs/nemo-guardrails/ingestor-server.log 2>&1 || true - docker logs compose-nv-ingest-ms-runtime-1 > logs/nemo-guardrails/nvingest.log 2>&1 || true - docker logs nemo-guardrails-microservice > logs/nemo-guardrails/nemo-guardrails.log 2>&1 || true - cp tests/integration/integration_test.log logs/nemo-guardrails/ 2>/dev/null || true - - - name: Stop NemoGuardrails microservice - if: always() - run: | - docker compose -f deploy/compose/docker-compose-nemo-guardrails.yaml down nemo-guardrails-microservice || true - - # ======================================================================== - # IMAGE CAPTIONING TESTS - # ======================================================================== - - - name: Configure environment for image captioning tests - run: | - echo "APP_NVINGEST_EXTRACTIMAGES=True" >> $GITHUB_ENV - echo "ENABLE_GUARDRAILS=False" >> $GITHUB_ENV - echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV - echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV - - - name: Restart services for image captioning tests - env: - CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - run: | - echo "Restarting services with image captioning configuration..." - docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true - sleep 5 - docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build - sleep 30 - docker ps - - - name: Run image captioning integration tests - id: image-captioning-tests - continue-on-error: true - run: | - source venv/bin/activate - echo "Running image captioning integration tests..." - python -m tests.integration.main --sequence image_captioning - echo "Image captioning integration tests completed" - - - name: Collect logs after image captioning tests - if: always() - run: | - mkdir -p logs/image-captioning - docker logs rag-server > logs/image-captioning/rag-server.log 2>&1 || true - docker logs ingestor-server > logs/image-captioning/ingestor-server.log 2>&1 || true - docker logs compose-nv-ingest-ms-runtime-1 > logs/image-captioning/nvingest.log 2>&1 || true - cp tests/integration/integration_test.log logs/image-captioning/ 2>/dev/null || true - - # ======================================================================== - # VLM GENERATION TESTS - # ======================================================================== - - - name: Configure environment for VLM generation tests - run: | - echo "ENABLE_VLM_INFERENCE=True" >> $GITHUB_ENV - echo "APP_NVINGEST_EXTRACTIMAGES=False" >> $GITHUB_ENV - echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV - echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV - - - name: Restart services for VLM generation tests - env: - CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - run: | - echo "Restarting services with VLM generation configuration..." - docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true - sleep 5 - docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build - sleep 30 - docker ps - - - name: Run VLM generation integration tests - id: vlm-generation-tests - continue-on-error: true - run: | - source venv/bin/activate - echo "Running VLM generation integration tests..." - python -m tests.integration.main --sequence vlm_generation - echo "VLM generation integration tests completed" - - - name: Collect logs after VLM generation tests - if: always() - run: | - mkdir -p logs/vlm-generation - docker logs rag-server > logs/vlm-generation/rag-server.log 2>&1 || true - docker logs ingestor-server > logs/vlm-generation/ingestor-server.log 2>&1 || true - docker logs compose-nv-ingest-ms-runtime-1 > logs/vlm-generation/nvingest.log 2>&1 || true - cp tests/integration/integration_test.log logs/vlm-generation/ 2>/dev/null || true - - # ======================================================================== - # MULTIMODAL QUERY TESTS - # ======================================================================== - - - name: Prepare multimodal test data - run: | - mkdir -p data/multimodal/query - [ -f tests/data/product_catalog.pdf ] && cp tests/data/product_catalog.pdf data/multimodal/ || true - [ -f tests/data/Creme_clutch_purse1-small.jpg ] && cp tests/data/Creme_clutch_purse1-small.jpg data/multimodal/query/ || true - [ -f tests/data/query/Creme_clutch_purse1-small.jpg ] && cp tests/data/query/Creme_clutch_purse1-small.jpg data/multimodal/query/ || true - - - name: Configure environment for multimodal query tests - run: | - # VLM embedding (required for multimodal queries) - echo "APP_EMBEDDINGS_MODELNAME=nvidia/llama-nemotron-embed-vl-1b-v2" >> $GITHUB_ENV - echo "APP_EMBEDDINGS_SERVERURL=https://integrate.api.nvidia.com/v1" >> $GITHUB_ENV - # VLM model for generation - echo "ENABLE_VLM_INFERENCE=True" >> $GITHUB_ENV - echo "APP_VLM_MODELNAME=nvidia/nemotron-nano-12b-v2-vl" >> $GITHUB_ENV - echo "APP_VLM_SERVERURL=https://integrate.api.nvidia.com/v1" >> $GITHUB_ENV - # Disable reranker (not supported with multimodal) - echo "ENABLE_RERANKER=False" >> $GITHUB_ENV - echo "APP_RANKING_SERVERURL=" >> $GITHUB_ENV - # Image extraction for ingestion - echo "APP_NVINGEST_EXTRACTIMAGES=True" >> $GITHUB_ENV - echo "APP_NVINGEST_STRUCTURED_ELEMENTS_MODALITY=" >> $GITHUB_ENV - echo "APP_NVINGEST_IMAGE_ELEMENTS_MODALITY=image" >> $GITHUB_ENV - # Reset other flags - echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV - echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV - - - name: Restart services for multimodal query tests - env: - CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - run: | - echo "Restarting services with multimodal query configuration..." - docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true - sleep 5 - docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build - sleep 30 - docker ps - - - name: Run multimodal query integration tests - id: multimodal-query-tests - continue-on-error: true - run: | - source venv/bin/activate - echo "Running multimodal query integration tests..." - python -m tests.integration.main --sequence multimodal_query - echo "Multimodal query integration tests completed" - - - name: Collect logs after multimodal query tests - if: always() - run: | - mkdir -p logs/multimodal-query - docker logs rag-server > logs/multimodal-query/rag-server.log 2>&1 || true - docker logs ingestor-server > logs/multimodal-query/ingestor-server.log 2>&1 || true - docker logs compose-nv-ingest-ms-runtime-1 > logs/multimodal-query/nvingest.log 2>&1 || true - cp tests/integration/integration_test.log logs/multimodal-query/ 2>/dev/null || true - - # ======================================================================== - # CUSTOM PROMPT TESTS - # ======================================================================== - - - name: Configure environment for custom prompt tests - run: | - echo "PROMPT_CONFIG_FILE=$(pwd)/tests/data/test_prompt.yaml" >> $GITHUB_ENV - echo "ENABLE_VLM_INFERENCE=False" >> $GITHUB_ENV - echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV - echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV - - - name: Restart services for custom prompt tests - env: - CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - run: | - echo "Restarting services with custom prompt configuration..." - docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true - sleep 5 - docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build - sleep 30 - docker ps - - - name: Run custom prompt integration tests - id: custom-prompt-tests - continue-on-error: true - run: | - source venv/bin/activate - echo "Running custom prompt integration tests..." - python -m tests.integration.main --sequence custom_prompt - echo "Custom prompt integration tests completed" - - - name: Collect logs after custom prompt tests - if: always() - run: | - mkdir -p logs/custom-prompt - docker logs rag-server > logs/custom-prompt/rag-server.log 2>&1 || true - docker logs ingestor-server > logs/custom-prompt/ingestor-server.log 2>&1 || true - docker logs compose-nv-ingest-ms-runtime-1 > logs/custom-prompt/nvingest.log 2>&1 || true - cp tests/integration/integration_test.log logs/custom-prompt/ 2>/dev/null || true - - # ======================================================================== - # LIBRARY USAGE TESTS - # ======================================================================== - - - name: Stop rag-server and ingestor-server for library tests - run: | - echo "Stopping rag-server and ingestor-server containers (library mode doesn't need them)..." - echo "Keeping nv-ingest-ms-runtime, Milvus, Redis, MinIO running for library mode..." - docker stop rag-server || true - docker stop ingestor-server || true - echo "Services stopped. Library tests will use the nvidia_rag library directly." - echo "Remaining services (needed for library mode):" - docker ps - - - name: Configure environment for library usage tests - run: | - echo "PROMPT_CONFIG_FILE=${PWD}/src/nvidia_rag/rag_server/prompt.yaml" >> $GITHUB_ENV - echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV - echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV - - - name: Run library usage integration tests - id: library-usage-tests - continue-on-error: true - run: | - source venv/bin/activate - echo "Running library usage integration tests..." - python -m tests.integration.main --sequence library_usage - echo "Library usage integration tests completed" - - - name: Collect logs after library usage tests - if: always() - run: | - mkdir -p logs/library-usage - docker logs compose-nv-ingest-ms-runtime-1 > logs/library-usage/nvingest.log 2>&1 || true - cp tests/integration/integration_test.log logs/library-usage/ 2>/dev/null || true - - # ======================================================================== - # LIBRARY SUMMARIZATION TESTS - # ======================================================================== - - - name: Run library summarization integration tests - id: library-summarization-tests - continue-on-error: true - run: | - source venv/bin/activate - echo "Running library summarization integration tests..." - python -m tests.integration.main --sequence library_summarization - echo "Library summarization integration tests completed" - - - name: Collect logs after library summarization tests - if: always() - run: | - mkdir -p logs/library-summarization - docker logs compose-nv-ingest-ms-runtime-1 > logs/library-summarization/nvingest.log 2>&1 || true - cp tests/integration/integration_test.log logs/library-summarization/ 2>/dev/null || true - - # ======================================================================== - # OBSERVABILITY TESTS - # ======================================================================== - - - name: Configure environment for observability tests - run: | - echo "APP_TRACING_ENABLED=True" >> $GITHUB_ENV - echo "OPENTELEMETRY_CONFIG_FILE=$(pwd)/deploy/config/otel-collector-config.yaml" >> $GITHUB_ENV - echo "PROMPT_CONFIG_FILE=${PWD}/src/nvidia_rag/rag_server/prompt.yaml" >> $GITHUB_ENV - echo "ENABLE_QUERYREWRITER=False" >> $GITHUB_ENV - echo "CONVERSATION_HISTORY=0" >> $GITHUB_ENV - - - name: Restart services for observability tests - env: - CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - run: | - echo "Restarting services with observability configuration..." - echo "(rag-server and ingestor-server were stopped for library tests, now restarting)" - docker compose -f tests/integration/vectordb.yaml down -v || true - docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true - sleep 5 - docker compose -f tests/integration/vectordb.yaml up -d || true - docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build - sleep 30 - docker ps - - - name: Start observability services - run: | - echo "Starting observability services..." - docker compose -f deploy/compose/observability.yaml up -d otel-collector prometheus - sleep 30 - - - name: Run observability integration tests - id: observability-tests - continue-on-error: true - run: | - source venv/bin/activate - echo "Running observability integration tests..." - python -m tests.integration.main --sequence observability - echo "Observability integration tests completed" - - - name: Collect logs after observability tests - if: always() - run: | - mkdir -p logs/observability - echo "=== Container Status (docker ps -a) ===" | tee logs/observability/container-status.log - docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | tee -a logs/observability/container-status.log - echo "" | tee -a logs/observability/container-status.log - echo "=== Milvus Container Stats ===" | tee -a logs/observability/container-status.log - docker stats --no-stream milvus-standalone 2>&1 | tee -a logs/observability/container-status.log || echo "Milvus container not running" | tee -a logs/observability/container-status.log - docker logs rag-server > logs/observability/rag-server.log 2>&1 || true - docker logs ingestor-server > logs/observability/ingestor-server.log 2>&1 || true - docker logs compose-nv-ingest-ms-runtime-1 > logs/observability/nvingest.log 2>&1 || true - docker logs milvus-standalone > logs/observability/milvus.log 2>&1 || true - docker logs otel-collector > logs/observability/otel-collector.log 2>&1 || true - docker logs zipkin > logs/observability/zipkin.log 2>&1 || true - docker logs prometheus > logs/observability/prometheus.log 2>&1 || true - docker logs grafana-service > logs/observability/grafana.log 2>&1 || true - cp tests/integration/integration_test.log logs/observability/ 2>/dev/null || true - - - name: Stop observability services - if: always() - run: | - docker compose -f deploy/compose/observability.yaml down || true - - # ======================================================================== - # MILVUS VDB AUTH VIA REST TESTS - # ======================================================================== - - - name: Configure Milvus VDB auth - run: | - echo "Configuring Milvus authentication and override compose..." - # Set auth credentials for the servers (running in Docker) - echo "APP_VECTORSTORE_USERNAME=root" >> $GITHUB_ENV - echo "APP_VECTORSTORE_PASSWORD=Milvus" >> $GITHUB_ENV - # Set auth token for the integration tests (running on host) - echo "VDB_AUTH_TOKEN=root:Milvus" >> $GITHUB_ENV - echo "MILVUS_ROOT_TOKEN=root:Milvus" >> $GITHUB_ENV - - # Stop and clean up existing Milvus containers and volumes - echo "Stopping and cleaning up existing Milvus containers..." - docker compose -f tests/integration/vectordb.yaml down -v || true - - # Remove the old data directories to ensure clean start with auth - echo "Cleaning up old Milvus data directories..." - sudo rm -rf /tmp/milvus-${MILVUS_VERSION}/volumes/milvus || true - sudo rm -rf /tmp/milvus-${MILVUS_VERSION}/volumes/etcd || true - - # Ensure milvus.yaml exists in tests/integration by copying from running container - # Since we stopped the container, we need to start a temporary one to extract config - mkdir -p tests/integration - if [ ! -f tests/integration/milvus.yaml ]; then - echo "Creating temporary Milvus container to extract config..." - docker run --rm -d --name milvus-temp milvusdb/milvus:${MILVUS_VERSION:-v2.6.5-gpu} sleep 60 - docker cp milvus-temp:/milvus/configs/milvus.yaml tests/integration/milvus.yaml - docker stop milvus-temp || true - fi - - # Update authentication settings in milvus.yaml - sed -i 's/authorizationEnabled:.*/authorizationEnabled: true/' tests/integration/milvus.yaml - sed -i 's/defaultRootPassword:.*/defaultRootPassword: Milvus/' tests/integration/milvus.yaml - echo "MILVUS_CONFIG_FILE=$(pwd)/tests/integration/milvus.yaml" >> $GITHUB_ENV - - # Verify the changes - echo "Verifying milvus.yaml authentication settings:" - grep -A2 "security:" tests/integration/milvus.yaml | head -5 - - # Update vectordb.yaml to comment out data volume and enable config volume - sed -i 's|- \${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus|# - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus|' tests/integration/vectordb.yaml - sed -i 's|# - \${MILVUS_CONFIG_FILE:-./milvus.yaml}:/milvus/configs/milvus.yaml|- ${MILVUS_CONFIG_FILE:-./milvus.yaml}:/milvus/configs/milvus.yaml|' tests/integration/vectordb.yaml - - - name: Restart vector database with auth - run: | - echo "Starting vector database with authentication enabled..." - docker compose -f tests/integration/vectordb.yaml up -d - echo "Waiting for Milvus services to be ready..." - sleep 60 - docker ps - echo "Checking Milvus logs..." - docker logs --tail 100 milvus-standalone || true - docker logs --tail 50 milvus-etcd || true - docker logs --tail 50 milvus-minio || true - - - name: Restart services for Milvus VDB auth tests - env: - CI_NVSTAGING_BLUEPRINT_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - run: | - echo "Restarting rag/ingestor services to pick up Milvus auth configuration..." - docker compose -f deploy/compose/docker-compose-rag-server.yaml down || true - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down || true - sleep 5 - docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --build - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --build - sleep 30 - docker ps - - - name: Run Milvus VDB Auth tests - id: milvus-vdb-auth-tests - continue-on-error: true - env: - APP_VECTORSTORE_URL: http://localhost:19530 - run: | - source venv/bin/activate - echo "Running Milvus VDB Auth tests..." - echo "APP_VECTORSTORE_URL set to: $APP_VECTORSTORE_URL" - python -m tests.integration.main --sequence milvus_vdb_auth_through_rest_api - echo "Milvus VDB Auth tests completed" - - - name: Collect logs after Milvus VDB auth tests - if: always() - run: | - mkdir -p logs/milvus-vdb-auth - docker logs rag-server > logs/milvus-vdb-auth/rag-server.log 2>&1 || true - docker logs ingestor-server > logs/milvus-vdb-auth/ingestor-server.log 2>&1 || true - docker logs milvus-standalone > logs/milvus-vdb-auth/milvus.log 2>&1 || true - docker logs milvus-etcd > logs/milvus-vdb-auth/etcd.log 2>&1 || true - docker logs milvus-minio > logs/milvus-vdb-auth/minio.log 2>&1 || true - cp tests/integration/integration_test.log logs/milvus-vdb-auth/ 2>/dev/null || true - - - name: Revert Milvus VDB auth configurations - if: always() - run: | - echo "Reverting Milvus VDB auth configurations..." - # Stop Milvus with auth enabled - docker compose -f tests/integration/vectordb.yaml down -v || true - - # Clean up auth-specific data to ensure fresh start - sudo rm -rf /tmp/milvus-${MILVUS_VERSION}/volumes/milvus || true - sudo rm -rf /tmp/milvus-${MILVUS_VERSION}/volumes/etcd || true - - # Remove the auth config file - rm -f tests/integration/milvus.yaml || true - - # Revert vectordb.yaml to original state - sed -i 's|- \${MILVUS_CONFIG_FILE:-./milvus.yaml}:/milvus/configs/milvus.yaml|# - ${MILVUS_CONFIG_FILE:-./milvus.yaml}:/milvus/configs/milvus.yaml|' tests/integration/vectordb.yaml - sed -i 's|# - \${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus|- \${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus|' tests/integration/vectordb.yaml - - # Restart Milvus without auth for subsequent tests - echo "Restarting Milvus without authentication for subsequent tests..." - docker compose -f tests/integration/vectordb.yaml up -d - sleep 30 - - # Unset auth environment variables - echo "APP_VECTORSTORE_USERNAME=" >> $GITHUB_ENV - echo "APP_VECTORSTORE_PASSWORD=" >> $GITHUB_ENV - echo "VDB_AUTH_TOKEN=" >> $GITHUB_ENV - echo "MILVUS_ROOT_TOKEN=" >> $GITHUB_ENV - - # ======================================================================== - # FAIL JOB IF ANY INTEGRATION TEST FAILED - # ======================================================================== - # All test steps use continue-on-error so every suite runs; this step - # marks the job as failed if any of them failed. - - name: Fail job if any integration test failed - if: always() && (steps.basic-tests.outcome == 'failure' || steps.query-rewriter-tests.outcome == 'failure' || steps.reflection-tests.outcome == 'failure' || steps.guardrails-tests.outcome == 'failure' || steps.image-captioning-tests.outcome == 'failure' || steps.vlm-generation-tests.outcome == 'failure' || steps.multimodal-query-tests.outcome == 'failure' || steps.custom-prompt-tests.outcome == 'failure' || steps.library-usage-tests.outcome == 'failure' || steps.library-summarization-tests.outcome == 'failure' || steps.observability-tests.outcome == 'failure' || steps.milvus-vdb-auth-tests.outcome == 'failure') - run: | - echo "=== Failed integration test suites ===" - [ "${{ steps.basic-tests.outcome }}" = "failure" ] && echo " - basic-tests" - [ "${{ steps.query-rewriter-tests.outcome }}" = "failure" ] && echo " - query-rewriter-tests" - [ "${{ steps.reflection-tests.outcome }}" = "failure" ] && echo " - reflection-tests" - [ "${{ steps.guardrails-tests.outcome }}" = "failure" ] && echo " - guardrails-tests" - [ "${{ steps.image-captioning-tests.outcome }}" = "failure" ] && echo " - image-captioning-tests" - [ "${{ steps.vlm-generation-tests.outcome }}" = "failure" ] && echo " - vlm-generation-tests" - [ "${{ steps.multimodal-query-tests.outcome }}" = "failure" ] && echo " - multimodal-query-tests" - [ "${{ steps.custom-prompt-tests.outcome }}" = "failure" ] && echo " - custom-prompt-tests" - [ "${{ steps.library-usage-tests.outcome }}" = "failure" ] && echo " - library-usage-tests" - [ "${{ steps.library-summarization-tests.outcome }}" = "failure" ] && echo " - library-summarization-tests" - [ "${{ steps.observability-tests.outcome }}" = "failure" ] && echo " - observability-tests" - [ "${{ steps.milvus-vdb-auth-tests.outcome }}" = "failure" ] && echo " - milvus-vdb-auth-tests" - echo "One or more integration test suites failed. Failing job." - exit 1 - - # ======================================================================== - # UPLOAD ALL LOGS - # ======================================================================== - - - name: Sanitize branch name for artifacts - if: always() - id: sanitize - run: | - SANITIZED_REF="${GITHUB_REF_NAME//\//-}" - echo "ref_name=$SANITIZED_REF" >> $GITHUB_OUTPUT - - - name: Upload all integration test logs - uses: actions/upload-artifact@v4 - if: always() - with: - name: integration-tests-logs-${{ steps.sanitize.outputs.ref_name }}-${{ github.sha }} - path: logs/ - retention-days: 7 - - # ======================================================================== - # CLEANUP - # ======================================================================== - - - name: Cleanup virtual environment - if: always() - run: | - echo "Cleaning up virtual environment..." - rm -rf venv - echo "Virtual environment cleanup completed" - - - name: Cleanup Docker containers and volumes - if: always() - run: | - echo "Cleaning up integration test environment..." - # Bring down only the specific compose stacks used in this workflow, - # and clean up their containers, networks, and locally built images. - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down -v --remove-orphans --rmi local || true - docker compose -f deploy/compose/docker-compose-rag-server.yaml down -v --remove-orphans --rmi local || true - docker compose -f tests/integration/vectordb.yaml down -v --remove-orphans --rmi local || true - echo "Cleanup completed for integration test Docker resources" diff --git a/.github/workflows/publish-artifacts.yml b/.github/workflows/publish-artifacts.yml deleted file mode 100644 index 2be3979e1..000000000 --- a/.github/workflows/publish-artifacts.yml +++ /dev/null @@ -1,276 +0,0 @@ -name: Publish Artifacts - -# Workflow control - runs nightly or on manual trigger -on: - schedule: - # Run nightly at 6.30 PM UTC or 12 AM IST - - cron: '30 18 * * *' - workflow_dispatch: - inputs: - CONTAINER_TAG: - description: 'Custom tag for containers (optional)' - required: false - default: '' - ARTIFACTORY_VERSION: - description: 'Artifactory version (optional, defaults to auto-generated from get_version.sh)' - required: false - default: '' - -env: - RELEASE_TYPE: dev - -jobs: - # ============================================================================ - # PUBLISH WHEEL - # ============================================================================ - publish-wheel: - name: Build and Publish Python Wheel - runs-on: ubuntu-latest - container: - image: python:3.10 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set artifactory version - run: | - if [ -n "${{ github.event.inputs.ARTIFACTORY_VERSION }}" ]; then - echo "Using custom Artifactory version: ${{ github.event.inputs.ARTIFACTORY_VERSION }}" - echo "ARTIFACTORY_VERSION=${{ github.event.inputs.ARTIFACTORY_VERSION }}" >> $GITHUB_ENV - else - echo "Using version from get_version.sh..." - chmod +x ./ci/get_version.sh - DEFAULT_VERSION=$(./ci/get_version.sh) - echo "Generated default version: $DEFAULT_VERSION" - echo "ARTIFACTORY_VERSION=$DEFAULT_VERSION" >> $GITHUB_ENV - fi - - - name: Install dependencies - run: | - pip install uv==0.8.12 - apt-get update && apt-get install -y wget unzip - - - name: Build wheel with version - run: | - echo "Building wheel with version: $ARTIFACTORY_VERSION" - sed -i "s#^version = \".*\"#version = \"$ARTIFACTORY_VERSION\"#" pyproject.toml - uv build - ls -la dist/ - - - name: Upload wheel artifact - uses: actions/upload-artifact@v4 - with: - name: wheel-${{ env.ARTIFACTORY_VERSION }} - path: dist/*.whl - retention-days: 30 - - - name: Install NGC CLI - env: - NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - run: | - echo "Installing NGC CLI..." - wget --content-disposition https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/4.9.10/files/ngccli_linux.zip -O ngccli_linux.zip - unzip -o ngccli_linux.zip - chmod u+x ngc-cli/ngc - - # Add NGC CLI to PATH for subsequent steps - echo "$(pwd)/ngc-cli" >> $GITHUB_PATH - - echo "NGC CLI installed successfully" - - - name: Publish wheel to NGC - env: - NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - run: | - # Find the wheel file - WHEEL_FILE=$(ls dist/*.whl | tail -n 1) - echo "Found wheel file: $WHEEL_FILE" - - # Extract just the filename for display - WHEEL_FILENAME=$(basename "$WHEEL_FILE") - echo "Wheel filename: $WHEEL_FILENAME" - - # Publish to NGC - echo "Publishing wheel to NGC: nvstaging/blueprint/nvidia_rag:$ARTIFACTORY_VERSION" - ngc registry resource upload-version \ - "nvstaging/blueprint/nvidia_rag:$ARTIFACTORY_VERSION" \ - --source "$WHEEL_FILE" \ - --org nvstaging - - echo "Wheel published to NGC successfully" - echo "NGC Resource: nvstaging/blueprint/nvidia_rag:$ARTIFACTORY_VERSION" - - # ============================================================================ - # PUBLISH RAG SERVER CONTAINER - # ============================================================================ - publish-rag-server: - name: Build and Publish RAG Server Container - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Determine TAG - id: tag - run: | - if [ -n "${{ github.event.inputs.CONTAINER_TAG }}" ]; then - echo "Using custom TAG from input: ${{ github.event.inputs.CONTAINER_TAG }}" - TAG="${{ github.event.inputs.CONTAINER_TAG }}" - else - echo "Using auto-generated version as TAG" - VERSION=$(./ci/get_version.sh) - TAG=$VERSION - fi - echo "TAG=$TAG" >> $GITHUB_ENV - echo "tag=$TAG" >> $GITHUB_OUTPUT - echo "Final TAG value: $TAG" - - - name: Login to NGC Container Registry - uses: docker/login-action@v3 - with: - registry: nvcr.io - username: '$oauthtoken' - password: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - - - name: Build and push RAG server container - env: - NGC_API_KEY: ${{ secrets.NGC_API_KEY }} - run: | - echo "Building rag-server container with tag ${TAG}..." - export TAG=${TAG} - export DOWNLOAD_LEGAL_COMPLIANCE=true - docker compose -f deploy/compose/docker-compose-rag-server.yaml build rag-server - - # Tag and push to NGC Container Registry - echo "Pushing rag-server to NGC Container Registry..." - docker push nvcr.io/nvstaging/blueprint/rag-server:$TAG - docker tag nvcr.io/nvstaging/blueprint/rag-server:$TAG nvcr.io/nvstaging/blueprint/rag-server:latest - docker push nvcr.io/nvstaging/blueprint/rag-server:latest - echo "RAG server container publishing completed successfully" - - - name: Cleanup Docker images - if: always() - run: | - echo "Cleaning up rag-server Docker images..." - docker images | grep "rag-server" | awk '{print $3}' | xargs -r docker rmi -f || echo "No rag-server images to delete" - docker system prune -f || true - - # ============================================================================ - # PUBLISH INGESTOR SERVER CONTAINER - # ============================================================================ - publish-ingestor-server: - name: Build and Publish Ingestor Server Container - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Determine TAG - id: tag - run: | - if [ -n "${{ github.event.inputs.CONTAINER_TAG }}" ]; then - echo "Using custom TAG from input: ${{ github.event.inputs.CONTAINER_TAG }}" - TAG="${{ github.event.inputs.CONTAINER_TAG }}" - else - echo "Using auto-generated version as TAG" - VERSION=$(./ci/get_version.sh) - TAG=$VERSION - fi - echo "TAG=$TAG" >> $GITHUB_ENV - echo "tag=$TAG" >> $GITHUB_OUTPUT - echo "Final TAG value: $TAG" - - - name: Login to NGC Container Registry - uses: docker/login-action@v3 - with: - registry: nvcr.io - username: '$oauthtoken' - password: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - - - name: Build and push ingestor server container - env: - NGC_API_KEY: ${{ secrets.NGC_API_KEY }} - run: | - echo "Building ingestor-server container with tag ${TAG}..." - export TAG=${TAG} - export DOWNLOAD_LEGAL_COMPLIANCE=true - docker compose -f deploy/compose/docker-compose-ingestor-server.yaml build ingestor-server - - # Tag and push to NGC Container Registry - echo "Pushing ingestor-server to NGC Container Registry..." - docker push nvcr.io/nvstaging/blueprint/ingestor-server:$TAG - docker tag nvcr.io/nvstaging/blueprint/ingestor-server:$TAG nvcr.io/nvstaging/blueprint/ingestor-server:latest - docker push nvcr.io/nvstaging/blueprint/ingestor-server:latest - echo "Ingestor server container publishing completed successfully" - - - name: Cleanup Docker images - if: always() - run: | - echo "Cleaning up ingestor-server Docker images..." - docker images | grep "ingestor-server" | awk '{print $3}' | xargs -r docker rmi -f || echo "No ingestor-server images to delete" - docker system prune -f || true - - # ============================================================================ - # PUBLISH RAG FRONTEND CONTAINER - # ============================================================================ - publish-rag-frontend: - name: Build and Publish RAG Frontend Container - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Determine TAG - id: tag - run: | - if [ -n "${{ github.event.inputs.CONTAINER_TAG }}" ]; then - echo "Using custom TAG from input: ${{ github.event.inputs.CONTAINER_TAG }}" - TAG="${{ github.event.inputs.CONTAINER_TAG }}" - else - echo "Using auto-generated version as TAG" - VERSION=$(./ci/get_version.sh) - TAG=$VERSION - fi - echo "TAG=$TAG" >> $GITHUB_ENV - echo "tag=$TAG" >> $GITHUB_OUTPUT - echo "Final TAG value: $TAG" - - - name: Login to NGC Container Registry - uses: docker/login-action@v3 - with: - registry: nvcr.io - username: '$oauthtoken' - password: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} - - - name: Build and push RAG frontend container - env: - NGC_API_KEY: ${{ secrets.NGC_API_KEY }} - run: | - echo "Building rag-frontend container with tag ${TAG}..." - export TAG=${TAG} - export DOWNLOAD_LEGAL_COMPLIANCE=true - docker compose -f deploy/compose/docker-compose-rag-server.yaml build rag-frontend - - # Tag and push to NGC Container Registry - echo "Pushing rag-frontend to NGC Container Registry..." - docker push nvcr.io/nvstaging/blueprint/rag-frontend:$TAG - docker tag nvcr.io/nvstaging/blueprint/rag-frontend:$TAG nvcr.io/nvstaging/blueprint/rag-frontend:latest - docker push nvcr.io/nvstaging/blueprint/rag-frontend:latest - echo "RAG frontend container publishing completed successfully" - - - name: Cleanup Docker images - if: always() - run: | - echo "Cleaning up rag-frontend Docker images..." - docker images | grep "rag-frontend" | awk '{print $3}' | xargs -r docker rmi -f || echo "No rag-frontend images to delete" - docker system prune -f || true - diff --git a/src/nvidia_rag/rag_server/Dockerfile b/src/nvidia_rag/rag_server/Dockerfile index af8a2452d..286d120b4 100644 --- a/src/nvidia_rag/rag_server/Dockerfile +++ b/src/nvidia_rag/rag_server/Dockerfile @@ -69,7 +69,7 @@ WORKDIR /workspace COPY --from=builder /build/pyproject.toml /workspace/ COPY --from=builder /build/uv.lock /workspace/ -RUN uv sync --locked --no-install-project --no-dev --extra rag --extra elasticsearch +RUN uv sync --locked --no-install-project --no-dev --extra rag --extra oracle COPY --from=builder /build/dist/*.whl /workspace/ diff --git a/src/nvidia_rag/rag_server/response_generator.py b/src/nvidia_rag/rag_server/response_generator.py index b0ff06379..a274a5898 100644 --- a/src/nvidia_rag/rag_server/response_generator.py +++ b/src/nvidia_rag/rag_server/response_generator.py @@ -782,7 +782,7 @@ def prepare_citations( document_type = "text" else: file_name = os.path.basename( - doc.metadata.get("source").get("source_id") + doc.metadata.get("source_id") ) if doc.metadata.get("content_metadata", {}).get("type") in [ diff --git a/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py b/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py index 8471a7fde..a246f4e1e 100644 --- a/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py +++ b/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py @@ -167,12 +167,17 @@ def get_unique_sources_query(table_name: str) -> str: SQL query string """ return f""" - SELECT DISTINCT source, - MAX(content_metadata) KEEP (DENSE_RANK FIRST ORDER BY created_at DESC) as content_metadata - FROM {table_name} - WHERE source IS NOT NULL - GROUP BY source - ORDER BY source + WITH unique_sources AS ( + SELECT source, + ROW_NUMBER() OVER (PARTITION BY source ORDER BY created_at DESC) as rn + FROM {table_name} + WHERE source IS NOT NULL + ) + SELECT us.source, t.content_metadata + FROM unique_sources us + JOIN {table_name} t ON us.source = t.source + WHERE us.rn = 1 + ORDER BY us.source """ diff --git a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py index 12d3e6ed6..7dc7fa847 100644 --- a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py +++ b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py @@ -33,9 +33,14 @@ import time from concurrent.futures import Future from typing import Any +from array import array import oracledb -from langchain_community.vectorstores.oraclevs import OracleVS +from langchain_community.vectorstores.oraclevs import ( + OracleVS, + _get_connection, + _get_distance_function, +) from langchain_community.vectorstores.utils import DistanceStrategy from langchain_core.documents import Document from langchain_core.runnables import RunnableAssign, RunnableLambda @@ -77,6 +82,84 @@ from nvidia_rag.utils.vdb.vdb_ingest_base import VDBRagIngest logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class OracleVSCompat(OracleVS): + """Compatibility shim to align LangChain OracleVS with our schema and vector bind needs.""" + + def __init__( + self, + *args, + **kwargs, + ): + super().__init__(*args, **kwargs) + # Cache embedding dimension to avoid re-embedding per query + self._embedding_dim = self.get_embedding_dimension() + + def similarity_search_by_vector_with_relevance_scores( + self, + embedding, + k: int = 4, + filter=None, + **kwargs, + ): + # Bind embedding as CLOB or array, then cast via TO_VECTOR to avoid ORA-00904 on bind name + if self.insert_mode == "clob": + embedding_arr = json.dumps(embedding) + else: + embedding_arr = array("f", embedding) + + distance_fn = _get_distance_function(self.distance_strategy) + query = f""" + SELECT id, + text, + source, + content_metadata, + VECTOR_DISTANCE( + vector, + TO_VECTOR(:embedding, {self._embedding_dim}, FLOAT32), + {distance_fn} + ) AS distance + FROM {self.table_name} + ORDER BY distance + FETCH APPROX FIRST {k} ROWS ONLY + """ + + docs_and_scores = [] + connection = _get_connection(self.client) + if connection is None: + raise ValueError("Failed to acquire a connection.") + + with connection.cursor() as cursor: + cursor.execute(query, embedding=embedding_arr) + results = cursor.fetchall() + + for result in results: + source = json.loads(result[2]) if result[2] else {} + metadata = {} + if isinstance(result[3], oracledb.LOB) and result[3]: + metadata = json.loads(self._get_clob_value(result[3])) + elif isinstance(result[3], dict) and result[3]: + metadata = result[3] + + # combining source data and metadata + metadata.update(source) + + # Apply filter if provided + if filter: + logger.info(f'Filtering on :{filter}') + if not all(metadata.get(key) in value for key, value in filter.items()): + continue + + doc = Document( + page_content=(self._get_clob_value(result[1]) if result[1] is not None else ""), + metadata=metadata, + ) + distance = result[4] + docs_and_scores.append((doc, distance)) + + return docs_and_scores class OracleVDB(VDBRagIngest): @@ -244,6 +327,8 @@ def write_to_index(self, records: list, **kwargs) -> None: meta_dataframe = self.meta_dataframe if meta_dataframe is None and self.csv_file_path is not None: meta_dataframe = pandas_file_reader(self.csv_file_path) + elif isinstance(meta_dataframe, str): + meta_dataframe = pandas_file_reader(meta_dataframe) # Clean records cleaned_records = cleanup_records( @@ -263,7 +348,7 @@ def write_to_index(self, records: list, **kwargs) -> None: with conn.cursor() as cursor: insert_sql = f""" INSERT INTO {self._collection_name} (text, vector, source, content_metadata) - VALUES (:text, :vector, :source, :content_metadata) + VALUES (:text, TO_VECTOR(:vector, {self.config.embeddings.dimensions}, FLOAT32), :source, :content_metadata) """ for i in range(0, total_records, batch_size): @@ -272,14 +357,22 @@ def write_to_index(self, records: list, **kwargs) -> None: for record in batch: vector = record.get("vector", []) - # Convert vector to Oracle VECTOR format - vector_str = "[" + ",".join(str(v) for v in vector) + "]" + + source_val = record.get("source", "") + if isinstance(source_val, dict): + source_val = json.dumps(source_val) + + content_metadata_val = record.get("content_metadata", {}) + if not isinstance(content_metadata_val, str): + content_metadata_val = json.dumps(content_metadata_val) + # Bind vector using TO_VECTOR with dense float32 format + vector_array = array("f", vector) batch_data.append({ "text": record.get("text", ""), - "vector": vector_str, - "source": record.get("source", ""), - "content_metadata": json.dumps(record.get("content_metadata", {})), + "vector": vector_array, + "source": source_val, + "content_metadata": content_metadata_val, }) cursor.executemany(insert_sql, batch_data) @@ -503,8 +596,8 @@ def get_documents(self, collection_name: str) -> list[dict[str, Any]]: with conn.cursor() as cursor: cursor.execute(get_unique_sources_query(table_name)) for row in cursor: - source_name = row[0] - content_metadata = json.loads(row[1]) if row[1] else {} + source_name = json.loads(row[0]).get('source_name') + content_metadata = row[1] if row[1] else {} metadata_dict = {} for item in metadata_schema: @@ -829,7 +922,7 @@ def get_langchain_vectorstore(self, collection_name: str) -> OracleVS: wallet_password=self._ewallet_password ) - return OracleVS( + return OracleVSCompat( client=conn, embedding_function=self._embedding_model, table_name=table_name, @@ -840,9 +933,9 @@ def retrieval_langchain( self, query: str, collection_name: str, - vectorstore: OracleVS | None = None, top_k: int = 10, filter_expr: str | list[dict[str, Any]] = "", + vectorstore: OracleVS | None = None, otel_ctx: Any | None = None, ) -> list[Document]: """Perform semantic search and return documents.""" From e7ad2fdbe4c95909cac8709218fd156e134fe379 Mon Sep 17 00:00:00 2001 From: Swapnil Masurekar Date: Thu, 5 Mar 2026 10:17:48 +0530 Subject: [PATCH 27/81] Upgraded to GA containers for page-elements, graphic-elements, table-structure (#403) Signed-off-by: Swapnil Masurekar --- deploy/compose/nims.yaml | 6 +++--- deploy/helm/nvidia-blueprint-rag/values.yaml | 12 ++++++------ deploy/workbench/compose.yaml | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/deploy/compose/nims.yaml b/deploy/compose/nims.yaml index eb29bfc06..497ed6646 100644 --- a/deploy/compose/nims.yaml +++ b/deploy/compose/nims.yaml @@ -119,7 +119,7 @@ services: profiles: ["", "rag", "vlm-generation"] page-elements: - image: ${YOLOX_IMAGE:-nvcr.io/nvstaging/nim/nemotron-page-elements-v3}:${YOLOX_TAG:-1.8.0-rc.20260218170155-6b1b2aa9dfefc9c3} + image: ${YOLOX_IMAGE:-nvcr.io/nim/nvidia/nemotron-page-elements-v3}:${YOLOX_TAG:-1.8.0} shm_size: 16gb ports: - "8000:8000" @@ -157,7 +157,7 @@ services: profiles: ["", "ingest", "vlm-ingest"] graphic-elements: - image: ${YOLOX_GRAPHIC_ELEMENTS_IMAGE:-nvcr.io/nvstaging/nim/nemotron-graphic-elements-v1}:${YOLOX_GRAPHIC_ELEMENTS_TAG:-1.8.0-rc.20260218170144-35d5a3f270662864} + image: ${YOLOX_GRAPHIC_ELEMENTS_IMAGE:-nvcr.io/nim/nvidia/nemotron-graphic-elements-v1}:${YOLOX_GRAPHIC_ELEMENTS_TAG:-1.8.0} shm_size: 16gb ports: - "8003:8000" @@ -183,7 +183,7 @@ services: profiles: ["", "ingest", "vlm-ingest"] table-structure: - image: ${YOLOX_TABLE_STRUCTURE_IMAGE:-nvcr.io/nvstaging/nim/nemotron-table-structure-v1}:${YOLOX_TABLE_STRUCTURE_TAG:-1.8.0-rc.20260218170204-cca5cb850146dbbc} + image: ${YOLOX_TABLE_STRUCTURE_IMAGE:-nvcr.io/nim/nvidia/nemotron-table-structure-v1}:${YOLOX_TABLE_STRUCTURE_TAG:-1.8.0} shm_size: 16gb ports: - "8006:8000" diff --git a/deploy/helm/nvidia-blueprint-rag/values.yaml b/deploy/helm/nvidia-blueprint-rag/values.yaml index 986caacc9..a3b501983 100644 --- a/deploy/helm/nvidia-blueprint-rag/values.yaml +++ b/deploy/helm/nvidia-blueprint-rag/values.yaml @@ -1050,8 +1050,8 @@ nv-ingest: tolerations: [] replicaCount: 1 image: - repository: nvcr.io/nvstaging/nim/nemotron-graphic-elements-v1 - tag: "1.8.0-rc.20260218170144-35d5a3f270662864" + repository: nvcr.io/nim/nvidia/nemotron-graphic-elements-v1 + tag: "1.8.0" env: - name: NIM_HTTP_API_PORT value: "8000" @@ -1083,8 +1083,8 @@ nv-ingest: tolerations: [] replicaCount: 1 image: - repository: nvcr.io/nvstaging/nim/nemotron-page-elements-v3 - tag: "1.8.0-rc.20260218170155-6b1b2aa9dfefc9c3" + repository: nvcr.io/nim/nvidia/nemotron-page-elements-v3 + tag: "1.8.0" env: - name: NIM_HTTP_API_PORT value: "8000" @@ -1134,8 +1134,8 @@ nv-ingest: tolerations: [] replicaCount: 1 image: - repository: nvcr.io/nvstaging/nim/nemotron-table-structure-v1 - tag: "1.8.0-rc.20260218170204-cca5cb850146dbbc" + repository: nvcr.io/nim/nvidia/nemotron-table-structure-v1 + tag: "1.8.0" env: - name: NIM_HTTP_API_PORT value: "8000" diff --git a/deploy/workbench/compose.yaml b/deploy/workbench/compose.yaml index c6867b9d6..7bdaadf47 100644 --- a/deploy/workbench/compose.yaml +++ b/deploy/workbench/compose.yaml @@ -86,7 +86,7 @@ services: profiles: ["local"] page-elements: - image: ${YOLOX_IMAGE:-nvcr.io/nvstaging/nim/nemotron-page-elements-v3}:${YOLOX_TAG:-1.8.0-rc.20260218170155-6b1b2aa9dfefc9c3} + image: ${YOLOX_IMAGE:-nvcr.io/nim/nvidia/nemotron-page-elements-v3}:${YOLOX_TAG:-1.8.0} ports: - "8000:8000" - "8001:8001" @@ -122,7 +122,7 @@ services: profiles: ["local"] graphic-elements: - image: ${YOLOX_GRAPHIC_ELEMENTS_IMAGE:-nvcr.io/nvstaging/nim/nemotron-graphic-elements-v1}:${YOLOX_GRAPHIC_ELEMENTS_TAG:-1.8.0-rc.20260218170144-35d5a3f270662864} + image: ${YOLOX_GRAPHIC_ELEMENTS_IMAGE:-nvcr.io/nim/nvidia/nemotron-graphic-elements-v1}:${YOLOX_GRAPHIC_ELEMENTS_TAG:-1.8.0} ports: - "8003:8000" - "8004:8001" @@ -147,7 +147,7 @@ services: profiles: ["local"] table-structure: - image: ${YOLOX_TABLE_STRUCTURE_IMAGE:-nvcr.io/nvstaging/nim/nemotron-table-structure-v1}:${YOLOX_TABLE_STRUCTURE_TAG:-1.8.0-rc.20260218170204-cca5cb850146dbbc} + image: ${YOLOX_TABLE_STRUCTURE_IMAGE:-nvcr.io/nim/nvidia/nemotron-table-structure-v1}:${YOLOX_TABLE_STRUCTURE_TAG:-1.8.0} ports: - "8006:8000" - "8007:8001" From 16f5ad136ee09152447dca778a94a7ccf4a212a3 Mon Sep 17 00:00:00 2001 From: Nikhil Kulkarni Date: Sat, 7 Mar 2026 01:39:41 +0530 Subject: [PATCH 28/81] Add SHM size 16gb to reranking and VLM NIM in compose (#405) --- deploy/compose/nims.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/compose/nims.yaml b/deploy/compose/nims.yaml index 497ed6646..2bca3dce2 100644 --- a/deploy/compose/nims.yaml +++ b/deploy/compose/nims.yaml @@ -108,6 +108,7 @@ services: interval: 10s timeout: 20s retries: 100 + shm_size: 16GB deploy: resources: reservations: @@ -323,6 +324,7 @@ services: interval: 10s timeout: 20s retries: 100 + shm_size: 16GB deploy: resources: reservations: From c347f4ff5b913abcc10184b4e360102d6d5dfb80 Mon Sep 17 00:00:00 2001 From: kumar-punit Date: Sat, 7 Mar 2026 01:40:23 +0530 Subject: [PATCH 29/81] Fixed the query to be derived from messages if query is not explicitly given in argument and adding messages list in logs (#404) --- src/nvidia_rag/rag_server/server.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/nvidia_rag/rag_server/server.py b/src/nvidia_rag/rag_server/server.py index dbd5698ed..2ec1db341 100644 --- a/src/nvidia_rag/rag_server/server.py +++ b/src/nvidia_rag/rag_server/server.py @@ -741,6 +741,26 @@ def validate_messages_structure(cls, values): raise ValueError("The last message must have role='user'") return values + @model_validator(mode="before") + @classmethod + def derive_query_from_messages(cls, data): + """When query is not explicitly provided but messages are, derive query from the last user message.""" + if isinstance(data, dict) and "query" not in data: + messages = data.get("messages", []) + for msg in reversed(messages): + if ( + isinstance(msg, dict) + and msg.get("role") == "user" + and msg.get("content") + ): + data["query"] = msg["content"] + break + else: + raise ValueError( + "Either 'query' must be provided or 'messages' must contain at least one user message with content." + ) + return data + # Define the summary response model class SummaryResponse(BaseModel): @@ -1596,6 +1616,11 @@ def sanitize_query_for_logging(query): request_data = { "query": sanitize_query_for_logging(data.query), + "messages": [ + {"role": msg.role, "content": msg.content} for msg in data.messages + ] + if data.messages + else [], "reranker_top_k": data.reranker_top_k, "vdb_top_k": data.vdb_top_k, "collection_names": data.collection_names, From 41b8f34cecad3c5ca1f74a5c86eff8cdaf2dd899 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Mon, 9 Mar 2026 11:06:57 -0500 Subject: [PATCH 30/81] mispelled ingestor --- .github/workflows/build-push-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push-pr.yml b/.github/workflows/build-push-pr.yml index e858077c7..3e49edd99 100644 --- a/.github/workflows/build-push-pr.yml +++ b/.github/workflows/build-push-pr.yml @@ -79,7 +79,7 @@ jobs: - name: Build Ingest Docker Image run: | - docker build -t "${{ env.INGEST_IMAGE_NAME }}":pr-${{ steps.short-sha.outputs.value }} -f src/nvidia_rag/ingest_server/Dockerfile . + docker build -t "${{ env.INGEST_IMAGE_NAME }}":pr-${{ steps.short-sha.outputs.value }} -f src/nvidia_rag/ingestor_server/Dockerfile . - name: Tag and Push Ingest Docker Image env: From 37081110f9012c1fa9e5cb8b61cc133566d32b71 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Mon, 9 Mar 2026 11:13:44 -0500 Subject: [PATCH 31/81] added more vars for login --- .github/workflows/build-push-pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-push-pr.yml b/.github/workflows/build-push-pr.yml index 3e49edd99..65400c6d4 100644 --- a/.github/workflows/build-push-pr.yml +++ b/.github/workflows/build-push-pr.yml @@ -31,6 +31,8 @@ jobs: id: login-ocir with: auth_token: ${{ secrets.OCI_AUTH_TOKEN }} + username: ${{ secrets.OCI_CLI_USER }} + tenancy_namespace: ${{ secrets.OCI_TENANCY_NAMESPACE }} - name: Compute short SHA id: short-sha From 930cf936f1f54aa249bbc73fd62157588b8975b2 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Mon, 9 Mar 2026 11:17:28 -0500 Subject: [PATCH 32/81] added more vars for login for other build --- .github/workflows/build-push-pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-push-pr.yml b/.github/workflows/build-push-pr.yml index 65400c6d4..2a8b7d656 100644 --- a/.github/workflows/build-push-pr.yml +++ b/.github/workflows/build-push-pr.yml @@ -74,6 +74,8 @@ jobs: id: login-ocir with: auth_token: ${{ secrets.OCI_AUTH_TOKEN }} + username: ${{ secrets.OCI_CLI_USER }} + tenancy_namespace: ${{ secrets.OCI_TENANCY_NAMESPACE }} - name: Compute short SHA id: short-sha From 1a11733169fb17e80537f7ded879472a1ada4d8c Mon Sep 17 00:00:00 2001 From: Swapnil Masurekar Date: Mon, 9 Mar 2026 23:24:16 +0530 Subject: [PATCH 33/81] Added url for nvidia/llama-nemotron-rerank-1b-v2 model for cloud endpoint (#412) Signed-off-by: Swapnil Masurekar --- deploy/compose/.env | 2 +- docs/python-client.md | 4 ++-- docs/retrieval-only-deployment.md | 2 +- docs/text_only_ingest.md | 2 +- notebooks/building_rag_vdb_operator.ipynb | 2 +- notebooks/image_input.ipynb | 2 +- notebooks/rag_library_lite_usage.ipynb | 2 +- notebooks/rag_library_usage.ipynb | 4 ++-- notebooks/summarization.ipynb | 2 +- tests/unit/test_rag_server/test_rag_server_health.py | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/deploy/compose/.env b/deploy/compose/.env index cc80cfaf0..e9c4cd68d 100644 --- a/deploy/compose/.env +++ b/deploy/compose/.env @@ -45,7 +45,7 @@ export YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL=grpc # export APP_FILTEREXPRESSIONGENERATOR_MODELNAME=nvidia/llama-3.3-nemotron-super-49b-v1.5 # export APP_FILTEREXPRESSIONGENERATOR_SERVERURL="" # export SUMMARY_LLM="nvidia/llama-3.3-nemotron-super-49b-v1.5" -# export APP_RANKING_SERVERURL="" +# export APP_RANKING_SERVERURL="https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking" # export SUMMARY_LLM_SERVERURL="" # export OCR_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr # export OCR_INFER_PROTOCOL=http diff --git a/docs/python-client.md b/docs/python-client.md index e052b5e91..e3d8b3119 100644 --- a/docs/python-client.md +++ b/docs/python-client.md @@ -362,7 +362,7 @@ from nvidia_rag.utils.configuration import NvidiaRAGConfig # }, # "ranking": { # "model_name": "nvidia/llama-nemotron-rerank-1b-v2", -# "server_url": "", +# "server_url": "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking", # }, # }) @@ -371,7 +371,7 @@ config_rag = NvidiaRAGConfig.from_yaml("config.yaml") # Update config for cloud deployment if using Option 2 if DEPLOYMENT_MODE == "cloud": config_rag.embeddings.server_url = "https://integrate.api.nvidia.com/v1" - config_rag.ranking.server_url = "" # Empty uses NVIDIA API catalog + config_rag.ranking.server_url = "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking" config_rag.llm.server_url = "" # Empty uses NVIDIA API catalog # Initialize NvidiaRAG with config diff --git a/docs/retrieval-only-deployment.md b/docs/retrieval-only-deployment.md index 7f7f94475..09d70c311 100644 --- a/docs/retrieval-only-deployment.md +++ b/docs/retrieval-only-deployment.md @@ -116,7 +116,7 @@ For an even lighter deployment, use [NVIDIA-hosted NIMs](deploy-docker-nvidia-ho ```bash # Configure to use NVIDIA-hosted endpoints export APP_EMBEDDINGS_SERVERURL="" -export APP_RANKING_SERVERURL="" +export APP_RANKING_SERVERURL="https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking" ``` :::{note} diff --git a/docs/text_only_ingest.md b/docs/text_only_ingest.md index 10cb6e9b1..cf61f69ec 100644 --- a/docs/text_only_ingest.md +++ b/docs/text_only_ingest.md @@ -69,7 +69,7 @@ In case you are [interacting with cloud hosted models](deploy-docker-nvidia-host ```bash export APP_EMBEDDINGS_SERVERURL="" export APP_LLM_SERVERURL="" - export APP_RANKING_SERVERURL="" + export APP_RANKING_SERVERURL="https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking" export YOLOX_HTTP_ENDPOINT="https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3" export YOLOX_INFER_PROTOCOL="http" ``` diff --git a/notebooks/building_rag_vdb_operator.ipynb b/notebooks/building_rag_vdb_operator.ipynb index 1852f1d9c..0cab35d6e 100644 --- a/notebooks/building_rag_vdb_operator.ipynb +++ b/notebooks/building_rag_vdb_operator.ipynb @@ -2092,7 +2092,7 @@ " config_ingestor.embeddings.server_url = \"https://integrate.api.nvidia.com/v1\"\n", " config_ingestor.llm.server_url = \"\" # Empty uses NVIDIA API catalog\n", " config_rag.embeddings.server_url = \"https://integrate.api.nvidia.com/v1\"\n", - " config_rag.ranking.server_url = \"\" # Empty uses NVIDIA API catalog\n", + " config_rag.ranking.server_url = \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking\"\n", " config_rag.llm.server_url = \"\" # Empty uses NVIDIA API catalog\n", "\n", "# Create embedding model using the configuration\n", diff --git a/notebooks/image_input.ipynb b/notebooks/image_input.ipynb index 4f778a2fa..9332f4144 100644 --- a/notebooks/image_input.ipynb +++ b/notebooks/image_input.ipynb @@ -360,7 +360,7 @@ "outputs": [], "source": [ "# Start the RAG server (accessible at localhost:8081)\n", - "os.environ[\"APP_RANKING_SERVERURL\"] = \"\"\n", + "os.environ[\"APP_RANKING_SERVERURL\"] = \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking\"\n", "! docker compose -f ../deploy/compose/docker-compose-rag-server.yaml up -d --build" ] }, diff --git a/notebooks/rag_library_lite_usage.ipynb b/notebooks/rag_library_lite_usage.ipynb index 915eb00ca..ded451558 100644 --- a/notebooks/rag_library_lite_usage.ipynb +++ b/notebooks/rag_library_lite_usage.ipynb @@ -482,7 +482,7 @@ "\n", "# Set config for cloud API endpoints\n", "config_rag.embeddings.server_url = \"https://integrate.api.nvidia.com/v1\"\n", - "config_rag.ranking.server_url = \"\" # Empty uses NVIDIA API catalog\n", + "config_rag.ranking.server_url = \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking\"\n", "config_rag.llm.server_url = \"\" # Empty uses NVIDIA API catalog\n", "\n", "# Initialize NvidiaRAG with config\n", diff --git a/notebooks/rag_library_usage.ipynb b/notebooks/rag_library_usage.ipynb index 894e82620..659f2f304 100644 --- a/notebooks/rag_library_usage.ipynb +++ b/notebooks/rag_library_usage.ipynb @@ -630,7 +630,7 @@ "# },\n", "# \"ranking\": {\n", "# \"model_name\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", - "# \"server_url\": \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking/v1\",\n", + "# \"server_url\": \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking\",\n", "# },\n", "# }\n", "# )\n", @@ -640,7 +640,7 @@ "# Update config for cloud deployment if using Option 2\n", "if DEPLOYMENT_MODE == \"cloud\":\n", " config_rag.embeddings.server_url = \"https://integrate.api.nvidia.com/v1\"\n", - " config_rag.ranking.server_url = \"\" # Empty uses NVIDIA API catalog\n", + " config_rag.ranking.server_url = \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking\"\n", " config_rag.llm.server_url = \"\" # Empty uses NVIDIA API catalog\n", "\n", "# Initialize NvidiaRAG with config\n", diff --git a/notebooks/summarization.ipynb b/notebooks/summarization.ipynb index d0c7ef285..63d39c5ac 100644 --- a/notebooks/summarization.ipynb +++ b/notebooks/summarization.ipynb @@ -545,7 +545,7 @@ "if DEPLOYMENT_MODE == \"cloud\":\n", " config.embeddings.server_url = \"https://integrate.api.nvidia.com/v1\"\n", " config.llm.server_url = \"\" # Empty uses NVIDIA API catalog\n", - " config.ranking.server_url = \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking/v1\"\n", + " config.ranking.server_url = \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking\"\n", " config.summarizer.server_url = \"\" # Empty uses NVIDIA API catalog\n", "else:\n", " config.embeddings.server_url = \"nemotron-embedding-ms:8000/v1\"\n", diff --git a/tests/unit/test_rag_server/test_rag_server_health.py b/tests/unit/test_rag_server/test_rag_server_health.py index 966a85e74..fb8783dff 100644 --- a/tests/unit/test_rag_server/test_rag_server_health.py +++ b/tests/unit/test_rag_server/test_rag_server_health.py @@ -415,7 +415,7 @@ async def test_check_all_services_health_nvidia_api_catalog(self, mock_config): mock_config.query_rewriter.server_url = "https://ai.api.nvidia.com/v1" mock_config.embeddings.server_url = "https://api.nvcf.nvidia.com/v2" mock_config.ranking.server_url = ( - "" # Empty URL should be treated as API catalog + "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking" # Empty URL should be treated as API catalog ) mock_vdb_op = MagicMock() From 65e965b527b6cb1f88a7cc6f42e88a583c56e502 Mon Sep 17 00:00:00 2001 From: nv-pranjald <150428320+nv-pranjald@users.noreply.github.com> Date: Mon, 9 Mar 2026 23:24:46 +0530 Subject: [PATCH 34/81] fix start proble failureThreshold to 750 (#411) --- .../nvidia-blueprint-rag/templates/llm-nim.yaml | 4 ++++ deploy/helm/nvidia-blueprint-rag/values.yaml | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/deploy/helm/nvidia-blueprint-rag/templates/llm-nim.yaml b/deploy/helm/nvidia-blueprint-rag/templates/llm-nim.yaml index 60f043973..f103a1a72 100644 --- a/deploy/helm/nvidia-blueprint-rag/templates/llm-nim.yaml +++ b/deploy/helm/nvidia-blueprint-rag/templates/llm-nim.yaml @@ -61,4 +61,8 @@ spec: {{- end }} expose: {{ toYaml $nimLlm.expose | nindent 4 }} + {{- with $nimLlm.startupProbe }} + startupProbe: +{{ toYaml . | nindent 4 }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/deploy/helm/nvidia-blueprint-rag/values.yaml b/deploy/helm/nvidia-blueprint-rag/values.yaml index a3b501983..3a39c1224 100644 --- a/deploy/helm/nvidia-blueprint-rag/values.yaml +++ b/deploy/helm/nvidia-blueprint-rag/values.yaml @@ -703,6 +703,8 @@ nimOperator: value: "1" - name: NIM_SERVED_MODEL_NAME value: "nvidia/llama-3.3-nemotron-super-49b-v1.5" + - name: NIM_MAX_MODEL_LEN + value: "131072" # - name: CUDA_VISIBLE_DEVICES # value: "0" expose: @@ -711,6 +713,17 @@ nimOperator: type: ClusterIP port: 8000 grpcPort: 8001 + startupProbe: + enabled: true + probe: + httpGet: + path: /v1/health/ready + port: 8000 + initialDelaySeconds: 60 + periodSeconds: 10 + failureThreshold: 750 + timeoutSeconds: 5 + # subsection: nvidia-nim-llama-32-nv-embedqa-1b-v2 # NIM Text Embedding nvidia-nim-llama-32-nv-embedqa-1b-v2: From 5a71ec16cedca7bbdb33ec12066acb3690d4e56b Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Mon, 9 Mar 2026 21:42:12 -0500 Subject: [PATCH 35/81] reverting to the right version and cleaning up --- pyproject.toml | 8 +- src/nvidia_rag/ingestor_server/nvingest.py | 2 +- uv.lock | 349 ++++++++------------- 3 files changed, 139 insertions(+), 220 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3ce5fbff4..2fddea4a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,8 +59,8 @@ rag = [ ] ingest = [ # nv-ingest dependencies (required for ingestion operations) - "nv-ingest-api==26.1.1", - "nv-ingest-client==26.1.1", + "nv-ingest-api==25.9.0", + "nv-ingest-client==25.9.0", "tritonclient==2.57.0", # Other ingest dependencies "langchain-openai>=0.2", @@ -81,8 +81,8 @@ ingest = [ ] all = [ # nv-ingest dependencies (required for ingestion operations) - "nv-ingest-api==26.1.1", - "nv-ingest-client==26.1.1", + "nv-ingest-api==25.9.0", + "nv-ingest-client==25.9.0", "tritonclient==2.57.0", # RAG + Ingest dependencies "langchain-openai>=0.2", diff --git a/src/nvidia_rag/ingestor_server/nvingest.py b/src/nvidia_rag/ingestor_server/nvingest.py index 55e218c53..42d350b13 100644 --- a/src/nvidia_rag/ingestor_server/nvingest.py +++ b/src/nvidia_rag/ingestor_server/nvingest.py @@ -66,7 +66,7 @@ def get_nv_ingest_client( message_client_hostname=config.nv_ingest.message_client_hostname, # REST port, defaults to 7670 message_client_port=config.nv_ingest.message_client_port, - message_client_kwargs={"api_version": "v2"}, + message_client_kwargs={"api_version": "v1"}, ) return client diff --git a/uv.lock b/uv.lock index b1af55d6b..2a1f0d61d 100644 --- a/uv.lock +++ b/uv.lock @@ -292,15 +292,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cd/3a/577b549de0cc09d95f11087ee63c739bba856cd3952697eec4c4bb91350a/bleach-6.3.0-py3-none-any.whl", hash = "sha256:fe10ec77c93ddf3d13a73b035abaac7a9f5e436513864ccdad516693213c65d6", size = 164437, upload-time = "2025-10-27T17:57:37.538Z" }, ] -[[package]] -name = "boltons" -version = "25.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/63/54/71a94d8e02da9a865587fb3fff100cb0fc7aa9f4d5ed9ed3a591216ddcc7/boltons-25.0.0.tar.gz", hash = "sha256:e110fbdc30b7b9868cb604e3f71d4722dd8f4dcb4a5ddd06028ba8f1ab0b5ace", size = 246294, upload-time = "2025-02-03T05:57:59.129Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/7f/0e961cf3908bc4c1c3e027de2794f867c6c89fb4916fc7dba295a0e80a2d/boltons-25.0.0-py3-none-any.whl", hash = "sha256:dc9fb38bf28985715497d1b54d00b62ea866eca3938938ea9043e254a3a6ca62", size = 194210, upload-time = "2025-02-03T05:57:56.705Z" }, -] - [[package]] name = "build" version = "1.4.0" @@ -542,24 +533,24 @@ wheels = [ ] [[package]] -name = "decorator" -version = "5.2.1" +name = "deprecated" +version = "1.3.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/fa/6d96a0978d19e17b68d634497769987b16c8f4cd0a7a05048bec693caa6b/decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360", size = 56711, upload-time = "2025-02-24T04:41:34.073Z" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" }, + { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] [[package]] -name = "deprecation" -version = "2.1.0" +name = "dirtyjson" +version = "1.0.8" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788, upload-time = "2020-04-20T14:23:38.738Z" } +sdist = { url = "https://files.pythonhosted.org/packages/db/04/d24f6e645ad82ba0ef092fa17d9ef7a21953781663648a01c9371d9e8e98/dirtyjson-1.0.8.tar.gz", hash = "sha256:90ca4a18f3ff30ce849d100dcf4a003953c79d3a2348ef056f1d9c22231a25fd", size = 30782, upload-time = "2022-11-28T23:32:33.319Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178, upload-time = "2020-04-20T14:23:36.581Z" }, + { url = "https://files.pythonhosted.org/packages/68/69/1bcf70f81de1b4a9f21b3a62ec0c83bdff991c88d6cc2267d02408457e88/dirtyjson-1.0.8-py3-none-any.whl", hash = "sha256:125e27248435a58acace26d5c2c4c11a1c0de0a9c5124c5a94ba78e517d74f53", size = 25197, upload-time = "2022-11-28T23:32:31.219Z" }, ] [[package]] @@ -613,18 +604,6 @@ vectorstore-mmr = [ { name = "simsimd" }, ] -[[package]] -name = "face" -version = "24.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "boltons" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ac/79/2484075a8549cd64beae697a8f664dee69a5ccf3a7439ee40c8f93c1978a/face-24.0.0.tar.gz", hash = "sha256:611e29a01ac5970f0077f9c577e746d48c082588b411b33a0dd55c4d872949f6", size = 62732, upload-time = "2024-11-02T05:24:26.095Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/47/21867c2e5fd006c8d36a560df9e32cb4f1f566b20c5dd41f5f8a2124f7de/face-24.0.0-py3-none-any.whl", hash = "sha256:0e2c17b426fa4639a4e77d1de9580f74a98f4869ba4c7c8c175b810611622cd3", size = 54742, upload-time = "2024-11-02T05:24:24.939Z" }, -] - [[package]] name = "fastapi" version = "0.128.0" @@ -640,18 +619,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5c/05/5cbb59154b093548acd0f4c7c474a118eda06da25aa75c616b72d8fcd92a/fastapi-0.128.0-py3-none-any.whl", hash = "sha256:aebd93f9716ee3b4f4fcfe13ffb7cf308d99c9f3ab5622d8877441072561582d", size = 103094, upload-time = "2025-12-27T15:21:12.154Z" }, ] -[[package]] -name = "ffmpeg-python" -version = "0.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "future" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dd/5e/d5f9105d59c1325759d838af4e973695081fbbc97182baf73afc78dec266/ffmpeg-python-0.2.0.tar.gz", hash = "sha256:65225db34627c578ef0e11c8b1eb528bb35e024752f6f10b78c011f6f64c4127", size = 21543, upload-time = "2019-07-06T00:19:08.989Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/0c/56be52741f75bad4dc6555991fabd2e07b432d333da82c11ad701123888a/ffmpeg_python-0.2.0-py3-none-any.whl", hash = "sha256:ac441a0404e053f8b6a1113a77c0f452f1cfc62f6344a769475ffdc0f56c23c5", size = 25024, upload-time = "2019-07-06T00:19:07.215Z" }, -] - [[package]] name = "filelock" version = "3.20.3" @@ -760,29 +727,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/01/c9/97cc5aae1648dcb851958a3ddf73ccd7dbe5650d95203ecb4d7720b4cdbf/fsspec-2026.1.0-py3-none-any.whl", hash = "sha256:cb76aa913c2285a3b49bdd5fc55b1d7c708d7208126b60f2eb8194fe1b4cbdcc", size = 201838, upload-time = "2026-01-09T15:21:34.041Z" }, ] -[[package]] -name = "future" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a7/b2/4140c69c6a66432916b26158687e821ba631a4c9273c474343badf84d3ba/future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05", size = 1228490, upload-time = "2024-02-21T11:52:38.461Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/71/ae30dadffc90b9006d77af76b393cb9dfbfc9629f339fc1574a1c52e6806/future-1.0.0-py3-none-any.whl", hash = "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216", size = 491326, upload-time = "2024-02-21T11:52:35.956Z" }, -] - -[[package]] -name = "glom" -version = "25.12.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "boltons" }, - { name = "face" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/78/74/8387f95565ba7c30cd152a585b275ebb9a834d1d32782425c5d2fe0a102c/glom-25.12.0.tar.gz", hash = "sha256:1ae7da88be3693df40ad27bdf57a765a55c075c86c971bcddd67927403eb0069", size = 196128, upload-time = "2025-12-29T06:29:07.274Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/e6/4129d9a3baa72d747533bb33376543ccadd9a7f9944e5a6e3ae2e245f5d6/glom-25.12.0-py3-none-any.whl", hash = "sha256:b9f21e77f71a6576a43864e85066b8cc3f0f778d0d50961563f8981377a6dcb1", size = 103295, upload-time = "2025-12-29T06:29:06.074Z" }, -] - [[package]] name = "googleapis-common-protos" version = "1.72.0" @@ -1007,33 +951,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, ] -[[package]] -name = "imageio" -version = "2.37.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "pillow" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a3/6f/606be632e37bf8d05b253e8626c2291d74c691ddc7bcdf7d6aaf33b32f6a/imageio-2.37.2.tar.gz", hash = "sha256:0212ef2727ac9caa5ca4b2c75ae89454312f440a756fcfc8ef1993e718f50f8a", size = 389600, upload-time = "2025-11-04T14:29:39.898Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/fe/301e0936b79bcab4cacc7548bf2853fc28dced0a578bab1f7ef53c9aa75b/imageio-2.37.2-py3-none-any.whl", hash = "sha256:ad9adfb20335d718c03de457358ed69f141021a333c40a53e57273d8a5bd0b9b", size = 317646, upload-time = "2025-11-04T14:29:37.948Z" }, -] - -[[package]] -name = "imageio-ffmpeg" -version = "0.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/44/bd/c3343c721f2a1b0c9fc71c1aebf1966a3b7f08c2eea8ed5437a2865611d6/imageio_ffmpeg-0.6.0.tar.gz", hash = "sha256:e2556bed8e005564a9f925bb7afa4002d82770d6b08825078b7697ab88ba1755", size = 25210, upload-time = "2025-01-16T21:34:32.747Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/58/87ef68ac83f4c7690961bce288fd8e382bc5f1513860fc7f90a9c1c1c6bf/imageio_ffmpeg-0.6.0-py3-none-macosx_10_9_intel.macosx_10_9_x86_64.whl", hash = "sha256:9d2baaf867088508d4a3458e61eeb30e945c4ad8016025545f66c4b5aaef0a61", size = 24932969, upload-time = "2025-01-16T21:34:20.464Z" }, - { url = "https://files.pythonhosted.org/packages/40/5c/f3d8a657d362cc93b81aab8feda487317da5b5d31c0e1fdfd5e986e55d17/imageio_ffmpeg-0.6.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b1ae3173414b5fc5f538a726c4e48ea97edc0d2cdc11f103afee655c463fa742", size = 21113891, upload-time = "2025-01-16T21:34:00.277Z" }, - { url = "https://files.pythonhosted.org/packages/33/e7/1925bfbc563c39c1d2e82501d8372734a5c725e53ac3b31b4c2d081e895b/imageio_ffmpeg-0.6.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1d47bebd83d2c5fc770720d211855f208af8a596c82d17730aa51e815cdee6dc", size = 25632706, upload-time = "2025-01-16T21:33:53.475Z" }, - { url = "https://files.pythonhosted.org/packages/a0/2d/43c8522a2038e9d0e7dbdf3a61195ecc31ca576fb1527a528c877e87d973/imageio_ffmpeg-0.6.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:c7e46fcec401dd990405049d2e2f475e2b397779df2519b544b8aab515195282", size = 29498237, upload-time = "2025-01-16T21:34:13.726Z" }, - { url = "https://files.pythonhosted.org/packages/a0/13/59da54728351883c3c1d9fca1710ab8eee82c7beba585df8f25ca925f08f/imageio_ffmpeg-0.6.0-py3-none-win32.whl", hash = "sha256:196faa79366b4a82f95c0f4053191d2013f4714a715780f0ad2a68ff37483cc2", size = 19652251, upload-time = "2025-01-16T21:34:06.812Z" }, - { url = "https://files.pythonhosted.org/packages/2c/c6/fa760e12a2483469e2bf5058c5faff664acf66cadb4df2ad6205b016a73d/imageio_ffmpeg-0.6.0-py3-none-win_amd64.whl", hash = "sha256:02fa47c83703c37df6bfe4896aab339013f62bf02c5ebf2dce6da56af04ffc0a", size = 31246824, upload-time = "2025-01-16T21:34:28.6Z" }, -] - [[package]] name = "imagesize" version = "1.4.1" @@ -1136,6 +1053,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2f/9c/6753e6522b8d0ef07d3a3d239426669e984fb0eba15a315cdbc1253904e4/jiter-0.12.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c24e864cb30ab82311c6425655b0cdab0a98c5d973b065c66a3f020740c2324c", size = 346110, upload-time = "2025-11-09T20:49:21.817Z" }, ] +[[package]] +name = "joblib" +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, +] + [[package]] name = "jsonpatch" version = "1.33" @@ -1157,56 +1083,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942", size = 7595, upload-time = "2024-06-10T19:24:40.698Z" }, ] -[[package]] -name = "lance-namespace" -version = "0.4.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lance-namespace-urllib3-client" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b4/b5/0c3c55cf336b1e90392c2e24ac833551659e8bb3c61644b2d94825eb31bd/lance_namespace-0.4.5.tar.gz", hash = "sha256:0aee0abed3a1fa762c2955c7d12bb3004cea5c82ba28f6fcb9fe79d0cc19e317", size = 9827, upload-time = "2026-01-07T19:20:23.005Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/34/88/173687dad72baf819223e3b506898e386bc88c26ff8da5e8013291e02daf/lance_namespace-0.4.5-py3-none-any.whl", hash = "sha256:cd1a4f789de03ba23a0c16f100b1464cca572a5d04e428917a54d09db912d548", size = 11703, upload-time = "2026-01-07T19:20:25.394Z" }, -] - -[[package]] -name = "lance-namespace-urllib3-client" -version = "0.4.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "python-dateutil" }, - { name = "typing-extensions" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/97/a9/4e527c2f05704565618b239b0965f829d1a194837f01234af3f8e2f33d92/lance_namespace_urllib3_client-0.4.5.tar.gz", hash = "sha256:184deda8cf8700926d994618187053c644eb1f2866a4479e7b80843cacc92b1c", size = 159726, upload-time = "2026-01-07T19:20:24.025Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/86/0adee7190408a28dcc5a0562c674537457e3de59ee51d1c724ecdc4a9930/lance_namespace_urllib3_client-0.4.5-py3-none-any.whl", hash = "sha256:2ee154d616ba4721f0bfdf043d33c4fef2e79d380653e2f263058ab00fb4adf4", size = 277969, upload-time = "2026-01-07T19:20:26.597Z" }, -] - -[[package]] -name = "lancedb" -version = "0.26.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecation" }, - { name = "lance-namespace" }, - { name = "numpy" }, - { name = "overrides", marker = "python_full_version < '3.12'" }, - { name = "packaging" }, - { name = "pyarrow" }, - { name = "pydantic" }, - { name = "tqdm" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/b5/110651418ceb1fa4ff2eb74ce4bad911ecf49dc765b134f0201d5564aab8/lancedb-0.26.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:b1c4389134ede49e4be0497b9719f573f447e627426bb9e6fc1b642db11fb22d", size = 43416143, upload-time = "2026-01-02T17:57:07.232Z" }, - { url = "https://files.pythonhosted.org/packages/81/8a/b48a14281d7875e5bfccf22d911d9e1fa019c1fe7b805d290a4449e3cf60/lancedb-0.26.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07abd18e0aa4730442d0361bab4491ad469de14f9087c3542e56ca6d7fcda473", size = 45302392, upload-time = "2026-01-02T18:04:55.963Z" }, - { url = "https://files.pythonhosted.org/packages/4b/d0/8f6bc531f290206c7a0061236928710506598a2591ff1fcaea477fc52e7f/lancedb-0.26.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df8eb631519c6ede9975099bea187ea25a09e4617a421fe19e5e1613651cd62f", size = 48372676, upload-time = "2026-01-02T18:08:12.373Z" }, - { url = "https://files.pythonhosted.org/packages/f5/13/d8db83335ddf28afe1fb814ca995da7f67826f337d547e54471d7d425dd1/lancedb-0.26.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2941c9f8aa22244002307c4da5d19f12ab77dcb0569eb4f8a48b60e9c4fdee79", size = 45318771, upload-time = "2026-01-02T18:04:26.429Z" }, - { url = "https://files.pythonhosted.org/packages/08/94/10e9d4b5ba49eeba72024d310dc42e0c24feb8d5676f48e989198121a8a0/lancedb-0.26.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0d5f125b98836a49095c492085f5ecf3a78906fafcab59c367d9347eb372a4cc", size = 48425627, upload-time = "2026-01-02T18:11:57.443Z" }, - { url = "https://files.pythonhosted.org/packages/17/5d/d7a834ce8dd9c5e6ef7a0e308c7de5f87bb8f04c0944a1bea617d9d42dc7/lancedb-0.26.1-cp39-abi3-win_amd64.whl", hash = "sha256:9338d34c6e7472c97e49fd6b2638b29d3d087e8b002d92cafdbb46a8b0b1480e", size = 53214501, upload-time = "2026-01-02T22:34:06.836Z" }, -] - [[package]] name = "langchain" version = "1.2.7" @@ -1431,6 +1307,65 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, ] +[[package]] +name = "llama-index-core" +version = "0.10.45" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "dataclasses-json" }, + { name = "deprecated" }, + { name = "dirtyjson" }, + { name = "fsspec" }, + { name = "httpx" }, + { name = "llamaindex-py-client" }, + { name = "nest-asyncio" }, + { name = "networkx" }, + { name = "nltk" }, + { name = "numpy" }, + { name = "openai" }, + { name = "pandas" }, + { name = "pillow" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sqlalchemy", extra = ["asyncio"] }, + { name = "tenacity" }, + { name = "tiktoken" }, + { name = "tqdm" }, + { name = "typing-extensions" }, + { name = "typing-inspect" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/6b/1cd00e76b5bdc0060567fdc2c41e9ba6f29a17ec90c36afc5ac9540a4306/llama_index_core-0.10.45.tar.gz", hash = "sha256:f32d0448e7193ff45c8e84abd49493be030998fc8f1a0cab069387deef3e577c", size = 15109807, upload-time = "2024-06-14T22:43:47.813Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/fc/75b116bafcf13346378fbaa138dc3c96226c672ccb123b32c4abba6ba3b3/llama_index_core-0.10.45-py3-none-any.whl", hash = "sha256:8c800c7221322b8e1cbbbc13325039b5fe3575d4b0e0be14ac9a8f1e5d14fee3", size = 15436131, upload-time = "2024-06-14T22:43:39.897Z" }, +] + +[[package]] +name = "llama-index-embeddings-nvidia" +version = "0.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llama-index-core" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/7b/01680981eda1d3f662451d86e9e2ab4cd1c64777f948c1c124473208b62a/llama_index_embeddings_nvidia-0.1.5.tar.gz", hash = "sha256:1b6d4d679a2176b8a174af21724c90d560ff417375b20e6bd8045200b45ccd54", size = 5579, upload-time = "2024-08-12T14:46:51.979Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/f7/3a5b84110761c7e4a5259b272775b98bcb41c7b5856ac19be1fd8b9c2bfd/llama_index_embeddings_nvidia-0.1.5-py3-none-any.whl", hash = "sha256:6fbe1efba56d96e67d434b77043f98f585a01d11067c1ea8fb52219609b6b1c3", size = 5885, upload-time = "2024-08-12T14:46:50.646Z" }, +] + +[[package]] +name = "llamaindex-py-client" +version = "0.1.19" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a0/1b/9dbd37e1c2efb21e91047b2ae7379e309e45a884ed35aa5d3e754ea69cb3/llamaindex_py_client-0.1.19.tar.gz", hash = "sha256:73f74792bb8c092bae6dc626627a09ac13a099fa8d10f8fcc83e17a2b332cca7", size = 54097, upload-time = "2024-04-26T15:59:05.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/9c/a1a3086f023a8957ebd527c98d9356b6dd9409edef827ffdeee484f47abf/llamaindex_py_client-0.1.19-py3-none-any.whl", hash = "sha256:fd9416fd78b97209bf323bc3c7fab314499778563e7274f10853ad560563d10e", size = 141926, upload-time = "2024-04-26T15:59:02.888Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -1558,24 +1493,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3e/9a/b697530a882588a84db616580f2ba5d1d515c815e11c30d219145afeec87/minio-7.2.20-py3-none-any.whl", hash = "sha256:eb33dd2fb80e04c3726a76b13241c6be3c4c46f8d81e1d58e757786f6501897e", size = 93751, upload-time = "2025-11-27T00:37:13.993Z" }, ] -[[package]] -name = "moviepy" -version = "2.2.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "decorator" }, - { name = "imageio" }, - { name = "imageio-ffmpeg" }, - { name = "numpy" }, - { name = "pillow" }, - { name = "proglog" }, - { name = "python-dotenv" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/de/61/15f9476e270f64c78a834e7459ca045d669f869cec24eed26807b8cd479d/moviepy-2.2.1.tar.gz", hash = "sha256:c80cb56815ece94e5e3e2d361aa40070eeb30a09d23a24c4e684d03e16deacb1", size = 58431438, upload-time = "2025-05-21T19:31:52.601Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/73/7d3b2010baa0b5eb1e4dfa9e4385e89b6716be76f2fa21a6c0fe34b68e5a/moviepy-2.2.1-py3-none-any.whl", hash = "sha256:6b56803fec2ac54b557404126ac1160e65448e03798fa282bd23e8fab3795060", size = 129871, upload-time = "2025-05-21T19:31:50.11Z" }, -] - [[package]] name = "mpmath" version = "1.3.0" @@ -1693,6 +1610,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d3/ac/686789b9145413f1a61878c407210e41bfdb097976864e0913078b24098c/myst_parser-5.0.0-py3-none-any.whl", hash = "sha256:ab31e516024918296e169139072b81592336f2fef55b8986aa31c9f04b5f7211", size = 84533, upload-time = "2026-01-15T09:08:16.788Z" }, ] +[[package]] +name = "nest-asyncio" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, +] + +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + +[[package]] +name = "nltk" +version = "3.9.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "joblib" }, + { name = "regex" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/8f/915e1c12df07c70ed779d18ab83d065718a926e70d3ea33eb0cd66ffb7c0/nltk-3.9.3.tar.gz", hash = "sha256:cb5945d6424a98d694c2b9a0264519fab4363711065a46aa0ae7a2195b92e71f", size = 2923673, upload-time = "2026-02-24T12:05:53.833Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/7e/9af5a710a1236e4772de8dfcc6af942a561327bb9f42b5b4a24d0cf100fd/nltk-3.9.3-py3-none-any.whl", hash = "sha256:60b3db6e9995b3dd976b1f0fa7dec22069b2677e759c28eb69b62ddd44870522", size = 1525385, upload-time = "2026-02-24T12:05:46.54Z" }, +] + [[package]] name = "numpy" version = "2.4.1" @@ -1753,28 +1703,23 @@ wheels = [ [[package]] name = "nv-ingest-api" -version = "26.1.1" +version = "25.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff" }, - { name = "ffmpeg-python" }, - { name = "fsspec" }, - { name = "glom" }, - { name = "moviepy" }, { name = "pandas" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "tritonclient" }, - { name = "universal-pathlib" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/41/eb/e0469e918d617175e1d3bcf952f0ca8e9b7756fce7817d5386ac4ddca154/nv_ingest_api-26.1.1.tar.gz", hash = "sha256:063d51f1d560bf03d7a595ff3ecebac1bffae45607cf6bd01e4fa8ca2265a884", size = 259532, upload-time = "2026-01-13T23:44:11.112Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8b/84/70a7d089f4b7961eff4dfa8e4d0a485a7fba5c4403bb061f81789f25578a/nv_ingest_api-25.9.0.tar.gz", hash = "sha256:45f611e51560b8175ce06097f9b954500ae19aba4322a59796f1fb533e6d5fc2", size = 209566, upload-time = "2025-09-16T21:06:10.994Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/42/51/cd93750a1c5797d8d12e843bb13645b0930e4490f96ca49f458aeb641018/nv_ingest_api-26.1.1-py3-none-any.whl", hash = "sha256:e4f8b860765cedba72622782692e2ffc69a100fd61956e8b8a81a47e6c852d66", size = 357481, upload-time = "2026-01-13T23:44:07.943Z" }, + { url = "https://files.pythonhosted.org/packages/32/38/9ec1cb9d594f64b6eb88ad1da31c891c70444b742e7f5645b5ef4646e509/nv_ingest_api-25.9.0-py3-none-any.whl", hash = "sha256:f39fcafbcd2842be9fce9485e65afd301992bdb2d42f31d8c6736dd8ba6ba4fe", size = 296295, upload-time = "2025-09-16T21:06:08.344Z" }, ] [[package]] name = "nv-ingest-client" -version = "26.1.1" +version = "25.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "build" }, @@ -1782,16 +1727,17 @@ dependencies = [ { name = "click" }, { name = "fsspec" }, { name = "httpx" }, - { name = "lancedb" }, + { name = "llama-index-embeddings-nvidia" }, + { name = "nv-ingest-api" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "requests" }, { name = "setuptools" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cb/c4/ae5e2b00a8ffdfc1a3cf660ded68c188140ca433b22446adbb72ccfc455d/nv_ingest_client-26.1.1.tar.gz", hash = "sha256:26d6844eac946b4fdb8da2f5f1e77feb22b52ac21e6d772cf6b1c8c21cef4bb8", size = 126865, upload-time = "2026-01-13T23:44:15.615Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/6c/04037168ac68ec9f6168e5c50572e84238225ab58e4c31cf80530cc7ad82/nv_ingest_client-25.9.0.tar.gz", hash = "sha256:1e43f925b28843f9e47108e37b22c0a3c2a77af15be4ed4b5977a83af1f55a09", size = 99358, upload-time = "2025-09-16T21:06:15.419Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/64/ea/042ad6d8ddfa887667159af8b473835bfcc5b9f51ba6e82ff14078e59a3e/nv_ingest_client-26.1.1-py3-none-any.whl", hash = "sha256:df9906c7021e6a1ae64140fbe7a345679b1cfad7dfa486442e38ca75d64f2b39", size = 147197, upload-time = "2026-01-13T23:44:12.761Z" }, + { url = "https://files.pythonhosted.org/packages/89/96/548b87f4e14bc014fccdc87edc2fa8ae70af299a36084bdd23a9f1884452/nv_ingest_client-25.9.0-py3-none-any.whl", hash = "sha256:ab07fdff253d678986db1bc2b083b7c94243be5d327155dbbc19883f49675e41", size = 114937, upload-time = "2025-09-16T21:06:12.764Z" }, ] [[package]] @@ -1812,6 +1758,7 @@ dependencies = [ { name = "langchain-nvidia-ai-endpoints" }, { name = "lark" }, { name = "minio" }, + { name = "nv-ingest-client" }, { name = "oracledb" }, { name = "pdfplumber" }, { name = "protobuf" }, @@ -1931,10 +1878,11 @@ requires-dist = [ { name = "langchain-openai", marker = "extra == 'rag'", specifier = ">=0.2" }, { name = "lark", specifier = ">=1.2.2" }, { name = "minio", specifier = ">=7.2,<8.0" }, - { name = "nv-ingest-api", marker = "extra == 'all'", specifier = "==26.1.1" }, - { name = "nv-ingest-api", marker = "extra == 'ingest'", specifier = "==26.1.1" }, - { name = "nv-ingest-client", marker = "extra == 'all'", specifier = "==26.1.1" }, - { name = "nv-ingest-client", marker = "extra == 'ingest'", specifier = "==26.1.1" }, + { name = "nv-ingest-api", marker = "extra == 'all'", specifier = "==25.9.0" }, + { name = "nv-ingest-api", marker = "extra == 'ingest'", specifier = "==25.9.0" }, + { name = "nv-ingest-client", specifier = "==25.9.0" }, + { name = "nv-ingest-client", marker = "extra == 'all'", specifier = "==25.9.0" }, + { name = "nv-ingest-client", marker = "extra == 'ingest'", specifier = "==25.9.0" }, { name = "opentelemetry-api", marker = "extra == 'all'", specifier = ">=1.29,<2.0" }, { name = "opentelemetry-api", marker = "extra == 'ingest'", specifier = ">=1.29,<2.0" }, { name = "opentelemetry-api", marker = "extra == 'rag'", specifier = ">=1.29,<2.0" }, @@ -2472,15 +2420,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/17/e756653095a083d8a37cbd816cb87148debcfcd920129b25f99dd8d04271/pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc", size = 13199233, upload-time = "2025-09-29T23:24:24.876Z" }, ] -[[package]] -name = "pathlib-abc" -version = "0.5.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/cb/448649d7f25d228bf0be3a04590ab7afa77f15e056f8fa976ed05ec9a78f/pathlib_abc-0.5.2.tar.gz", hash = "sha256:fcd56f147234645e2c59c7ae22808b34c364bb231f685ddd9f96885aed78a94c", size = 33342, upload-time = "2025-10-10T18:37:20.524Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/29/c028a0731e202035f0e2e0bfbf1a3e46ad6c628cbb17f6f1cc9eea5d9ff1/pathlib_abc-0.5.2-py3-none-any.whl", hash = "sha256:4c9d94cf1b23af417ce7c0417b43333b06a106c01000b286c99de230d95eefbb", size = 19070, upload-time = "2025-10-10T18:37:19.437Z" }, -] - [[package]] name = "pdfminer-six" version = "20251230" @@ -2570,18 +2509,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/26/c56ce33ca856e358d27fda9676c055395abddb82c35ac0f593877ed4562e/pillow-12.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:cb9bb857b2d057c6dfc72ac5f3b44836924ba15721882ef103cecb40d002d80e", size = 7029880, upload-time = "2026-02-11T04:23:04.783Z" }, ] -[[package]] -name = "proglog" -version = "0.1.12" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c2/af/c108866c452eda1132f3d6b3cb6be2ae8430c97e9309f38ca9dbd430af37/proglog-0.1.12.tar.gz", hash = "sha256:361ee074721c277b89b75c061336cb8c5f287c92b043efa562ccf7866cda931c", size = 8794, upload-time = "2025-05-09T14:36:18.316Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/1b/f7ea6cde25621cd9236541c66ff018f4268012a534ec31032bcb187dc5e7/proglog-0.1.12-py3-none-any.whl", hash = "sha256:ccaafce51e80a81c65dc907a460c07ccb8ec1f78dc660cfd8f9ec3a22f01b84c", size = 6337, upload-time = "2025-05-09T14:36:16.798Z" }, -] - [[package]] name = "prometheus-client" version = "0.24.1" @@ -3595,6 +3522,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/e1/3ccb13c643399d22289c6a9786c1a91e3dcbb68bce4beb44926ac2c557bf/sqlalchemy-2.0.45-py3-none-any.whl", hash = "sha256:5225a288e4c8cc2308dbdd874edad6e7d0fd38eac1e9e5f23503425c8eee20d0", size = 1936672, upload-time = "2025-12-09T21:54:52.608Z" }, ] +[package.optional-dependencies] +asyncio = [ + { name = "greenlet" }, +] + [[package]] name = "starlette" version = "0.50.0" @@ -3638,11 +3570,11 @@ wheels = [ [[package]] name = "tenacity" -version = "9.1.2" +version = "8.5.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0a/d4/2b0cd0fe285e14b36db076e78c93766ff1d529d70408bd1d2a5a84f1d929/tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb", size = 48036, upload-time = "2025-04-02T08:25:09.966Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/4d/6a19536c50b849338fcbe9290d562b52cbdcf30d8963d3588a68a4107df1/tenacity-8.5.0.tar.gz", hash = "sha256:8bc6c0c8a09b31e6cad13c47afbed1a567518250a9a171418582ed8d9c20ca78", size = 47309, upload-time = "2024-07-05T07:25:31.836Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3f/8ba87d9e287b9d385a02a7114ddcef61b26f86411e121c9003eb509a1773/tenacity-8.5.0-py3-none-any.whl", hash = "sha256:b594c2a5945830c267ce6b79a166228323ed52718f30302c1359836112346687", size = 28165, upload-time = "2024-07-05T07:25:29.591Z" }, ] [[package]] @@ -3814,19 +3746,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1", size = 348521, upload-time = "2025-12-13T17:45:33.889Z" }, ] -[[package]] -name = "universal-pathlib" -version = "0.3.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "fsspec" }, - { name = "pathlib-abc" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6e/ec/764b0d4593c6a8f5f66b347a19b5db9486dd0024b5e3339d468064a90c76/universal_pathlib-0.3.8.tar.gz", hash = "sha256:ead2b65bca3df6e11c3b7cb36fc9846340bc3c2db4ef57131550260422b0a3e8", size = 258837, upload-time = "2026-01-11T22:13:53.328Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/2c/fc9416619a418e94576aef84ef263906a24f76a21a1c3e96ddae25c82df9/universal_pathlib-0.3.8-py3-none-any.whl", hash = "sha256:dac4fd9a3df918d85bb6da678e794b5dfa9ecdb5ff74675b497553dbe50134b8", size = 82608, upload-time = "2026-01-11T22:13:51.313Z" }, -] - [[package]] name = "urllib3" version = "2.6.3" From 48671f76984706f098ac6d1c8b3f72c58b876c60 Mon Sep 17 00:00:00 2001 From: anngu-2xx3 Date: Tue, 10 Mar 2026 13:13:20 +0700 Subject: [PATCH 36/81] Update: Remove Vss on AIDP notebook (#409) * Update: Remove Vss Signed-off-by: Minh Nguyen * Remove video processing (VSS) from kafka consumer Video handler, video analyzer service, and all VSS-related configuration have been removed to simplify the event ingestion pipeline to document-only processing. Signed-off-by: Minh Nguyen Made-with: Cursor Signed-off-by: Minh Nguyen * fix: add USERID Signed-off-by: Minh Nguyen * fix: update document Signed-off-by: Minh Nguyen --------- Signed-off-by: Minh Nguyen Co-authored-by: Minh Nguyen --- examples/README.md | 8 +- .../deploy/docker-compose.yaml | 13 - .../kafka_consumer/config/__init__.py | 55 +- .../kafka_consumer/config/constants.py | 114 - .../kafka_consumer/config/settings.py | 64 - .../kafka_consumer/consumer.py | 7 - .../kafka_consumer/handlers/__init__.py | 3 +- .../kafka_consumer/handlers/video.py | 118 - .../rag_event_ingest/kafka_consumer/main.py | 10 +- .../rag_event_ingest/kafka_consumer/router.py | 15 +- .../kafka_consumer/services/__init__.py | 3 +- .../services/document_indexer.py | 71 - .../kafka_consumer/services/video_analyzer.py | 326 --- notebooks/rag_event_ingest.ipynb | 2050 +++++++---------- 14 files changed, 803 insertions(+), 2054 deletions(-) delete mode 100644 examples/rag_event_ingest/kafka_consumer/handlers/video.py delete mode 100644 examples/rag_event_ingest/kafka_consumer/services/video_analyzer.py diff --git a/examples/README.md b/examples/README.md index 2e8d54c1c..ba86d6765 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,7 +8,7 @@ This directory contains example integrations and extensions for NVIDIA RAG. |---------|-------------|---------------| | [rag_react_agent](./rag_react_agent/) | Integration with [NeMo Agent Toolkit (NAT)](https://github.com/NVIDIA/NeMo-Agent-Toolkit) providing RAG query and search capabilities for agent workflows | [README](./rag_react_agent/README.md) | | [nvidia_rag_mcp](./nvidia_rag_mcp/) | MCP (Model Context Protocol) server and client for exposing NVIDIA RAG capabilities to MCP-compatible applications | [Documentation](../docs/mcp.md) | -| [rag_event_ingest](./rag_event_ingest/) | Automated document & video ingestion from object storage (MinIO) via Kafka, with VSS video analysis | [Notebook](../notebooks/rag_event_ingest.ipynb) | +| [rag_event_ingest](./rag_event_ingest/) | Automated document ingestion from object storage (MinIO) via Kafka | [Notebook](../notebooks/rag_event_ingest.ipynb) | ## rag_react_agent @@ -31,11 +31,11 @@ See the [MCP documentation](../docs/mcp.md) for detailed setup and usage instruc ## rag_event_ingest -This example deploys an event-driven ingestion pipeline that monitors MinIO object storage for new file uploads via Kafka events. Documents are indexed through the RAG Ingestor, while videos are analyzed by NVIDIA VSS (Video Search & Summarization). All content becomes queryable through the RAG Agent. +This example deploys an event-driven ingestion pipeline that monitors MinIO object storage for new file uploads via Kafka events. Documents are automatically indexed through the RAG Ingestor and become queryable through the RAG Agent. Components: -- **kafka_consumer/** - Event-driven consumer that routes files to RAG or VSS based on file type +- **kafka_consumer/** - Event-driven consumer that routes files to RAG based on file type - **deploy/** - Docker Compose for Kafka, MinIO, and the consumer -- **data/** - Sample documents and videos for testing +- **data/** - Sample documents for testing See the [notebook](../notebooks/rag_event_ingest.ipynb) for step-by-step deployment and testing. diff --git a/examples/rag_event_ingest/deploy/docker-compose.yaml b/examples/rag_event_ingest/deploy/docker-compose.yaml index 774e5c6cd..05e5bbc0f 100644 --- a/examples/rag_event_ingest/deploy/docker-compose.yaml +++ b/examples/rag_event_ingest/deploy/docker-compose.yaml @@ -137,27 +137,14 @@ services: - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY:-minioadmin} - MINIO_SECRET_KEY=${MINIO_SECRET_KEY:-minioadmin} - MINIO_SECURE=false - - VSS_SERVER_URL=${VSS_SERVER_URL:-http://host.docker.internal:8110} - - VSS_TIMEOUT=${VSS_TIMEOUT:-1800} - - VSS_CHUNK_DURATION=${VSS_CHUNK_DURATION:-30} - - VSS_CHUNK_OVERLAP=${VSS_CHUNK_OVERLAP:-10} - - VSS_NUM_FRAMES_PER_CHUNK=${VSS_NUM_FRAMES_PER_CHUNK:-16} - - VSS_MAX_TOKENS=${VSS_MAX_TOKENS:-8192} # RAG Ingestor - INGESTOR_SERVER_URL=${INGESTOR_SERVER_URL:-http://ingestor-server:8082} - COLLECTION_NAME=${COLLECTION_NAME:-aidp_bucket} - # Video Analysis Prompts (customizable via environment variables) - - "VSS_PROMPT=${VSS_PROMPT:-Analyze this sports video. TIMESTAMP FORMAT: Convert seconds to MM:SS (40s=00:40, 80s=01:20, 150s=02:30, 200s=03:20). Describe: key plays, scoring, turnovers, big gains, defensive stops, celebrations.}" - - "VSS_SYSTEM_PROMPT=${VSS_SYSTEM_PROMPT:-You are a sports broadcaster. CRITICAL: Convert all timestamps from seconds to MM:SS format. Examples: 40 seconds = 00:40, 90 seconds = 01:30, 200 seconds = 03:20, 350 seconds = 05:50. Focus on action, field position, execution, and game momentum.}" - - "VSS_CAPTION_SUMMARIZATION_PROMPT=${VSS_CAPTION_SUMMARIZATION_PROMPT:-Format: [MM:SS] Play description. CONVERT seconds to MM:SS (40s=00:40, 80s=01:20, 150s=02:30). Describe: what happened, how it was executed, the result. Be vivid like a TV broadcast.}" - - "VSS_SUMMARY_AGGREGATION_PROMPT=${VSS_SUMMARY_AGGREGATION_PROMPT:-Create game summary with MM:SS timestamps. CONVERT all times: 40s=00:40, 80s=01:20, 200s=03:20, 350s=05:50. Highlight scoring plays, turnovers, momentum shifts, spectacular plays. Write like a highlight reel narrator - vivid and exciting.}" # Logging - LOG_LEVEL=${LOG_LEVEL:-INFO} restart: unless-stopped networks: - nvidia-rag - extra_hosts: - - "host.docker.internal:host-gateway" # ============================================================================= # VOLUMES diff --git a/examples/rag_event_ingest/kafka_consumer/config/__init__.py b/examples/rag_event_ingest/kafka_consumer/config/__init__.py index 3e38cf59c..6bd140441 100644 --- a/examples/rag_event_ingest/kafka_consumer/config/__init__.py +++ b/examples/rag_event_ingest/kafka_consumer/config/__init__.py @@ -3,9 +3,9 @@ Usage: import config.settings as cfg - print(cfg.VSS_SERVER_URL) + print(cfg.INGESTOR_SERVER_URL) - from config.constants import VIDEO_EXTENSIONS, DEST_VSS + from config.constants import DOCUMENT_EXTENSIONS, DEST_RAG """ # Settings (env vars) @@ -21,11 +21,7 @@ KAFKA_HEARTBEAT_INTERVAL_MS, # Services INGESTOR_SERVER_URL, - VSS_SERVER_URL, INGESTOR_TIMEOUT, - VSS_TIMEOUT, - VSS_UPLOAD_TIMEOUT, - VIDEO_INDEX_TIMEOUT, # MinIO MINIO_ENDPOINT, MINIO_ACCESS_KEY, @@ -34,10 +30,8 @@ MINIO_DEFAULT_COLLECTION, MINIO_SOURCES, # Features - ENABLE_MULTIMODAL_RAG, ENABLE_IMAGE_PROCESSING, ENABLE_AUDIO_PROCESSING, - VSS_EMBEDDING_ENABLED, # Collection EMBEDDING_DIMENSION, CHUNK_SIZE, @@ -47,31 +41,16 @@ LOG_FORMAT, # History HISTORY_FILE, - # VSS Prompts (configurable via env) - VSS_DEFAULT_PROMPT, - VSS_SYSTEM_PROMPT, - VSS_CAPTION_SUMMARIZATION_PROMPT, - VSS_SUMMARY_AGGREGATION_PROMPT, - # VSS Settings - VSS_CHUNK_DURATION, - VSS_CHUNK_OVERLAP, - VSS_NUM_FRAMES_PER_CHUNK, - VSS_MAX_TOKENS, - VSS_MODEL, - VSS_STREAM_ENABLED, # API Endpoints (configurable via env) API_INGESTOR_DOCUMENTS, API_INGESTOR_COLLECTIONS, API_INGESTOR_COLLECTION, API_INGESTOR_STATUS, - API_VSS_FILES, - API_VSS_SUMMARIZE, ) # Constants from .constants import ( # File extensions - VIDEO_EXTENSIONS, DOCUMENT_EXTENSIONS, IMAGE_EXTENSIONS, AUDIO_EXTENSIONS, @@ -80,7 +59,6 @@ CONTENT_TYPE_MAP, DEFAULT_CONTENT_TYPE, # Routing - DEST_VSS, DEST_RAG, DEST_SKIP, DEST_UNKNOWN, @@ -143,30 +121,7 @@ FIELD_GENERATE_SUMMARY, FIELD_EMBEDDING_DIMENSION, FIELD_TASK_ID, - # API request fields (VSS) - FIELD_ID, - FIELD_FILE, - FIELD_MODEL, - FIELD_PROMPT, - FIELD_SYSTEM_PROMPT, - FIELD_MAX_TOKENS, - FIELD_CHUNK_DURATION, - FIELD_CHUNK_OVERLAP_DURATION, - FIELD_NUM_FRAMES_PER_CHUNK, - FIELD_CAPTION_SUMMARIZATION_PROMPT, - FIELD_SUMMARY_AGGREGATION_PROMPT, - FIELD_PURPOSE, - FIELD_MEDIA_TYPE, - VALUE_VISION, - VALUE_VIDEO, - FIELD_STREAM, - RESP_DELTA, - RESP_DATA_PREFIX, # API response fields - RESP_CONTENT, - RESP_RESPONSE, - RESP_TEXT, - RESP_CHOICES, RESP_MESSAGE, RESP_ERROR, RESP_COLLECTIONS, @@ -178,12 +133,8 @@ # Timeouts TIMEOUT_DEFAULT, TIMEOUT_UPLOAD, - TIMEOUT_VIDEO_UPLOAD, TIMEOUT_TASK_CHECK, TIMEOUT_MAX_TASK_WAIT, - VIDEO_ANALYSIS_POLL_INTERVAL, - VIDEO_ANALYSIS_MAX_WAIT, - VIDEO_DESCRIPTION_MIN_LENGTH, # Kafka defaults KAFKA_DEFAULT_TOPIC, KAFKA_DEFAULT_CONSUMER_GROUP, @@ -196,6 +147,4 @@ COLLECTION_EMBEDDING_DIMENSION, COLLECTION_CHUNK_SIZE, COLLECTION_CHUNK_OVERLAP, - COLLECTION_VIDEO_CHUNK_SIZE, - COLLECTION_VIDEO_CHUNK_OVERLAP, ) diff --git a/examples/rag_event_ingest/kafka_consumer/config/constants.py b/examples/rag_event_ingest/kafka_consumer/config/constants.py index a90eee342..050cf70c4 100644 --- a/examples/rag_event_ingest/kafka_consumer/config/constants.py +++ b/examples/rag_event_ingest/kafka_consumer/config/constants.py @@ -6,11 +6,6 @@ # ==================== File Extensions ==================== -VIDEO_EXTENSIONS = frozenset({ - '.mp4', '.mkv', '.avi', '.mov', '.wmv', - '.flv', '.webm', '.m4v', '.mpeg', '.mpg' -}) - DOCUMENT_EXTENSIONS = frozenset({ '.pdf', '.docx', '.doc', '.txt', '.md', '.rst', '.html', '.htm', '.pptx', '.ppt', '.xlsx', '.xls', @@ -61,17 +56,6 @@ '.bmp': 'image/bmp', '.tiff': 'image/tiff', '.svg': 'image/svg+xml', - # Videos - '.mp4': 'video/mp4', - '.mkv': 'video/x-matroska', - '.avi': 'video/x-msvideo', - '.mov': 'video/quicktime', - '.wmv': 'video/x-ms-wmv', - '.flv': 'video/x-flv', - '.webm': 'video/webm', - '.m4v': 'video/x-m4v', - '.mpeg': 'video/mpeg', - '.mpg': 'video/mpeg', # Audio '.mp3': 'audio/mpeg', '.wav': 'audio/wav', @@ -88,7 +72,6 @@ # ==================== Routing ==================== # Destinations -DEST_VSS = 'vss' DEST_RAG = 'rag' DEST_SKIP = 'skip' DEST_UNKNOWN = 'unknown' @@ -100,7 +83,6 @@ KEY_REASON = 'reason' # File types -FILE_TYPE_VIDEO = 'video' FILE_TYPE_DOCUMENT = 'document' FILE_TYPE_IMAGE = 'image' FILE_TYPE_AUDIO = 'audio' @@ -108,7 +90,6 @@ FILE_TYPE_UNKNOWN = 'unknown' # Config keys -CFG_VIDEO_EXTENSIONS = 'video_extensions' CFG_DOCUMENT_EXTENSIONS = 'document_extensions' CFG_IMAGE_EXTENSIONS = 'image_extensions' CFG_AUDIO_EXTENSIONS = 'audio_extensions' @@ -199,30 +180,6 @@ FIELD_EMBEDDING_DIMENSION = 'embedding_dimension' FIELD_TASK_ID = 'task_id' -# VSS request fields -FIELD_ID = 'id' -FIELD_FILE = 'file' -FIELD_MODEL = 'model' -FIELD_PROMPT = 'prompt' -FIELD_SYSTEM_PROMPT = 'system_prompt' -FIELD_MAX_TOKENS = 'max_tokens' -FIELD_CHUNK_DURATION = 'chunk_duration' -FIELD_CHUNK_OVERLAP_DURATION = 'chunk_overlap_duration' -FIELD_NUM_FRAMES_PER_CHUNK = 'num_frames_per_chunk' -FIELD_CAPTION_SUMMARIZATION_PROMPT = 'caption_summarization_prompt' -FIELD_SUMMARY_AGGREGATION_PROMPT = 'summary_aggregation_prompt' -FIELD_PURPOSE = 'purpose' -FIELD_MEDIA_TYPE = 'media_type' - -# VSS request values -VALUE_VISION = 'vision' -VALUE_VIDEO = 'video' - -# VSS streaming fields -FIELD_STREAM = 'stream' -RESP_DELTA = 'delta' -RESP_DATA_PREFIX = 'data: ' - # ==================== API Response Fields ==================== @@ -247,16 +204,9 @@ TIMEOUT_DEFAULT = 30 TIMEOUT_UPLOAD = 600 -TIMEOUT_VIDEO_UPLOAD = 600 -TIMEOUT_VIDEO_INDEX = 300 TIMEOUT_TASK_CHECK = 30 TIMEOUT_MAX_TASK_WAIT = 300 -# Video analysis polling -VIDEO_ANALYSIS_POLL_INTERVAL = 10 # seconds between polls -VIDEO_ANALYSIS_MAX_WAIT = 180 # max seconds to wait for analysis -VIDEO_DESCRIPTION_MIN_LENGTH = 200 # minimum chars for valid description - # ==================== Kafka Defaults ==================== @@ -274,8 +224,6 @@ COLLECTION_EMBEDDING_DIMENSION = 2048 COLLECTION_CHUNK_SIZE = 512 COLLECTION_CHUNK_OVERLAP = 150 -COLLECTION_VIDEO_CHUNK_SIZE = 1024 -COLLECTION_VIDEO_CHUNK_OVERLAP = 100 # ==================== Environment Variable Keys ==================== @@ -292,19 +240,13 @@ # Service URLs ENV_INGESTOR_SERVER_URL = 'INGESTOR_SERVER_URL' -ENV_VSS_SERVER_URL = 'VSS_SERVER_URL' ENV_INGESTOR_TIMEOUT = 'INGESTOR_TIMEOUT' -ENV_VSS_TIMEOUT = 'VSS_TIMEOUT' -ENV_VSS_UPLOAD_TIMEOUT = 'VSS_UPLOAD_TIMEOUT' -ENV_VIDEO_INDEX_TIMEOUT = 'VIDEO_INDEX_TIMEOUT' # API Endpoints ENV_API_INGESTOR_DOCUMENTS = 'API_INGESTOR_DOCUMENTS' ENV_API_INGESTOR_COLLECTIONS = 'API_INGESTOR_COLLECTIONS' ENV_API_INGESTOR_COLLECTION = 'API_INGESTOR_COLLECTION' ENV_API_INGESTOR_STATUS = 'API_INGESTOR_STATUS' -ENV_API_VSS_FILES = 'API_VSS_FILES' -ENV_API_VSS_SUMMARIZE = 'API_VSS_SUMMARIZE' # MinIO ENV_MINIO_ENDPOINT = 'MINIO_ENDPOINT' @@ -315,10 +257,8 @@ ENV_MINIO_SOURCES = 'MINIO_SOURCES' # Feature Flags -ENV_ENABLE_MULTIMODAL_RAG = 'ENABLE_MULTIMODAL_RAG' ENV_ENABLE_IMAGE_PROCESSING = 'ENABLE_IMAGE_PROCESSING' ENV_ENABLE_AUDIO_PROCESSING = 'ENABLE_AUDIO_PROCESSING' -ENV_VSS_EMBEDDING = 'VSS_EMBEDDING' # Collection Settings ENV_EMBEDDING_DIMENSION = 'EMBEDDING_DIMENSION' @@ -332,27 +272,12 @@ # History ENV_HISTORY_FILE = 'HISTORY_FILE' -# VSS Settings -ENV_VSS_PROMPT = 'VSS_PROMPT' -ENV_VSS_SYSTEM_PROMPT = 'VSS_SYSTEM_PROMPT' -ENV_VSS_CAPTION_SUMMARIZATION_PROMPT = 'VSS_CAPTION_SUMMARIZATION_PROMPT' -ENV_VSS_SUMMARY_AGGREGATION_PROMPT = 'VSS_SUMMARY_AGGREGATION_PROMPT' -ENV_VSS_CHUNK_DURATION = 'VSS_CHUNK_DURATION' -ENV_VSS_CHUNK_OVERLAP = 'VSS_CHUNK_OVERLAP' -ENV_VSS_NUM_FRAMES_PER_CHUNK = 'VSS_NUM_FRAMES_PER_CHUNK' -ENV_VSS_MAX_TOKENS = 'VSS_MAX_TOKENS' -ENV_VSS_MODEL = 'VSS_MODEL' -ENV_VSS_STREAM_ENABLED = 'VSS_STREAM_ENABLED' - - # ==================== API Endpoint Defaults ==================== DEFAULT_API_INGESTOR_DOCUMENTS = '/v1/documents' DEFAULT_API_INGESTOR_COLLECTIONS = '/v1/collections' DEFAULT_API_INGESTOR_COLLECTION = '/v1/collection' DEFAULT_API_INGESTOR_STATUS = '/v1/status' -DEFAULT_API_VSS_FILES = '/files' -DEFAULT_API_VSS_SUMMARIZE = '/summarize' # ==================== Logging Defaults ==================== @@ -369,42 +294,3 @@ # ==================== MinIO Defaults ==================== DEFAULT_COLLECTION_NAME = 'multimodal_data' - - -# ==================== VSS Defaults ==================== - -DEFAULT_VSS_MODEL = 'Cosmos-Reason2-8B' -DEFAULT_VSS_CHUNK_DURATION = 60 # seconds per chunk -DEFAULT_VSS_CHUNK_OVERLAP = 5 # overlap between chunks -DEFAULT_VSS_NUM_FRAMES_PER_CHUNK = 8 # frames per chunk for VLM -DEFAULT_VSS_MAX_TOKENS = 1024 - -# Default prompts for general videos -DEFAULT_VSS_PROMPT = ( - "Analyze this video. TIMESTAMP FORMAT: Convert seconds to MM:SS " - "(40s=00:40, 80s=01:20, 150s=02:30, 200s=03:20). " - "Describe: key events, important moments, actions, transitions, and notable scenes. " - "Capture the flow and progression of the content." -) - -DEFAULT_VSS_SYSTEM_PROMPT = ( - "You are a video analyst providing detailed descriptions. " - "CRITICAL: Convert all timestamps from seconds to MM:SS format. " - "Examples: 40 seconds = 00:40, 90 seconds = 01:30, 200 seconds = 03:20. " - "Focus on what you clearly see: actions, scenes, transitions, and key moments. " - "Be descriptive and engaging." -) - -DEFAULT_VSS_CAPTION_SUMMARIZATION_PROMPT = ( - "Format: [MM:SS] Description. CONVERT seconds to MM:SS " - "(40s=00:40, 80s=01:20, 150s=02:30). " - "Describe: what happened, how it occurred, the result. " - "Be specific about the action and context." -) - -DEFAULT_VSS_SUMMARY_AGGREGATION_PROMPT = ( - "Create a comprehensive video summary with MM:SS timestamps. " - "CONVERT all times: 40s=00:40, 80s=01:20, 200s=03:20, 350s=05:50. " - "Highlight: key events, transitions, important moments, and notable scenes. " - "Write in an engaging narrative style." -) diff --git a/examples/rag_event_ingest/kafka_consumer/config/settings.py b/examples/rag_event_ingest/kafka_consumer/config/settings.py index 69794c2a0..bbfe4824c 100644 --- a/examples/rag_event_ingest/kafka_consumer/config/settings.py +++ b/examples/rag_event_ingest/kafka_consumer/config/settings.py @@ -13,8 +13,6 @@ KAFKA_DEFAULT_SESSION_TIMEOUT_MS, KAFKA_DEFAULT_HEARTBEAT_INTERVAL_MS, TIMEOUT_UPLOAD, - TIMEOUT_VIDEO_UPLOAD, - TIMEOUT_VIDEO_INDEX, COLLECTION_EMBEDDING_DIMENSION, COLLECTION_CHUNK_SIZE, COLLECTION_CHUNK_OVERLAP, @@ -23,8 +21,6 @@ DEFAULT_API_INGESTOR_COLLECTIONS, DEFAULT_API_INGESTOR_COLLECTION, DEFAULT_API_INGESTOR_STATUS, - DEFAULT_API_VSS_FILES, - DEFAULT_API_VSS_SUMMARIZE, # Logging defaults DEFAULT_LOG_LEVEL, DEFAULT_LOG_FORMAT, @@ -32,16 +28,6 @@ DEFAULT_HISTORY_FILE, # MinIO defaults DEFAULT_COLLECTION_NAME, - # VSS defaults - DEFAULT_VSS_MODEL, - DEFAULT_VSS_CHUNK_DURATION, - DEFAULT_VSS_CHUNK_OVERLAP, - DEFAULT_VSS_NUM_FRAMES_PER_CHUNK, - DEFAULT_VSS_MAX_TOKENS, - DEFAULT_VSS_PROMPT, - DEFAULT_VSS_SYSTEM_PROMPT, - DEFAULT_VSS_CAPTION_SUMMARIZATION_PROMPT, - DEFAULT_VSS_SUMMARY_AGGREGATION_PROMPT, # Environment variable keys ENV_KAFKA_BOOTSTRAP_SERVERS, ENV_KAFKA_TOPIC, @@ -52,43 +38,25 @@ ENV_KAFKA_SESSION_TIMEOUT_MS, ENV_KAFKA_HEARTBEAT_INTERVAL_MS, ENV_INGESTOR_SERVER_URL, - ENV_VSS_SERVER_URL, ENV_INGESTOR_TIMEOUT, - ENV_VSS_TIMEOUT, - ENV_VSS_UPLOAD_TIMEOUT, - ENV_VIDEO_INDEX_TIMEOUT, ENV_API_INGESTOR_DOCUMENTS, ENV_API_INGESTOR_COLLECTIONS, ENV_API_INGESTOR_COLLECTION, ENV_API_INGESTOR_STATUS, - ENV_API_VSS_FILES, - ENV_API_VSS_SUMMARIZE, ENV_MINIO_ENDPOINT, ENV_MINIO_ACCESS_KEY, ENV_MINIO_SECRET_KEY, ENV_MINIO_SECURE, ENV_COLLECTION_NAME, ENV_MINIO_SOURCES, - ENV_ENABLE_MULTIMODAL_RAG, ENV_ENABLE_IMAGE_PROCESSING, ENV_ENABLE_AUDIO_PROCESSING, - ENV_VSS_EMBEDDING, ENV_EMBEDDING_DIMENSION, ENV_CHUNK_SIZE, ENV_CHUNK_OVERLAP, ENV_LOG_LEVEL, ENV_LOG_FORMAT, ENV_HISTORY_FILE, - ENV_VSS_PROMPT, - ENV_VSS_SYSTEM_PROMPT, - ENV_VSS_CAPTION_SUMMARIZATION_PROMPT, - ENV_VSS_SUMMARY_AGGREGATION_PROMPT, - ENV_VSS_CHUNK_DURATION, - ENV_VSS_CHUNK_OVERLAP, - ENV_VSS_NUM_FRAMES_PER_CHUNK, - ENV_VSS_MAX_TOKENS, - ENV_VSS_MODEL, - ENV_VSS_STREAM_ENABLED, ) @@ -122,11 +90,7 @@ def _get_int(key: str, default: int) -> int: # ==================== Service URLs ==================== INGESTOR_SERVER_URL = os.getenv(ENV_INGESTOR_SERVER_URL) # Required -VSS_SERVER_URL = os.getenv(ENV_VSS_SERVER_URL) # Required INGESTOR_TIMEOUT = _get_int(ENV_INGESTOR_TIMEOUT, TIMEOUT_UPLOAD) -VSS_TIMEOUT = _get_int(ENV_VSS_TIMEOUT, TIMEOUT_VIDEO_UPLOAD) -VSS_UPLOAD_TIMEOUT = _get_int(ENV_VSS_UPLOAD_TIMEOUT, TIMEOUT_VIDEO_UPLOAD) -VIDEO_INDEX_TIMEOUT = _get_int(ENV_VIDEO_INDEX_TIMEOUT, TIMEOUT_VIDEO_INDEX) # API Endpoints - Ingestor Server API_INGESTOR_DOCUMENTS = os.getenv(ENV_API_INGESTOR_DOCUMENTS, DEFAULT_API_INGESTOR_DOCUMENTS) @@ -134,10 +98,6 @@ def _get_int(key: str, default: int) -> int: API_INGESTOR_COLLECTION = os.getenv(ENV_API_INGESTOR_COLLECTION, DEFAULT_API_INGESTOR_COLLECTION) API_INGESTOR_STATUS = os.getenv(ENV_API_INGESTOR_STATUS, DEFAULT_API_INGESTOR_STATUS) -# API Endpoints - VSS -API_VSS_FILES = os.getenv(ENV_API_VSS_FILES, DEFAULT_API_VSS_FILES) -API_VSS_SUMMARIZE = os.getenv(ENV_API_VSS_SUMMARIZE, DEFAULT_API_VSS_SUMMARIZE) - # ==================== MinIO Settings ==================== @@ -152,10 +112,8 @@ def _get_int(key: str, default: int) -> int: # ==================== Feature Flags ==================== -ENABLE_MULTIMODAL_RAG = _get_bool(ENV_ENABLE_MULTIMODAL_RAG, True) ENABLE_IMAGE_PROCESSING = _get_bool(ENV_ENABLE_IMAGE_PROCESSING, False) ENABLE_AUDIO_PROCESSING = _get_bool(ENV_ENABLE_AUDIO_PROCESSING, False) -VSS_EMBEDDING_ENABLED = _get_bool(ENV_VSS_EMBEDDING, False) # ==================== Collection Settings ==================== @@ -175,25 +133,3 @@ def _get_int(key: str, default: int) -> int: HISTORY_FILE = os.getenv(ENV_HISTORY_FILE, DEFAULT_HISTORY_FILE) - -# ==================== VSS Settings ==================== - -# Default prompts for general videos (can be overridden via environment variables) -# Use environment variables for specific video types (sports, talks, tutorials, etc.) -VSS_DEFAULT_PROMPT = os.getenv(ENV_VSS_PROMPT, DEFAULT_VSS_PROMPT) -VSS_SYSTEM_PROMPT = os.getenv(ENV_VSS_SYSTEM_PROMPT, DEFAULT_VSS_SYSTEM_PROMPT) -VSS_CAPTION_SUMMARIZATION_PROMPT = os.getenv( - ENV_VSS_CAPTION_SUMMARIZATION_PROMPT, DEFAULT_VSS_CAPTION_SUMMARIZATION_PROMPT -) -VSS_SUMMARY_AGGREGATION_PROMPT = os.getenv( - ENV_VSS_SUMMARY_AGGREGATION_PROMPT, DEFAULT_VSS_SUMMARY_AGGREGATION_PROMPT -) - -# VSS chunking settings for long videos -VSS_CHUNK_DURATION = _get_int(ENV_VSS_CHUNK_DURATION, DEFAULT_VSS_CHUNK_DURATION) -VSS_CHUNK_OVERLAP = _get_int(ENV_VSS_CHUNK_OVERLAP, DEFAULT_VSS_CHUNK_OVERLAP) -VSS_NUM_FRAMES_PER_CHUNK = _get_int(ENV_VSS_NUM_FRAMES_PER_CHUNK, DEFAULT_VSS_NUM_FRAMES_PER_CHUNK) -VSS_MAX_TOKENS = _get_int(ENV_VSS_MAX_TOKENS, DEFAULT_VSS_MAX_TOKENS) -VSS_MODEL = os.getenv(ENV_VSS_MODEL, DEFAULT_VSS_MODEL) -VSS_STREAM_ENABLED = _get_bool(ENV_VSS_STREAM_ENABLED, True) - diff --git a/examples/rag_event_ingest/kafka_consumer/consumer.py b/examples/rag_event_ingest/kafka_consumer/consumer.py index b78d3c322..87a5c0538 100644 --- a/examples/rag_event_ingest/kafka_consumer/consumer.py +++ b/examples/rag_event_ingest/kafka_consumer/consumer.py @@ -104,13 +104,6 @@ def _handle_delete(self, event: S3Event) -> HandlerResult: indexer = doc_handler.indexer success = indexer.delete(event.key, event.collection) - if self.router.is_video(event.key): - desc_filename = f"{event.key}_description.json" - desc_ok = indexer.delete(desc_filename, event.collection) - if desc_ok: - logger.info(f"✓ Deleted video description {desc_filename} from Milvus") - success = success or desc_ok - if success: logger.info(f"✓ Deleted {event.key} from Milvus") return HandlerResult(success=True, status='DELETED') diff --git a/examples/rag_event_ingest/kafka_consumer/handlers/__init__.py b/examples/rag_event_ingest/kafka_consumer/handlers/__init__.py index 112493e8e..e6f3efcff 100644 --- a/examples/rag_event_ingest/kafka_consumer/handlers/__init__.py +++ b/examples/rag_event_ingest/kafka_consumer/handlers/__init__.py @@ -1,6 +1,5 @@ # Handlers package from .base import BaseHandler from .document import DocumentHandler -from .video import VideoHandler -__all__ = ['BaseHandler', 'DocumentHandler', 'VideoHandler'] +__all__ = ['BaseHandler', 'DocumentHandler'] diff --git a/examples/rag_event_ingest/kafka_consumer/handlers/video.py b/examples/rag_event_ingest/kafka_consumer/handlers/video.py deleted file mode 100644 index 9e5d73a34..000000000 --- a/examples/rag_event_ingest/kafka_consumer/handlers/video.py +++ /dev/null @@ -1,118 +0,0 @@ -# handlers/video.py -"""Handler for video files.""" - -import logging - -import requests - -from .base import BaseHandler -from models.events import S3Event, HandlerResult -from services.storage import ObjectStorage -from services.video_analyzer import VideoAnalyzer -from services.document_indexer import DocumentIndexer - -logger = logging.getLogger(__name__) - - -class VideoHandler(BaseHandler): - """Handler for video files - analyzes and indexes in vector store.""" - - def __init__( - self, - storage: ObjectStorage, - analyzer: VideoAnalyzer, - indexer: DocumentIndexer, - enable_multimodal_rag: bool = True - ): - """Initialize video handler. - - Args: - storage: Object storage for file downloads - analyzer: Video analyzer for VLM analysis - indexer: Document indexer for storing descriptions - enable_multimodal_rag: Whether to index video descriptions - """ - self.storage = storage - self.analyzer = analyzer - self.indexer = indexer - self.enable_multimodal_rag = enable_multimodal_rag - - @property - def name(self) -> str: - return "VideoHandler" - - def handle(self, event: S3Event) -> HandlerResult: - """Process video file. - - 1. Download from MinIO - 2. Upload to VSS - 3. (Optional) Get description and index in Milvus - - Args: - event: S3 event with video info - - Returns: - HandlerResult - """ - self.log_start(event) - - try: - # Step 1: Download from storage - logger.info(f"📥 Downloading video from storage...") - video_data = self.storage.download(event.bucket, event.key) - - # Step 2: Upload to VSS - logger.info(f"📤 Uploading to analyzer...") - success, video_name = self.analyzer.upload_video(video_data, event.key) - - if not success: - result = HandlerResult.failed_result("Video upload failed") - self.log_failure(event, result) - return result - - logger.info(f"✓ Video uploaded: {video_name}") - - # Step 3: Multi-modal RAG indexing (optional) - if self.enable_multimodal_rag and video_name: - self._index_video_description(event, video_name) - - result = HandlerResult.success_result() - self.log_success(event, result) - return result - - except requests.RequestException as e: - logger.error(f"Network error processing video: {e}") - return HandlerResult.failed_result(str(e)) - except (IOError, OSError) as e: - logger.error(f"Storage error processing video: {e}") - return HandlerResult.failed_result(str(e)) - - def _index_video_description(self, event: S3Event, video_name: str): - """Get video description from VSS and index in Milvus. - - Args: - event: S3 event - video_name: Video name (stem of filename) - """ - logger.info(f"🔄 Starting Multi-Modal RAG indexing...") - - # Get description using VSS /generate with detailed prompt - logger.info(f"📹 Generating description for video: {video_name}...") - description = self.analyzer.get_video_description(video_name) - - if not description: - logger.warning("Failed to get video description from VSS") - return - - # Index description in Milvus - success = self.indexer.index_video_description( - collection=event.collection, - video_id=video_name, - video_name=event.key, - description=description - ) - - if success: - logger.info(f"✓ Indexed video description for {event.key}") - else: - logger.warning(f"⚠ Failed to index video description") diff --git a/examples/rag_event_ingest/kafka_consumer/main.py b/examples/rag_event_ingest/kafka_consumer/main.py index 02e86c137..df4384d4f 100644 --- a/examples/rag_event_ingest/kafka_consumer/main.py +++ b/examples/rag_event_ingest/kafka_consumer/main.py @@ -5,9 +5,9 @@ import logging import config.settings as cfg -from config.constants import DEST_RAG, DEST_VSS -from services import ObjectStorage, DocumentIndexer, VideoAnalyzer -from handlers import DocumentHandler, VideoHandler +from config.constants import DEST_RAG +from services import ObjectStorage, DocumentIndexer +from handlers import DocumentHandler from consumer import KafkaEventConsumer logging.basicConfig( @@ -27,13 +27,11 @@ def main(): logger.info("Initializing services...") storage = ObjectStorage() indexer = DocumentIndexer(cfg.INGESTOR_SERVER_URL) - analyzer = VideoAnalyzer(cfg.VSS_SERVER_URL) - + # Initialize handlers logger.info("Initializing handlers...") handlers = { DEST_RAG: DocumentHandler(storage, indexer), - DEST_VSS: VideoHandler(storage, analyzer, indexer, enable_multimodal_rag=cfg.ENABLE_MULTIMODAL_RAG), } # Initialize consumer diff --git a/examples/rag_event_ingest/kafka_consumer/router.py b/examples/rag_event_ingest/kafka_consumer/router.py index df9432357..41f5b8f23 100644 --- a/examples/rag_event_ingest/kafka_consumer/router.py +++ b/examples/rag_event_ingest/kafka_consumer/router.py @@ -6,25 +6,21 @@ from typing import Dict, Any, List, Set, Union from config.constants import ( - VIDEO_EXTENSIONS, DOCUMENT_EXTENSIONS, IMAGE_EXTENSIONS, AUDIO_EXTENSIONS, SKIP_EXTENSIONS, - DEST_VSS, DEST_RAG, DEST_SKIP, KEY_DESTINATION, KEY_FILE_TYPE, KEY_EXTENSION, KEY_REASON, - FILE_TYPE_VIDEO, FILE_TYPE_DOCUMENT, FILE_TYPE_IMAGE, FILE_TYPE_AUDIO, FILE_TYPE_SKIP, FILE_TYPE_UNKNOWN, - CFG_VIDEO_EXTENSIONS, CFG_DOCUMENT_EXTENSIONS, CFG_IMAGE_EXTENSIONS, CFG_AUDIO_EXTENSIONS, @@ -45,7 +41,6 @@ def __init__(self, config: Union[Dict[str, Any], Any] = None): config = {} elif hasattr(config, '__dataclass_fields__'): config = { - CFG_VIDEO_EXTENSIONS: config.video_extensions, CFG_DOCUMENT_EXTENSIONS: config.document_extensions, CFG_IMAGE_EXTENSIONS: config.image_extensions, CFG_AUDIO_EXTENSIONS: config.audio_extensions, @@ -55,7 +50,6 @@ def __init__(self, config: Union[Dict[str, Any], Any] = None): } self.config = config - self.video_extensions = self._to_set(config.get(CFG_VIDEO_EXTENSIONS, VIDEO_EXTENSIONS)) self.document_extensions = self._to_set(config.get(CFG_DOCUMENT_EXTENSIONS, DOCUMENT_EXTENSIONS)) self.image_extensions = self._to_set(config.get(CFG_IMAGE_EXTENSIONS, IMAGE_EXTENSIONS)) self.audio_extensions = self._to_set(config.get(CFG_AUDIO_EXTENSIONS, AUDIO_EXTENSIONS)) @@ -63,8 +57,7 @@ def __init__(self, config: Union[Dict[str, Any], Any] = None): self.enable_image_processing = config.get(CFG_ENABLE_IMAGE_PROCESSING, False) self.enable_audio_processing = config.get(CFG_ENABLE_AUDIO_PROCESSING, False) - logger.info(f"FileRouter initialized - Video: {len(self.video_extensions)} types, " - f"Documents: {len(self.document_extensions)} types") + logger.info(f"FileRouter initialized - Documents: {len(self.document_extensions)} types") @staticmethod def _to_set(value: Union[List, Set, None]) -> Set[str]: @@ -79,9 +72,6 @@ def route(self, filename: str) -> dict: if ext in self.skip_extensions: return {KEY_DESTINATION: DEST_SKIP, KEY_FILE_TYPE: FILE_TYPE_SKIP, KEY_EXTENSION: ext, KEY_REASON: 'File extension in skip list'} - if ext in self.video_extensions: - return {KEY_DESTINATION: DEST_VSS, KEY_FILE_TYPE: FILE_TYPE_VIDEO, KEY_EXTENSION: ext} - if ext in self.document_extensions: return {KEY_DESTINATION: DEST_RAG, KEY_FILE_TYPE: FILE_TYPE_DOCUMENT, KEY_EXTENSION: ext} @@ -97,8 +87,5 @@ def route(self, filename: str) -> dict: return {KEY_DESTINATION: DEST_RAG, KEY_FILE_TYPE: FILE_TYPE_UNKNOWN, KEY_EXTENSION: ext, KEY_REASON: 'Unknown extension, attempting RAG ingestion'} - def is_video(self, filename: str) -> bool: - return Path(filename).suffix.lower() in self.video_extensions - def is_document(self, filename: str) -> bool: return Path(filename).suffix.lower() in self.document_extensions diff --git a/examples/rag_event_ingest/kafka_consumer/services/__init__.py b/examples/rag_event_ingest/kafka_consumer/services/__init__.py index 24e2e98dd..db2c0e347 100644 --- a/examples/rag_event_ingest/kafka_consumer/services/__init__.py +++ b/examples/rag_event_ingest/kafka_consumer/services/__init__.py @@ -3,6 +3,5 @@ from .storage import ObjectStorage from .document_indexer import DocumentIndexer -from .video_analyzer import VideoAnalyzer -__all__ = ['ObjectStorage', 'DocumentIndexer', 'VideoAnalyzer'] +__all__ = ['ObjectStorage', 'DocumentIndexer'] diff --git a/examples/rag_event_ingest/kafka_consumer/services/document_indexer.py b/examples/rag_event_ingest/kafka_consumer/services/document_indexer.py index 55923accb..ac60d41a2 100644 --- a/examples/rag_event_ingest/kafka_consumer/services/document_indexer.py +++ b/examples/rag_event_ingest/kafka_consumer/services/document_indexer.py @@ -19,12 +19,9 @@ STATUS_FAILED, TIMEOUT_DEFAULT, TIMEOUT_MAX_TASK_WAIT, - VIDEO_INDEX_TIMEOUT, COLLECTION_EMBEDDING_DIMENSION, COLLECTION_CHUNK_SIZE, COLLECTION_CHUNK_OVERLAP, - COLLECTION_VIDEO_CHUNK_SIZE, - COLLECTION_VIDEO_CHUNK_OVERLAP, CONTENT_TYPE_MAP, DEFAULT_CONTENT_TYPE, FIELD_COLLECTION_NAME, @@ -224,74 +221,6 @@ def delete(self, filename: str, collection: str) -> bool: logger.error(f"Delete failed: {response.status_code}") return False - def index_video_description( - self, - collection: str, - video_id: str, - video_name: str, - description: str - ) -> bool: - """Index video description in Milvus.""" - try: - # Ensure collection exists first - if not self.ensure_collection_exists(collection): - logger.error(f"Failed to ensure collection '{collection}' exists") - return False - - doc_filename = f"{video_name}_description.json" - - # Delete existing document first to allow re-indexing - self.delete(doc_filename, collection) - - content = f"""Video Name: {video_name} -Video ID: {video_id} -Source: VSS Analysis - -Description: -{description}""" - - doc_data = { - 'content': content, - 'metadata': { - 'content_type': 'video', - 'video_id': video_id, - 'video_name': video_name, - 'source': 'vss_analysis' - } - } - - files = { - 'documents': (doc_filename, json.dumps(doc_data).encode(), 'application/json') - } - - data_config = { - FIELD_COLLECTION_NAME: collection, - FIELD_BLOCKING: True, - FIELD_SPLIT_OPTIONS: { - FIELD_CHUNK_SIZE: COLLECTION_VIDEO_CHUNK_SIZE, - FIELD_CHUNK_OVERLAP: COLLECTION_VIDEO_CHUNK_OVERLAP - }, - FIELD_GENERATE_SUMMARY: False - } - - response = requests.post( - f'{self.base_url}{API_INGESTOR_DOCUMENTS}', - files=files, - data={'data': json.dumps(data_config)}, - timeout=VIDEO_INDEX_TIMEOUT - ) - - if response.status_code in [200, 201, 202]: - logger.info(f"✓ Video description indexed to '{collection}'") - return True - else: - logger.error(f"Failed to index video description: {response.status_code} - {response.text}") - return False - - except requests.RequestException as e: - logger.error(f"Error indexing video description: {e}") - return False - def _get_content_type(self, filename: str) -> str: """Get content type from filename.""" ext = Path(filename).suffix.lower() diff --git a/examples/rag_event_ingest/kafka_consumer/services/video_analyzer.py b/examples/rag_event_ingest/kafka_consumer/services/video_analyzer.py deleted file mode 100644 index af7082550..000000000 --- a/examples/rag_event_ingest/kafka_consumer/services/video_analyzer.py +++ /dev/null @@ -1,326 +0,0 @@ -# services/video_analyzer.py -"""Video analysis service using VSS 2.4 API.""" - -import json -import re -import logging -from pathlib import Path -from typing import Optional, Tuple -import requests - -from config import ( - API_VSS_FILES, - API_VSS_SUMMARIZE, - CONTENT_TYPE_MAP, - DEFAULT_CONTENT_TYPE, - VSS_DEFAULT_PROMPT, - VSS_SYSTEM_PROMPT, - VSS_CAPTION_SUMMARIZATION_PROMPT, - VSS_SUMMARY_AGGREGATION_PROMPT, - VSS_CHUNK_DURATION, - VSS_CHUNK_OVERLAP, - VSS_NUM_FRAMES_PER_CHUNK, - VSS_MAX_TOKENS, - VSS_MODEL, - VSS_UPLOAD_TIMEOUT, - VSS_STREAM_ENABLED, - # API fields - FIELD_ID, - FIELD_FILE, - FIELD_MODEL, - FIELD_PROMPT, - FIELD_SYSTEM_PROMPT, - FIELD_MAX_TOKENS, - FIELD_CHUNK_DURATION, - FIELD_CHUNK_OVERLAP_DURATION, - FIELD_NUM_FRAMES_PER_CHUNK, - FIELD_CAPTION_SUMMARIZATION_PROMPT, - FIELD_SUMMARY_AGGREGATION_PROMPT, - FIELD_PURPOSE, - FIELD_MEDIA_TYPE, - FIELD_STREAM, - VALUE_VISION, - VALUE_VIDEO, - RESP_CONTENT, - RESP_RESPONSE, - RESP_TEXT, - RESP_CHOICES, - RESP_MESSAGE, - RESP_DELTA, - RESP_DATA_PREFIX, -) - -logger = logging.getLogger(__name__) - - -class VideoAnalyzer: - """Analyzes videos using VSS 2.4 /files and /summarize APIs.""" - - def __init__(self, base_url: str, timeout: int = 1800): - """Initialize video analyzer. - - Args: - base_url: VSS server URL - timeout: Timeout for summarization (default 30 min for long videos) - """ - self.base_url = base_url.rstrip('/') - self.timeout = timeout - - logger.info(f"VideoAnalyzer initialized (VSS 2.4): {self.base_url}") - - def upload_video(self, video_data: bytes, filename: str) -> Tuple[bool, Optional[str]]: - """Upload video to VSS using /files API. - - Args: - video_data: Video file bytes - filename: Original filename - - Returns: - Tuple of (success, file_id) - """ - file_name_sanitized = self._sanitize_filename(filename) - content_type = self._get_content_type(file_name_sanitized) - - logger.info(f"Uploading video to VSS: {file_name_sanitized} ({len(video_data)} bytes)") - - try: - response = requests.post( - f'{self.base_url}{API_VSS_FILES}', - files={FIELD_FILE: (file_name_sanitized, video_data, content_type)}, - data={FIELD_PURPOSE: VALUE_VISION, FIELD_MEDIA_TYPE: VALUE_VIDEO}, - timeout=VSS_UPLOAD_TIMEOUT - ) - except requests.RequestException as e: - logger.error(f"Error uploading video to VSS: {e}") - return False, None - - if response.status_code not in [200, 201]: - logger.error(f"Upload failed: {response.status_code}") - logger.error(f"Response: {response.text}") - return False, None - - result = response.json() - file_id = result.get(FIELD_ID) - - if not file_id: - logger.error(f"No file ID in response: {result}") - return False, None - - logger.info(f"✓ Video uploaded: {file_name_sanitized} (id: {file_id})") - return True, file_id - - def get_video_description(self, file_id: str) -> Optional[str]: - """Get video description using /summarize API. - - Uses streaming mode for real-time progress updates if enabled. - - Args: - file_id: File ID from upload - - Returns: - Video summary text, or None if failed - """ - if VSS_STREAM_ENABLED: - return self._get_description_streaming(file_id) - return self._get_description_blocking(file_id) - - def _build_summarize_payload(self, file_id: str, stream: bool = False) -> dict: - """Build the payload for summarize request.""" - payload = { - FIELD_ID: file_id, - FIELD_MODEL: VSS_MODEL, - FIELD_PROMPT: VSS_DEFAULT_PROMPT, - FIELD_SYSTEM_PROMPT: VSS_SYSTEM_PROMPT, - FIELD_MAX_TOKENS: VSS_MAX_TOKENS, - FIELD_STREAM: stream, - } - - # Add chunking for long videos - if VSS_CHUNK_DURATION > 0: - payload.update({ - FIELD_CHUNK_DURATION: VSS_CHUNK_DURATION, - FIELD_CHUNK_OVERLAP_DURATION: VSS_CHUNK_OVERLAP, - FIELD_NUM_FRAMES_PER_CHUNK: VSS_NUM_FRAMES_PER_CHUNK, - FIELD_CAPTION_SUMMARIZATION_PROMPT: VSS_CAPTION_SUMMARIZATION_PROMPT, - FIELD_SUMMARY_AGGREGATION_PROMPT: VSS_SUMMARY_AGGREGATION_PROMPT, - }) - - return payload - - def _get_description_blocking(self, file_id: str) -> Optional[str]: - """Get video description using blocking mode.""" - logger.info(f"Requesting summarization (blocking) for file: {file_id}") - - payload = self._build_summarize_payload(file_id, stream=False) - - if VSS_CHUNK_DURATION > 0: - logger.info(f"Using chunked processing: {VSS_CHUNK_DURATION}s chunks, " - f"{VSS_NUM_FRAMES_PER_CHUNK} frames/chunk") - - try: - response = requests.post( - f'{self.base_url}{API_VSS_SUMMARIZE}', - json=payload, - headers={'Content-Type': 'application/json'}, - timeout=self.timeout - ) - except requests.RequestException as e: - logger.error(f"Error requesting video summary: {e}") - return None - - if response.status_code != 200: - logger.error(f"Summarize failed: {response.status_code}") - logger.error(f"Response: {response.text}") - return None - - result = response.json() - content = self._extract_content(result) - - if not content: - logger.warning("No content in summarize response") - return None - - logger.info(f"✓ Got video summary ({len(content)} chars)") - return content - - def _get_description_streaming(self, file_id: str) -> Optional[str]: - """Get video description using streaming mode (Server-Sent Events).""" - logger.info(f"Requesting summarization (streaming) for file: {file_id}") - - payload = self._build_summarize_payload(file_id, stream=True) - - if VSS_CHUNK_DURATION > 0: - logger.info(f"Using chunked processing: {VSS_CHUNK_DURATION}s chunks, " - f"{VSS_NUM_FRAMES_PER_CHUNK} frames/chunk") - - try: - response = requests.post( - f'{self.base_url}{API_VSS_SUMMARIZE}', - json=payload, - headers={'Content-Type': 'application/json'}, - timeout=self.timeout, - stream=True # Enable streaming response - ) - except requests.RequestException as e: - logger.error(f"Error requesting video summary (streaming): {e}") - return None - - if response.status_code != 200: - logger.error(f"Summarize failed: {response.status_code}") - logger.error(f"Response: {response.text}") - return None - - # Collect streaming chunks - content = self._collect_stream_content(response) - - if not content: - logger.warning("No content in streaming response") - return None - - logger.info(f"✓ Got video summary via streaming ({len(content)} chars)") - return content - - def _collect_stream_content(self, response) -> Optional[str]: - """Collect content from streaming response (Server-Sent Events).""" - chunks = [] - chunk_count = 0 - - for line in response.iter_lines(decode_unicode=True): - if not line: - continue - - # SSE format: "data: {...json...}" - if line.startswith(RESP_DATA_PREFIX): - json_str = line[len(RESP_DATA_PREFIX):] - - # Handle [DONE] marker - if json_str.strip() == '[DONE]': - logger.info("Stream completed") - break - - try: - data = json.loads(json_str) - content = self._extract_stream_chunk(data) - if content: - chunks.append(content) - chunk_count += 1 - if chunk_count % 10 == 0: - logger.info(f"Received {chunk_count} chunks...") - elif chunk_count == 0: - # Debug: log first non-content chunk to understand format - logger.debug(f"SSE data (no content): {json_str[:200]}") - except json.JSONDecodeError as e: - logger.debug(f"SSE JSON parse error: {e}, line: {json_str[:100]}") - continue - else: - # Non-data line - could be event type or comment - if chunk_count == 0: - logger.debug(f"SSE non-data line: {line[:100]}") - - if not chunks: - logger.warning("No streaming chunks collected, checking full response") - return None - - logger.info(f"Collected {chunk_count} streaming chunks") - return ''.join(chunks) - - def _extract_stream_chunk(self, data: dict) -> Optional[str]: - """Extract content from a single streaming chunk.""" - choices = data.get(RESP_CHOICES, []) - if choices: - delta = choices[0].get(RESP_DELTA, {}) - content = delta.get(RESP_CONTENT) - if content: - return content - - message = choices[0].get(RESP_MESSAGE, {}) - content = message.get(RESP_CONTENT) - if content: - return content - - if RESP_CONTENT in data: - return data[RESP_CONTENT] - - if RESP_RESPONSE in data: - return data[RESP_RESPONSE] - - if RESP_TEXT in data: - return data[RESP_TEXT] - - return None - - def _extract_content(self, result: dict) -> Optional[str]: - """Extract text content from VSS response. - - returns OpenAI-compatible format: - { - "choices": [{"message": {"content": "..."}}] - } - """ - # Try OpenAI format first - choices = result.get(RESP_CHOICES, []) - if choices: - message = choices[0].get(RESP_MESSAGE, {}) - content = message.get(RESP_CONTENT) - if content: - return content.strip() - - if RESP_CONTENT in result: - return result[RESP_CONTENT].strip() - - if RESP_RESPONSE in result: - return result[RESP_RESPONSE].strip() - - return None - - def _sanitize_filename(self, filename: str) -> str: - """Sanitize filename for VSS compatibility.""" - name = Path(filename).name - file_name_sanitized = re.sub(r'[^A-Za-z0-9_.\-]', '_', name) - file_name_sanitized = re.sub(r'_+', '_', file_name_sanitized) - return file_name_sanitized - - def _get_content_type(self, filename: str) -> str: - """Get content type from filename.""" - ext = Path(filename).suffix.lower() - return CONTENT_TYPE_MAP.get(ext, DEFAULT_CONTENT_TYPE) diff --git a/notebooks/rag_event_ingest.ipynb b/notebooks/rag_event_ingest.ipynb index 6609a6e1d..b2bde4727 100644 --- a/notebooks/rag_event_ingest.ipynb +++ b/notebooks/rag_event_ingest.ipynb @@ -1,1263 +1,793 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Video/Document Continuous Ingestion from Object Storage\n", - "\n", - "## Purpose\n", - "\n", - "This notebook demonstrates an **automated document and video ingestion pipeline** that:\n", - "\n", - "1. Monitors emulated object storage for new uploads via Kafka events\n", - "2. Routes files to appropriate AI services based on file type, currently supports document and video ingestion\n", - "5. Enables RAG Agent for semantic search and contextual Q&A over all ingested content\n", - "\n", - "## What Gets Deployed\n", - "\n", - "1. **NVIDIA RAG** - Document indexing, vector search, and AI-powered Q&A (NIMs, Milvus, Ingestor)\n", - "2. **NVIDIA VSS** - Video understanding and summarization (VLM, LLM NIMs, VSS Engine)\n", - "3. **Continuous Ingestion** - Event-driven ingestion pipeline (Kafka, MinIO, Consumer)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prerequisites\n", - "\n", - "### Hardware\n", - "- **GPU**: 4x RTX PRO 6000 Blackwell or 4x H100\n", - "\n", - "#### Default GPU Assignment\n", - "\n", - "| GPU | Service |\n", - "|-----|---------|\n", - "| 0 | RAG NIMs (Embedding, Reranker) |\n", - "| 1 | RAG LLM NIM (Llama-3.3-Nemotron-Super-49B) |\n", - "| 2 | VSS LLM NIM (Llama-3.1-8B-Instruct) |\n", - "| 3 | VSS VLM / via-server (Cosmos-Reason2-8B) |\n", - "\n", - "\n", - "### Software (pre-installed required)\n", - "- Ubuntu 22.04 or later\n", - "- Docker 24.0+ with Docker Compose v2\n", - "- NVIDIA Driver 570+\n", - "- NVIDIA Container Toolkit\n", - "\n", - "### API Keys\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
KeyPurposeHow to Get
NGC_API_KEYDocker login, NIM deploymentsNGC Portal → Generate API Key
HF_TOKENDownload VSS modelsHuggingFace Tokens → Create token with Read access
\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Table of Contents\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SectionDescription
SetupClone repo, install deps, set API keys, load helpers
Deploy RAGNIMs, Vector DB, Ingestor, RAG Server
Deploy VSSClone VSS, deploy NIMs and VLM
Deploy Continuous IngestionKafka, MinIO, Consumer
TestingUpload documents & videos, query RAG
Clean UpStop services, clean data
\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## References\n", - "\n", - "- **RAG Blueprint**: [NVIDIA RAG Documentation](https://github.com/NVIDIA-AI-Blueprints/rag/blob/develop/docs/deploy-docker-self-hosted.md)\n", - "- **VSS**: [Video Search & Summarization Documentation](https://docs.nvidia.com/vss/latest/index.html)\n", - "- **NIM**: [NVIDIA NIM Documentation](https://docs.nvidia.com/nim/index.html)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Setup\n", - "\n", - "Clone the repository, configure API keys, and load helper functions.\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Clone Repository\n", - "\n", - "Clone the RAG Blueprint repo to `~/rag`. This includes the consumer source code, deploy configs, and sample test data.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess, sys, os, shutil\n", - "\n", - "RAG_REPO_DIR = os.path.expanduser(\"~/rag\")\n", - "RAG_REPO_URL = \"https://github.com/NVIDIA-AI-Blueprints/rag.git\"\n", - "\n", - "# Ensure git-lfs is installed before any LFS operations\n", - "if not shutil.which(\"git-lfs\"):\n", - " print(\"[INSTALLING] git-lfs...\")\n", - " subprocess.run(\"sudo apt-get update && sudo apt-get install -y git-lfs && git lfs install\", shell=True, check=True)\n", - "else:\n", - " print(\"[OK] git-lfs found\")\n", - "\n", - "# Clone from correct branch (skip if already exists)\n", - "if not os.path.exists(RAG_REPO_DIR):\n", - " subprocess.run(f\"git clone {RAG_REPO_URL} {RAG_REPO_DIR}\", shell=True, check=True)\n", - "else:\n", - " print(f\"[OK] RAG repo already exists: {RAG_REPO_DIR}\")\n", - "subprocess.run(\"git lfs pull\", shell=True, cwd=RAG_REPO_DIR, check=True)\n", - "\n", - "# Verify\n", - "for path in [\"deploy/compose\", \"examples/rag_event_ingest/kafka_consumer\", \"examples/rag_event_ingest/data\"]:\n", - " status = \"[OK]\" if os.path.exists(os.path.join(RAG_REPO_DIR, path)) else \"[MISSING]\"\n", - " print(f\" {status} {path}\")\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Install Dependencies\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "! python3 -m ensurepip --upgrade" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Ensure pip is available (some minimal Python installs lack it)\n", - "subprocess.run([sys.executable, \"-m\", \"ensurepip\", \"--upgrade\"], capture_output=True)\n", - "\n", - "def check_install_system_pkg(cmd: str, install_cmd: str):\n", - " if shutil.which(cmd):\n", - " print(f\" [OK] {cmd} found\")\n", - " return True\n", - " print(f\" [INSTALLING] {cmd}...\")\n", - " result = subprocess.run(install_cmd, shell=True, capture_output=True, text=True)\n", - " if result.returncode == 0:\n", - " print(f\" [OK] {cmd} installed\")\n", - " return True\n", - " print(f\" [ERROR] Failed to install {cmd}. Please install manually: {install_cmd}\")\n", - " return False\n", - "\n", - "check_install_system_pkg(\"git\", \"sudo apt-get update && sudo apt-get install -y git\")\n", - "\n", - "# Install Python packages\n", - "subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"minio\", \"aiohttp\", \"requests\", \"python-dotenv\", \"pyyaml\"])\n", - "print(\"[OK] Ready\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Set API Keys\n", - "\n", - "Configure NGC and HuggingFace API keys for NIM deployments and model downloads.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import getpass\n", - "\n", - "def set_api_key(env_var: str, prompt: str, required: bool = True):\n", - " if os.environ.get(env_var):\n", - " print(f\" [OK] {env_var} already set ({os.environ[env_var][:10]}...)\")\n", - " return True\n", - " key = getpass.getpass(prompt)\n", - " if key:\n", - " os.environ[env_var] = key\n", - " print(f\" [OK] {env_var} set\")\n", - " return True\n", - " if required:\n", - " print(f\" [ERROR] {env_var} is required\")\n", - " return False\n", - " print(f\" [SKIP] {env_var} (optional)\")\n", - " return True\n", - "\n", - "set_api_key(\"NGC_API_KEY\", \"Enter NGC_API_KEY (starts with 'nvapi-'): \", required=True)\n", - "set_api_key(\"HF_TOKEN\", \"Enter HF_TOKEN (optional, press Enter to skip): \", required=False)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Helper Functions\n", - "\n", - "Shared utilities for deployment, file upload, status checks, and RAG queries.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Install dependencies\n", - "import sys\n", - "!{sys.executable} -m pip install -q minio aiohttp requests python-dotenv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os, sys, json, re, subprocess, time, socket, asyncio\n", - "import aiohttp, requests\n", - "from typing import List, Optional, Dict\n", - "\n", - "try:\n", - " from minio import Minio\n", - " from minio.error import S3Error\n", - "except ImportError:\n", - " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"minio\"])\n", - " from minio import Minio\n", - " from minio.error import S3Error\n", - "\n", - "# =============================================================================\n", - "# CONFIGURATION\n", - "# =============================================================================\n", - "\n", - "# Paths relative to RAG repo root\n", - "RAG_REPO_DIR = os.path.expanduser(\"~/rag\")\n", - "EXAMPLE_DIR = os.path.join(RAG_REPO_DIR, \"examples/rag_event_ingest\")\n", - "AIDP_COMPOSE_FILE = os.path.join(EXAMPLE_DIR, \"deploy/docker-compose.yaml\")\n", - "DATA_DIR = os.path.join(EXAMPLE_DIR, \"data\")\n", - "RAG_SERVER_URL = \"http://localhost:8081\"\n", - "INGESTOR_URL = \"http://localhost:8082\"\n", - "\n", - "VSS_DIR = os.path.expanduser(\"~/video-search-and-summarization\")\n", - "VSS_UI_PORT = 9110\n", - "VSS_API_PORT = 8110\n", - "VSS_LLM_PORT = 8107\n", - "VSS_EMBED_PORT = 8106\n", - "VSS_RERANK_PORT = 8105\n", - "LOCAL_NIM_CACHE = os.path.expanduser(\"~/.cache/nim\")\n", - "\n", - "MINIO_ENDPOINT = \"localhost:9201\"\n", - "MINIO_ACCESS_KEY = \"minioadmin\"\n", - "MINIO_SECRET_KEY = \"minioadmin\"\n", - "MINIO_BUCKET = \"aidp-bucket\"\n", - "MINIO_COLLECTION = \"aidp_bucket\"\n", - "MINIO_CONSOLE_PORT = 9211\n", - "\n", - "# =============================================================================\n", - "# SHARED UTILITIES\n", - "# =============================================================================\n", - "\n", - "def run_command(cmd: str, capture: bool = False) -> Optional[str]:\n", - " \"\"\"Execute a shell command and print it.\"\"\"\n", - " print(f\"$ {cmd}\")\n", - " result = subprocess.run(cmd, shell=True, capture_output=capture, text=True)\n", - " return result.stdout if capture else None\n", - "\n", - "def get_host_ip() -> str:\n", - " \"\"\"Get host IP address for external access URLs.\"\"\"\n", - " try:\n", - " s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n", - " s.connect((\"8.8.8.8\", 80))\n", - " ip = s.getsockname()[0]\n", - " s.close()\n", - " return ip\n", - " except OSError:\n", - " return \"localhost\"\n", - "\n", - "def get_minio_client() -> Minio:\n", - " \"\"\"Create MinIO client for AIDP bucket operations.\"\"\"\n", - " return Minio(MINIO_ENDPOINT, access_key=MINIO_ACCESS_KEY, secret_key=MINIO_SECRET_KEY, secure=False)\n", - "\n", - "def upload_file(local_path: str, object_name: Optional[str] = None) -> bool:\n", - " \"\"\"Upload a local file to MinIO AIDP bucket.\"\"\"\n", - " if not os.path.exists(local_path):\n", - " print(f\"[ERROR] File not found: {local_path}\")\n", - " return False\n", - " obj = object_name or os.path.basename(local_path)\n", - " try:\n", - " client = get_minio_client()\n", - " if not client.bucket_exists(MINIO_BUCKET):\n", - " client.make_bucket(MINIO_BUCKET)\n", - " client.fput_object(MINIO_BUCKET, obj, local_path)\n", - " print(f\"[OK] Uploaded: {obj}\")\n", - " return True\n", - " except S3Error as e:\n", - " print(f\"[ERROR] {e}\")\n", - " return False\n", - "\n", - "def verify_file_in_storage(object_name: str, bucket: str = MINIO_BUCKET) -> bool:\n", - " \"\"\"Check if a file exists in MinIO bucket and print verification status.\"\"\"\n", - " try:\n", - " client = get_minio_client()\n", - " stat = client.stat_object(bucket, object_name)\n", - " print(f\"[OK] File verified in storage:\")\n", - " print(f\" Bucket: {bucket}\")\n", - " print(f\" Object: {object_name}\")\n", - " print(f\" Size: {stat.size:,} bytes\")\n", - " print(f\" Modified: {stat.last_modified}\")\n", - " return True\n", - " except S3Error as e:\n", - " print(f\"[ERROR] File not found in storage: {object_name}\")\n", - " print(f\" Error: {e}\")\n", - " return False\n", - "\n", - "def get_consumer_logs(lines: int = 30) -> None:\n", - " \"\"\"Show recent Kafka consumer logs.\"\"\"\n", - " run_command(f\"docker logs kafka-consumer --tail {lines}\")\n", - "\n", - "async def query_rag(question: str, collection: str = None) -> Optional[str]:\n", - " \"\"\"Query RAG system and print the answer.\"\"\"\n", - " coll = collection or MINIO_COLLECTION\n", - " print(f\"Q: {question}\\nCollection: {coll}\\n\" + \"-\" * 40)\n", - "\n", - " payload = {\n", - " \"messages\": [{\"role\": \"user\", \"content\": question}],\n", - " \"use_knowledge_base\": True,\n", - " \"collection_names\": [coll],\n", - " }\n", - " try:\n", - " async with aiohttp.ClientSession() as session:\n", - " async with session.post(\n", - " f\"{RAG_SERVER_URL}/generate\", json=payload,\n", - " timeout=aiohttp.ClientTimeout(total=120),\n", - " ) as resp:\n", - " text = await resp.text()\n", - " # Parse SSE response: extract content from each \"data: {...}\" line\n", - " chunks = []\n", - " for line in text.split(\"\\n\"):\n", - " if not line.startswith(\"data: \") or line[6:] == \"[DONE]\":\n", - " continue\n", - " try:\n", - " msg = json.loads(line[6:]).get(\"choices\", [{}])[0].get(\"message\", {})\n", - " if msg.get(\"content\"):\n", - " chunks.append(msg[\"content\"])\n", - " except json.JSONDecodeError:\n", - " pass\n", - " answer = \"\".join(chunks)\n", - " print(f\"Answer: {answer}\")\n", - " return answer\n", - " except aiohttp.ClientError as e:\n", - " print(f\"[ERROR] {e}\")\n", - " return None\n", - "\n", - "def extract_rag_default_compose_var(file_path, var_name):\n", - " \"\"\"Extract default value from ${VAR:-default} pattern in a compose file.\"\"\"\n", - " pattern = re.compile(r'\\$\\{' + var_name + r':-\"?([^\"}\\s]+)\"?\\}')\n", - " with open(file_path) as f:\n", - " for line in f:\n", - " if line.lstrip().startswith('#'):\n", - " continue\n", - " m = pattern.search(line)\n", - " if m:\n", - " return m.group(1)\n", - " return None\n", - "\n", - "print(f\"[OK] Helpers loaded | Host IP: {get_host_ip()}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Deploy NVIDIA RAG\n", - "\n", - "Deploy the NVIDIA RAG: NIMs (LLM, Embedding, Reranker), Milvus vector database, Ingestor server, and RAG server.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ngc_key = os.environ.get(\"NGC_API_KEY\")\n", - "if not ngc_key:\n", - " raise RuntimeError(\"NGC_API_KEY not set! Run the API keys cell first.\")\n", - "\n", - "os.chdir(RAG_REPO_DIR)\n", - "\n", - "# Set env vars needed by docker compose\n", - "os.environ[\"NGC_API_KEY\"] = ngc_key\n", - "os.environ[\"USERID\"] = f\"{os.getuid()}:{os.getgid()}\"\n", - "os.environ[\"COLLECTION_NAME\"] = MINIO_COLLECTION\n", - "\n", - "# Load RAG .env defaults (MODEL_DIRECTORY, etc.)\n", - "from dotenv import load_dotenv\n", - "env_file = os.path.join(RAG_REPO_DIR, \"deploy/compose/.env\")\n", - "if os.path.exists(env_file):\n", - " load_dotenv(env_file, override=False)\n", - "\n", - "# Login to nvcr.io\n", - "subprocess.run(f\"echo {ngc_key} | docker login nvcr.io -u '$oauthtoken' --password-stdin\",\n", - " shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", - "\n", - "# Deploy components\n", - "for label, compose_file in [\n", - " (\"NIMs\", \"deploy/compose/nims.yaml\"),\n", - " (\"Vector DB\", \"deploy/compose/vectordb.yaml\"),\n", - "]:\n", - " print(f\"Deploying {label}...\")\n", - " run_command(f\"docker compose -f {compose_file} up -d\")\n", - "\n", - "print(\"Waiting 30s for Milvus...\")\n", - "time.sleep(30)\n", - "\n", - "for label, compose_file in [\n", - " (\"Ingestor\", \"deploy/compose/docker-compose-ingestor-server.yaml\"),\n", - " (\"RAG Server\", \"deploy/compose/docker-compose-rag-server.yaml\"),\n", - "]:\n", - " print(f\"Deploying {label}...\")\n", - " run_command(f\"docker compose -f {compose_file} up -d\")\n", - "\n", - "ip = get_host_ip()\n", - "print(f\"\\nRAG deployed: http://{ip}:8081 (server) | http://{ip}:8082 (ingestor) | http://{ip}:8090 (UI)\")\n", - "print(f\"COLLECTION_NAME: {MINIO_COLLECTION}\")\n", - "print(\"Wait ~10 minutes for NIMs to load models, then run the status check cell.\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Verify RAG services are healthy. Wait ~10 minutes for NIMs to load models.\n", - "\n", - "The deployment status should be:\n", - "```\n", - "NAMES STATUS\n", - "rag-frontend Up About a minute\n", - "rag-server Up About a minute\n", - "ingestor-server Up About a minute\n", - "milvus-standalone Up 2 minutes (healthy)\n", - "milvus-etcd Up 2 minutes (healthy)\n", - "milvus-minio Up 2 minutes (healthy)\n", - "nim-llm-ms Up 2 minutes (healthy)\n", - "nemotron-embedding-ms Up 2 minutes (healthy)\n", - "nemotron-ranking-ms Up 2 minutes (healthy)\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Check service status and print access URLs\n", - "print(\"Wait ~10 minutes for services to become healthy.\")\n", - "print(\"Run this cell again after waiting.\\n\")\n", - "\n", - "ip = get_host_ip()\n", - "for name, port, path in [\n", - " (\"RAG Server\", 8081, \"/health\"), (\"Ingestor\", 8082, \"/health\"),\n", - " (\"Frontend\", 8090, \"/\"), (\"Milvus\", 19530, \"/v1/vector/collections\"),\n", - "]:\n", - " try:\n", - " s = \"[OK]\" if requests.get(f\"http://localhost:{port}{path}\", timeout=10).status_code == 200 else \"[WARN]\"\n", - " except requests.ConnectionError:\n", - " s = \"[DOWN]\"\n", - " except requests.Timeout:\n", - " s = \"[TIMEOUT]\"\n", - " print(f\" {s} {name}: http://{ip}:{port}\")\n", - "run_command(\"docker ps --format 'table {{.Names}}\\t{{.Status}}' | grep -E '(rag|milvus|ingestor|nim|nemotron|NAMES)'\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Deploy NVIDIA VSS\n", - "\n", - "Deploy the NVIDIA VSS: NIMs (LLM, Embedding, Reranker) and VLM for video analysis.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# VSS deployment configuration\n", - "VSS_REPO_URL = \"https://github.com/NVIDIA-AI-Blueprints/video-search-and-summarization.git\"\n", - "VSS_TAG = \"v2.4.1\" # VSS Blueprint version (must match VIA_IMAGE tag)\n", - "VSS_GPU_DEVICE = 2 # GPU for VSS LLM NIM\n", - "VSS_VLM_GPU_DEVICE = 3 # GPU for VLM (via-server with Cosmos-Reason2)\n", - "\n", - "_rag_compose_path = os.path.join(RAG_REPO_DIR, \"deploy/compose/docker-compose-rag-server.yaml\")\n", - "RAG_EMBED_MODEL = extract_rag_default_compose_var(_rag_compose_path, \"APP_EMBEDDINGS_MODELNAME\")\n", - "RAG_RERANK_MODEL = extract_rag_default_compose_var(_rag_compose_path, \"APP_RANKING_MODELNAME\")\n", - "print(f\"RAG models from compose: embed={RAG_EMBED_MODEL}, rerank={RAG_RERANK_MODEL}\")\n", - "\n", - "# Only deploy LLM for VSS; embedding and reranker are shared from RAG stack\n", - "# RAG NIMs on nvidia-rag network: nemotron-embedding-ms:8000, nemotron-ranking-ms:8000\n", - "NIM_IMAGES = {\n", - " \"vss-llm\": (\"nvcr.io/nim/meta/llama-3.1-8b-instruct:1.12.0\", VSS_LLM_PORT),\n", - "}\n", - "\n", - "ngc_key = os.environ.get(\"NGC_API_KEY\", \"\")\n", - "hf_token = os.environ.get(\"HF_TOKEN\", \"\")\n", - "if not ngc_key:\n", - " raise RuntimeError(\"NGC_API_KEY not set!\")\n", - "\n", - "# Docker login\n", - "subprocess.run(f\"echo {ngc_key} | docker login nvcr.io -u '$oauthtoken' --password-stdin\",\n", - " shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", - "\n", - "# Clone VSS repo with correct release tag\n", - "if not os.path.exists(VSS_DIR):\n", - " print(f\"Cloning {VSS_REPO_URL} (tag: {VSS_TAG})...\")\n", - " subprocess.run(f\"git clone --branch {VSS_TAG} --depth 1 {VSS_REPO_URL} {VSS_DIR}\", shell=True)\n", - "else:\n", - " print(f\"[OK] VSS repo exists: {VSS_DIR}\")\n", - "\n", - "# Deploy VSS LLM NIM container\n", - "os.makedirs(LOCAL_NIM_CACHE, exist_ok=True)\n", - "for name, (image, port) in NIM_IMAGES.items():\n", - " subprocess.run(f\"docker rm -f {name} 2>/dev/null\", shell=True, capture_output=True)\n", - " cmd = f\"\"\"docker run -d --name {name} \\\n", - " -u $(id -u) --gpus '\"device={VSS_GPU_DEVICE}\"' --shm-size=16GB \\\n", - " --network nvidia-rag -e NGC_API_KEY={ngc_key} \\\n", - " -v \"{LOCAL_NIM_CACHE}:/opt/nim/.cache\" \\\n", - " -p {port}:8000 -e NIM_LOW_MEMORY_MODE=1 -e NIM_RELAX_MEM_CONSTRAINTS=1 \\\n", - " {image}\"\"\"\n", - " result = subprocess.run(cmd, shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", - " status = \"[OK]\" if result.returncode == 0 else \"[ERROR]\"\n", - " print(f\" {status} {name} -> port {port}\")\n", - "\n", - "# Deploy VSS application (VLM on separate GPU)\n", - "vss_deploy_dir = f\"{VSS_DIR}/deploy/docker/local_deployment_single_gpu\"\n", - "env_content = f\"\"\"NGC_API_KEY={ngc_key}\n", - "HF_TOKEN={hf_token}\n", - "VIA_IMAGE=nvcr.io/nvidia/blueprint/vss-engine:2.4.1\n", - "FRONTEND_PORT={VSS_UI_PORT}\n", - "BACKEND_PORT={VSS_API_PORT}\n", - "MILVUS_DB_HTTP_PORT=19091\n", - "MILVUS_DB_GRPC_PORT=29530\n", - "MINIO_PORT=9002\n", - "MINIO_WEBUI_PORT=9003\n", - "GRAPH_DB_USERNAME=neo4j\n", - "GRAPH_DB_PASSWORD=password\n", - "ARANGO_DB_USERNAME=arangodb\n", - "ARANGO_DB_PASSWORD=password\n", - "CA_RAG_CONFIG=./config.yaml\n", - "GUARDRAILS_CONFIG=./guardrails\n", - "NVIDIA_VISIBLE_DEVICES={VSS_VLM_GPU_DEVICE}\n", - "VLM_MODEL_TO_USE=cosmos-reason2\n", - "MODEL_PATH=git:https://huggingface.co/nvidia/Cosmos-Reason2-8B\n", - "VLLM_GPU_MEMORY_UTILIZATION=0.4\n", - "VLM_MAX_MODEL_LEN=20480\n", - "DISABLE_GUARDRAILS=true\n", - "DISABLE_CV_PIPELINE=true\n", - "ENABLE_AUDIO=false\n", - "\"\"\"\n", - "with open(f\"{vss_deploy_dir}/.env\", \"w\") as f:\n", - " f.write(env_content)\n", - "\n", - "# Patch config.yaml: LLM points to vss-llm, embedding/reranker reuse RAG NIMs\n", - "config_file = f\"{vss_deploy_dir}/config.yaml\"\n", - "if os.path.exists(config_file):\n", - " cfg = open(config_file).read()\n", - " cfg = re.sub(r\"http://[^:]+:8007/v1\", f\"http://host.docker.internal:{VSS_LLM_PORT}/v1\", cfg)\n", - " cfg = re.sub(r\"http://[^:]+:8006/v1\", \"http://host.docker.internal:9080/v1\", cfg)\n", - " cfg = re.sub(r\"http://[^:]+:8005/v1\", \"http://host.docker.internal:1976/v1\", cfg)\n", - " cfg = re.sub(r\"model:\\s*nvidia/[\\w.-]+-embed[\\w.-]+\", f\"model: {RAG_EMBED_MODEL}\", cfg)\n", - " cfg = re.sub(r\"model:\\s*nvidia/[\\w.-]+-re?rank[\\w.-]+\", f\"model: {RAG_RERANK_MODEL}\", cfg)\n", - "\n", - " open(config_file, \"w\").write(cfg)\n", - "\n", - "cmd = f\"cd {vss_deploy_dir} && set -a && source .env && set +a && docker compose up -d\"\n", - "subprocess.run(cmd, shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", - "\n", - "ip = get_host_ip()\n", - "print(f\"\\nVSS deployed: http://{ip}:{VSS_UI_PORT} (UI) | http://{ip}:{VSS_API_PORT} (API)\")\n", - "print(\"Embedding/Reranker: shared from RAG stack (nemotron-embedding-ms, nemotron-ranking-ms)\")\n", - "print(\"Wait ~10 minutes for NIMs to load models, then run the status check cell.\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Verify VSS services are healthy. Wait ~10 minutes for NIMs to load models.\n", - "\n", - "> **Note**: Embedding and reranker NIMs are shared from the RAG stack (`nemoretriever-embedding-ms`, `nemoretriever-ranking-ms`) — no separate VSS containers needed.\n", - "\n", - "The health check output should be:\n", - "```\n", - " [OK] VSS UI: http://:9110\n", - " [OK] VSS API: http://:8110\n", - " [OK] VSS LLM NIM: http://:8107\n", - " [OK] Embedding (shared from RAG): http://:9080\n", - " [OK] Reranker (shared from RAG): http://:1976\n", - "```\n", - "\n", - "The deployment status should be:\n", - "```\n", - "NAMES STATUS\n", - "local_deployment_single_gpu-via-server-1 Up About a minute\n", - "local_deployment_single_gpu-elasticsearch-1 Up About a minute\n", - "local_deployment_single_gpu-graph-db-1 Up About a minute\n", - "local_deployment_single_gpu-minio-1 Up About a minute\n", - "local_deployment_single_gpu-arango-db-1 Up About a minute\n", - "local_deployment_single_gpu-milvus-standalone-1 Up About a minute (healthy)\n", - "vss-llm Up About a minute\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Check service status and print access URLs\n", - "ip = get_host_ip()\n", - "for name, port, path in [\n", - " (\"VSS UI\", VSS_UI_PORT, \"/\"), (\"VSS API\", VSS_API_PORT, \"/\"),\n", - " (\"VSS LLM NIM\", VSS_LLM_PORT, \"/v1/health/ready\"),\n", - " (\"Embedding (shared from RAG)\", 9080, \"/v1/health/ready\"),\n", - " (\"Reranker (shared from RAG)\", 1976, \"/v1/health/ready\"),\n", - "]:\n", - " try:\n", - " requests.get(f\"http://localhost:{port}{path}\", timeout=10)\n", - " s = \"[OK]\"\n", - " except requests.ConnectionError:\n", - " s = \"[DOWN]\"\n", - " except requests.Timeout:\n", - " s = \"[TIMEOUT]\"\n", - " print(f\" {s} {name}: http://{ip}:{port}\")\n", - "run_command(\"docker ps --format 'table {{.Names}}\\t{{.Status}}' | grep -E '(vss|via|local_deployment|NAMES)'\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Deploy Continuous Ingestion from emulated object storage\n", - "\n", - "Deploy the Continuous Ingestion: Kafka message broker, MinIO object storage, and Kafka consumer for automated ingestion.\n", - "\n", - "## 1. Configure Video Analysis Prompts\n", - "\n", - "Customize the prompts used by the Kafka consumer when processing videos through VSS.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Consumer prompts for video analysis - customize these for your video content\n", - "# These are passed to the Kafka consumer which sends them to VSS for video summarization\n", - "\n", - "CONSUMER_VSS_PROMPT = \"\"\"Analyze this sports video. TIMESTAMP FORMAT: Convert seconds to MM:SS (40s=00:40, 80s=01:20, 150s=02:30, 200s=03:20). \\\n", - "Describe: key plays, scoring, turnovers, big gains, defensive stops, celebrations.\"\"\"\n", - "\n", - "CONSUMER_VSS_SYSTEM_PROMPT = \"\"\"You are a sports broadcaster. CRITICAL: Convert all timestamps from seconds to MM:SS format. \\\n", - "Examples: 40 seconds = 00:40, 90 seconds = 01:30, 200 seconds = 03:20, 350 seconds = 05:50. Focus on action, field position, execution, and game momentum.\"\"\"\n", - "\n", - "CONSUMER_VSS_CAPTION_SUMMARIZATION_PROMPT = \"\"\"Format: [MM:SS] Play description. CONVERT seconds to MM:SS (40s=00:40, 80s=01:20, 150s=02:30). \\\n", - "Describe: what happened, how it was executed, the result. Be vivid like a TV broadcast.\"\"\"\n", - "\n", - "CONSUMER_VSS_SUMMARY_AGGREGATION_PROMPT = \"\"\"Create game summary with MM:SS timestamps. CONVERT all times: 40s=00:40, 80s=01:20, 200s=03:20, 350s=05:50. \\\n", - "Highlight scoring plays, turnovers, momentum shifts, spectacular plays. Write like a highlight reel narrator - vivid and exciting.\"\"\"\n", - "\n", - "print(\"Consumer VSS prompts configured:\")\n", - "print(f\" VSS_PROMPT: {len(CONSUMER_VSS_PROMPT)} chars\")\n", - "print(f\" VSS_SYSTEM_PROMPT: {len(CONSUMER_VSS_SYSTEM_PROMPT)} chars\")\n", - "print(f\" VSS_CAPTION_SUMMARIZATION_PROMPT: {len(CONSUMER_VSS_CAPTION_SUMMARIZATION_PROMPT)} chars\")\n", - "print(f\" VSS_SUMMARY_AGGREGATION_PROMPT: {len(CONSUMER_VSS_SUMMARY_AGGREGATION_PROMPT)} chars\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Deploy Services\n", - "\n", - "Deploy Kafka, MinIO, and the Kafka consumer with custom prompts." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Verify prerequisites\n", - "net_check = subprocess.run(\"docker network inspect nvidia-rag\", shell=True, capture_output=True)\n", - "if net_check.returncode != 0:\n", - " raise RuntimeError(\"nvidia-rag network not found. Deploy RAG first.\")\n", - "\n", - "ngc_key = os.environ.get(\"NGC_API_KEY\", \"\")\n", - "if not ngc_key:\n", - " raise RuntimeError(\"NGC_API_KEY not set!\")\n", - "\n", - "host_ip = get_host_ip()\n", - "\n", - "# Set environment variables for docker compose (including custom prompts)\n", - "os.environ[\"HOST_IP\"] = host_ip\n", - "os.environ[\"VSS_SERVER_URL\"] = f\"http://{host_ip}:{VSS_API_PORT}\"\n", - "os.environ[\"VSS_PROMPT\"] = CONSUMER_VSS_PROMPT\n", - "os.environ[\"VSS_SYSTEM_PROMPT\"] = CONSUMER_VSS_SYSTEM_PROMPT\n", - "os.environ[\"VSS_CAPTION_SUMMARIZATION_PROMPT\"] = CONSUMER_VSS_CAPTION_SUMMARIZATION_PROMPT\n", - "os.environ[\"VSS_SUMMARY_AGGREGATION_PROMPT\"] = CONSUMER_VSS_SUMMARY_AGGREGATION_PROMPT\n", - "\n", - "# Login + pull + build\n", - "subprocess.run(f\"echo {ngc_key} | docker login nvcr.io -u '$oauthtoken' --password-stdin\",\n", - " shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", - "\n", - "compose = f\"docker compose -f {AIDP_COMPOSE_FILE}\"\n", - "subprocess.run(f\"{compose} pull --ignore-pull-failures\", shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", - "subprocess.run(f\"{compose} up -d --build\", shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", - "\n", - "print(f\"Continuous Ingestion deployed:\")\n", - "print(f\" Kafka UI: http://{host_ip}:8080\")\n", - "print(f\" MinIO Console: http://{host_ip}:{MINIO_CONSOLE_PORT}\")\n", - "print(f\" Credentials: minioadmin / minioadmin\")\n", - "print(f\" Custom prompts: ✓ passed to consumer\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Verify continuous ingestion services are running.\n", - "\n", - "The deployment status should be:\n", - "```\n", - "NAMES STATUS\n", - "kafka-consumer Up About a minute\n", - "aidp-kafka-ui Up About a minute\n", - "aidp-minio-mc Up About a minute\n", - "aidp-minio Up About a minute (healthy)\n", - "kafka Up About a minute (healthy)\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Check service status and print access URLs\n", - "ip = get_host_ip()\n", - "for name, port, path in [\n", - " (\"Kafka UI\", 8080, \"/\"),\n", - " (\"MinIO Console\", MINIO_CONSOLE_PORT, \"/\"),\n", - "]:\n", - " try:\n", - " s = \"[OK]\" if requests.get(f\"http://localhost:{port}{path}\", timeout=10).status_code == 200 else \"[WARN]\"\n", - " except requests.ConnectionError:\n", - " s = \"[DOWN]\"\n", - " except requests.Timeout:\n", - " s = \"[TIMEOUT]\"\n", - " print(f\" {s} {name}: http://{ip}:{port}\")\n", - "\n", - "# Check kafka-consumer container status\n", - "result = subprocess.run(\"docker inspect -f '{{.State.Status}}' kafka-consumer 2>/dev/null\",\n", - " shell=True, capture_output=True, text=True)\n", - "status = result.stdout.strip()\n", - "s = \"[OK]\" if status == \"running\" else \"[DOWN]\"\n", - "print(f\" {s} Kafka Consumer: {status or 'not found'}\")\n", - "\n", - "run_command(\"docker ps --format 'table {{.Names}}\\t{{.Status}}' | grep -E '(kafka|minio|NAMES)'\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Testing\n", - "\n", - "Test the deployment by uploading documents and videos, then querying via RAG.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Document Upload\n", - "\n", - "Upload a PDF document to MinIO, which triggers automatic ingestion via Kafka consumer.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.1 Upload to Storage\n", - "\n", - "Upload the document to MinIO object storage.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Sample documents are included in the repo under examples/rag_event_ingest/data/\n", - "pdf_path = os.path.join(DATA_DIR, \"documents\", \"Seahawks-Patriots in Super Bowl LX_ What We Learned from Seattle's 29-13 win.pdf\")\n", - "upload_file(pdf_path, \"Seahawks-Patriots_SuperBowl_LX_Analysis.pdf\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.2 Verify Document Ingestion" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check consumer logs to verify document processing status.\n", - "\n", - "The logs should show the document being picked up and successfully ingested:\n", - "```\n", - "services.document_indexer - INFO - Task ...: PENDING (0s)\n", - "services.document_indexer - INFO - Task ...: PENDING (5s)\n", - "handlers.base - INFO - [DocumentHandler] ✓ Seahawks-Patriots_SuperBowl_LX_Analysis.pdf → SUCCESS\n", - "consumer - INFO - ✓ SUMMARY: Seahawks-Patriots_SuperBowl_LX_Analysis.pdf | Collection: aidp_bucket | Duration: 12.76s | Status: SUCCESS\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Verify file landed in object storage\n", - "verify_file_in_storage(\"Seahawks-Patriots_SuperBowl_LX_Analysis.pdf\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.3 Verify Document Ingestion\n", - "\n", - "Check consumer logs to verify document processing status.\n", - "\n", - "The logs should show the document being picked up and successfully ingested:\n", - "```\n", - "services.document_indexer - INFO - Task ...: PENDING (0s)\n", - "services.document_indexer - INFO - Task ...: PENDING (5s)\n", - "handlers.base - INFO - [DocumentHandler] ✓ Seahawks-Patriots_SuperBowl_LX_Analysis.pdf → SUCCESS\n", - "consumer - INFO - ✓ SUMMARY: Seahawks-Patriots_SuperBowl_LX_Analysis.pdf | Collection: aidp_bucket | Duration: 12.76s | Status: SUCCESS\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Check consumer logs for ingestion status\n", - "print(\"Waiting for document processing...\")\n", - "get_consumer_logs(50)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.4 Query Document via RAG\n", - "\n", - "You can query the ingested document either **programmatically** below or via the **RAG Frontend UI**.\n", - "\n", - "> **💡 RAG Frontend**: Open `http://:8090` in your browser for an interactive Q&A interface.\n", - "> Make sure to select the collection **`aidp_bucket`** in the UI.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Query the document\n", - "await query_rag(\"What was the final score and who won Super Bowl LX?\", MINIO_COLLECTION)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Ask another question about the document.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Query about key takeaways\n", - "await query_rag(\"What were the key lessons learned from Seattle's victory in Super Bowl LX?\", MINIO_COLLECTION)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Video Upload\n", - "\n", - "Upload a video to MinIO, which triggers automatic ingestion via Kafka consumer → VSS for video analysis → RAG for indexing.\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.1 Upload to Storage\n", - "\n", - "Upload the video to MinIO object storage.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Sample videos are included in the repo under examples/rag_event_ingest/data/\n", - "video_path = os.path.join(DATA_DIR, \"videos\", \"Seattle Seahawks vs New England Patriots - Super Bowl LX Game Highlights.mp4\")\n", - "upload_file(video_path)\n", - "\n", - "print(\"\\nVideo processing takes longer than documents. Check consumer logs for progress.\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.2 Verify Video Ingestion\n", - "\n", - "Check consumer logs to verify video processing status.\n", - "\n", - "The logs should show the video being picked up and processed by VSS:\n", - "```\n", - "handlers.video - INFO - [VideoHandler] Processing video: Seattle Seahawks vs New England Patriots - Super Bowl LX Game Highlights.mp4\n", - "services.video_analyzer - INFO - Submitting video to VSS...\n", - "services.video_analyzer - INFO - VSS processing complete\n", - "handlers.base - INFO - [VideoHandler] ✓ Seattle Seahawks...mp4 → SUCCESS\n", - "consumer - INFO - ✓ SUMMARY: Seattle Seahawks...mp4 | Collection: aidp_bucket | Duration: ~120s | Status: SUCCESS\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Verify video landed in object storage\n", - "video_filename = os.path.basename(video_path)\n", - "verify_file_in_storage(video_filename)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.3 Verify Video Ingestion\n", - "\n", - "Check consumer logs to verify video processing status.\n", - "\n", - "The logs should show the video being picked up and processed by VSS:\n", - "```\n", - "handlers.video - INFO - [VideoHandler] Processing video: Seattle Seahawks vs New England Patriots - Super Bowl LX Game Highlights.mp4\n", - "services.video_analyzer - INFO - Submitting video to VSS...\n", - "services.video_analyzer - INFO - VSS processing complete\n", - "handlers.base - INFO - [VideoHandler] ✓ Seattle Seahawks...mp4 → SUCCESS\n", - "consumer - INFO - ✓ SUMMARY: Seattle Seahawks...mp4 | Collection: aidp_bucket | Duration: ~120s | Status: SUCCESS\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Check consumer logs for ingestion status\n", - "print(\"Waiting for video processing...\")\n", - "get_consumer_logs(50)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.4 Query Video via RAG\n", - "\n", - "Query the video content.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Query about the video content\n", - "await query_rag(\"Summarize the video content\", MINIO_COLLECTION)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Query about a specific time range in the video.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Query about specific time range\n", - "await query_rag(\"What happened between 15:00 and 20:00?\", MINIO_COLLECTION)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Additional query: analyze key defensive plays and turnovers.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Defensive Analysis\n", - "await query_rag(\"Describe the key defensive plays and turnovers that impacted the game outcome.\", MINIO_COLLECTION)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Additional query: identify critical momentum-changing plays in the second half.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Momentum Shifts\n", - "await query_rag(\"What were the critical momentum-changing plays in the second half of the game?\", MINIO_COLLECTION)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.5 Delete Video and Verify Removal from RAG\n", - "\n", - "Delete the video from MinIO and confirm the ingested content is removed from the RAG collection." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Delete video from MinIO bucket\n", - "video_filename = os.path.basename(video_path)\n", - "try:\n", - " client = get_minio_client()\n", - " client.remove_object(MINIO_BUCKET, video_filename)\n", - " print(f\"[OK] Deleted '{video_filename}' from MinIO bucket '{MINIO_BUCKET}'\")\n", - "except S3Error as e:\n", - " print(f\"[ERROR] Failed to delete: {e}\")\n", - "\n", - "# Wait for Kafka consumer to process the delete event\n", - "print(\"Waiting 15s for consumer to process delete event...\")\n", - "time.sleep(15)\n", - "get_consumer_logs(20)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Verify the video description document was removed from the ingestor\n", - "video_basename = os.path.basename(video_path)\n", - "desc_filename = f\"{video_basename}_description.json\"\n", - "\n", - "resp = requests.get(f\"{INGESTOR_URL}/documents\", params={\"collection_name\": MINIO_COLLECTION}, timeout=30)\n", - "if resp.status_code == 200:\n", - " docs = resp.json().get(\"documents\", [])\n", - " video_docs = [d for d in docs if desc_filename in str(d)]\n", - " if not video_docs:\n", - " print(f\"[OK] '{desc_filename}' removed from collection '{MINIO_COLLECTION}'\")\n", - " else:\n", - " print(f\"[WARN] '{desc_filename}' still found in collection\")\n", - "else:\n", - " print(f\"[INFO] Could not list documents (status {resp.status_code}), checking via query instead\")\n", - "\n", - "print(\"\\nQuerying RAG (answers may come from the PDF document, not the deleted video):\")\n", - "await query_rag(\"Summarize the video content\", MINIO_COLLECTION)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Clean Up\n", - "\n", - "Stop all services and clean up ingested data.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Stop RAG Deployment\n", - "\n", - "Stop all RAG services (NIMs, Milvus, Ingestor, RAG server).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "os.chdir(RAG_REPO_DIR)\n", - "for f in [\n", - " \"deploy/compose/docker-compose-rag-server.yaml\",\n", - " \"deploy/compose/docker-compose-ingestor-server.yaml\",\n", - " \"deploy/compose/vectordb.yaml\",\n", - " \"deploy/compose/nims.yaml\",\n", - "]:\n", - " run_command(f\"docker compose -f {f} down\")\n", - "print(\"[OK] RAG stopped\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Stop VSS Deployment\n", - "\n", - "Stop all VSS services (NIMs, VLM, via-server).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vss_deploy_dir = f\"{VSS_DIR}/deploy/docker/local_deployment_single_gpu\"\n", - "if os.path.exists(vss_deploy_dir):\n", - " subprocess.run(f\"cd {vss_deploy_dir} && set -a && source .env 2>/dev/null && set +a && docker compose down\",\n", - " shell=True, executable=\"/bin/bash\", capture_output=True)\n", - "subprocess.run(\"docker rm -f vss-llm 2>/dev/null\", shell=True, capture_output=True)\n", - "print(\"[OK] VSS stopped\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Stop Continuous ingestion Deployment\n", - "\n", - "Stop Continuous ingestion services (Kafka, MinIO, Consumer).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "run_command(f\"docker compose -f {AIDP_COMPOSE_FILE} down\")\n", - "print(\"[OK] Continuous ingestion stopped\")\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Document Continuous Ingestion from Object Storage\n", + "\n", + "## Purpose\n", + "\n", + "This notebook demonstrates an **automated document ingestion pipeline** that:\n", + "\n", + "1. Monitors emulated object storage for new uploads via Kafka events\n", + "2. Routes documents to appropriate AI services for indexing\n", + "5. Enables RAG Agent for semantic search and contextual Q&A over all ingested content\n", + "\n", + "## What Gets Deployed\n", + "\n", + "1. **NVIDIA RAG** - Document indexing, vector search, and AI-powered Q&A (NIMs, Milvus, Ingestor)\n", + "2. **Continuous Ingestion** - Event-driven ingestion pipeline (Kafka, MinIO, Consumer)\n" + ] }, - "nbformat": 4, - "nbformat_minor": 4 + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "\n", + "### Hardware\n", + "- **GPU**: 2x RTX PRO 6000 Blackwell or 2x H100\n", + "\n", + "#### Default GPU Assignment\n", + "\n", + "| GPU | Service |\n", + "|-----|---------|\n", + "| 0 | RAG NIMs (Embedding, Reranker) |\n", + "| 1 | RAG LLM NIM (Llama-3.3-Nemotron-Super-49B) |\n", + "\n", + "\n", + "### Software (pre-installed required)\n", + "- Ubuntu 22.04 or later\n", + "- Docker 24.0+ with Docker Compose v2\n", + "- NVIDIA Driver 570+\n", + "- NVIDIA Container Toolkit\n", + "\n", + "### API Keys\n", + "\n", + "\n", + "\n", + "\n", + "
KeyPurposeHow to Get
NGC_API_KEYDocker login, NIM deploymentsNGC Portal → Generate API Key
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Table of Contents\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SectionDescription
SetupClone repo, install deps, set API keys, load helpers
Deploy RAGNIMs, Vector DB, Ingestor, RAG Server
Deploy Continuous IngestionKafka, MinIO, Consumer
TestingUpload documents, query RAG
Clean UpStop services, clean data
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## References\n", + "\n", + "- **RAG Blueprint**: [NVIDIA RAG Documentation](https://github.com/NVIDIA-AI-Blueprints/rag/blob/develop/docs/deploy-docker-self-hosted.md)\n", + "- **NIM**: [NVIDIA NIM Documentation](https://docs.nvidia.com/nim/index.html)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup\n", + "\n", + "Clone the repository, configure API keys, and load helper functions.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Clone Repository\n", + "\n", + "Clone the RAG Blueprint repo to `~/rag`. This includes the consumer source code, deploy configs, and sample test data.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess, sys, os, shutil\n", + "\n", + "RAG_REPO_DIR = os.path.expanduser(\"~/rag\")\n", + "RAG_REPO_URL = \"https://github.com/NVIDIA-AI-Blueprints/rag.git\"\n", + "\n", + "# Ensure git-lfs is installed before any LFS operations\n", + "if not shutil.which(\"git-lfs\"):\n", + " print(\"[INSTALLING] git-lfs...\")\n", + " subprocess.run(\"sudo apt-get update && sudo apt-get install -y git-lfs && git lfs install\", shell=True, check=True)\n", + "else:\n", + " print(\"[OK] git-lfs found\")\n", + "\n", + "# Clone from correct branch (skip if already exists)\n", + "if not os.path.exists(RAG_REPO_DIR):\n", + " subprocess.run(f\"git clone {RAG_REPO_URL} {RAG_REPO_DIR}\", shell=True, check=True)\n", + "else:\n", + " print(f\"[OK] RAG repo already exists: {RAG_REPO_DIR}\")\n", + "subprocess.run(\"git lfs pull\", shell=True, cwd=RAG_REPO_DIR, check=True)\n", + "\n", + "# Verify\n", + "for path in [\"deploy/compose\", \"examples/rag_event_ingest/kafka_consumer\", \"examples/rag_event_ingest/data\"]:\n", + " status = \"[OK]\" if os.path.exists(os.path.join(RAG_REPO_DIR, path)) else \"[MISSING]\"\n", + " print(f\" {status} {path}\")\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Install Dependencies\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! python3 -m ensurepip --upgrade" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Ensure pip is available (some minimal Python installs lack it)\n", + "subprocess.run([sys.executable, \"-m\", \"ensurepip\", \"--upgrade\"], capture_output=True)\n", + "\n", + "def check_install_system_pkg(cmd: str, install_cmd: str):\n", + " if shutil.which(cmd):\n", + " print(f\" [OK] {cmd} found\")\n", + " return True\n", + " print(f\" [INSTALLING] {cmd}...\")\n", + " result = subprocess.run(install_cmd, shell=True, capture_output=True, text=True)\n", + " if result.returncode == 0:\n", + " print(f\" [OK] {cmd} installed\")\n", + " return True\n", + " print(f\" [ERROR] Failed to install {cmd}. Please install manually: {install_cmd}\")\n", + " return False\n", + "\n", + "check_install_system_pkg(\"git\", \"sudo apt-get update && sudo apt-get install -y git\")\n", + "\n", + "# Install Python packages\n", + "subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"minio\", \"aiohttp\", \"requests\", \"python-dotenv\", \"pyyaml\"])\n", + "print(\"[OK] Ready\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Set API Keys\n", + "\n", + "Configure NGC API key for NIM deployments.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "\n", + "def set_api_key(env_var: str, prompt: str, required: bool = True):\n", + " if os.environ.get(env_var):\n", + " print(f\" [OK] {env_var} already set ({os.environ[env_var][:10]}...)\")\n", + " return True\n", + " key = getpass.getpass(prompt)\n", + " if key:\n", + " os.environ[env_var] = key\n", + " print(f\" [OK] {env_var} set\")\n", + " return True\n", + " if required:\n", + " print(f\" [ERROR] {env_var} is required\")\n", + " return False\n", + " print(f\" [SKIP] {env_var} (optional)\")\n", + " return True\n", + "\n", + "set_api_key(\"NGC_API_KEY\", \"Enter NGC_API_KEY (starts with 'nvapi-'): \", required=True)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Helper Functions\n", + "\n", + "Shared utilities for deployment, file upload, status checks, and RAG queries.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Install dependencies\n", + "import sys\n", + "!{sys.executable} -m pip install -q minio aiohttp requests python-dotenv" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os, sys, json, re, subprocess, time, socket, asyncio\n", + "import aiohttp, requests\n", + "from typing import List, Optional, Dict\n", + "\n", + "try:\n", + " from minio import Minio\n", + " from minio.error import S3Error\n", + "except ImportError:\n", + " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"minio\"])\n", + " from minio import Minio\n", + " from minio.error import S3Error\n", + "\n", + "# =============================================================================\n", + "# CONFIGURATION\n", + "# =============================================================================\n", + "\n", + "# Paths relative to RAG repo root\n", + "RAG_REPO_DIR = os.path.expanduser(\"~/rag\")\n", + "EXAMPLE_DIR = os.path.join(RAG_REPO_DIR, \"examples/rag_event_ingest\")\n", + "AIDP_COMPOSE_FILE = os.path.join(EXAMPLE_DIR, \"deploy/docker-compose.yaml\")\n", + "DATA_DIR = os.path.join(EXAMPLE_DIR, \"data\")\n", + "RAG_SERVER_URL = \"http://localhost:8081\"\n", + "INGESTOR_URL = \"http://localhost:8082\"\n", + "\n", + "LOCAL_NIM_CACHE = os.path.expanduser(\"~/.cache/nim\")\n", + "\n", + "MINIO_ENDPOINT = \"localhost:9201\"\n", + "MINIO_ACCESS_KEY = \"minioadmin\"\n", + "MINIO_SECRET_KEY = \"minioadmin\"\n", + "MINIO_BUCKET = \"aidp-bucket\"\n", + "MINIO_COLLECTION = \"aidp_bucket\"\n", + "MINIO_CONSOLE_PORT = 9211\n", + "\n", + "# =============================================================================\n", + "# SHARED UTILITIES\n", + "# =============================================================================\n", + "\n", + "def run_command(cmd: str, capture: bool = False) -> Optional[str]:\n", + " \"\"\"Execute a shell command and print it.\"\"\"\n", + " print(f\"$ {cmd}\")\n", + " result = subprocess.run(cmd, shell=True, capture_output=capture, text=True)\n", + " return result.stdout if capture else None\n", + "\n", + "def get_host_ip() -> str:\n", + " \"\"\"Get host IP address for external access URLs.\"\"\"\n", + " try:\n", + " s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n", + " s.connect((\"8.8.8.8\", 80))\n", + " ip = s.getsockname()[0]\n", + " s.close()\n", + " return ip\n", + " except OSError:\n", + " return \"localhost\"\n", + "\n", + "def get_minio_client() -> Minio:\n", + " \"\"\"Create MinIO client for AIDP bucket operations.\"\"\"\n", + " return Minio(MINIO_ENDPOINT, access_key=MINIO_ACCESS_KEY, secret_key=MINIO_SECRET_KEY, secure=False)\n", + "\n", + "def upload_file(local_path: str, object_name: Optional[str] = None) -> bool:\n", + " \"\"\"Upload a local file to MinIO AIDP bucket.\"\"\"\n", + " if not os.path.exists(local_path):\n", + " print(f\"[ERROR] File not found: {local_path}\")\n", + " return False\n", + " obj = object_name or os.path.basename(local_path)\n", + " try:\n", + " client = get_minio_client()\n", + " if not client.bucket_exists(MINIO_BUCKET):\n", + " client.make_bucket(MINIO_BUCKET)\n", + " client.fput_object(MINIO_BUCKET, obj, local_path)\n", + " print(f\"[OK] Uploaded: {obj}\")\n", + " return True\n", + " except S3Error as e:\n", + " print(f\"[ERROR] {e}\")\n", + " return False\n", + "\n", + "def verify_file_in_storage(object_name: str, bucket: str = MINIO_BUCKET) -> bool:\n", + " \"\"\"Check if a file exists in MinIO bucket and print verification status.\"\"\"\n", + " try:\n", + " client = get_minio_client()\n", + " stat = client.stat_object(bucket, object_name)\n", + " print(f\"[OK] File verified in storage:\")\n", + " print(f\" Bucket: {bucket}\")\n", + " print(f\" Object: {object_name}\")\n", + " print(f\" Size: {stat.size:,} bytes\")\n", + " print(f\" Modified: {stat.last_modified}\")\n", + " return True\n", + " except S3Error as e:\n", + " print(f\"[ERROR] File not found in storage: {object_name}\")\n", + " print(f\" Error: {e}\")\n", + " return False\n", + "\n", + "def get_consumer_logs(lines: int = 30) -> None:\n", + " \"\"\"Show recent Kafka consumer logs.\"\"\"\n", + " run_command(f\"docker logs kafka-consumer --tail {lines}\")\n", + "\n", + "async def query_rag(question: str, collection: str = None) -> Optional[str]:\n", + " \"\"\"Query RAG system and print the answer.\"\"\"\n", + " coll = collection or MINIO_COLLECTION\n", + " print(f\"Q: {question}\\nCollection: {coll}\\n\" + \"-\" * 40)\n", + "\n", + " payload = {\n", + " \"messages\": [{\"role\": \"user\", \"content\": question}],\n", + " \"use_knowledge_base\": True,\n", + " \"collection_names\": [coll],\n", + " }\n", + " try:\n", + " async with aiohttp.ClientSession() as session:\n", + " async with session.post(\n", + " f\"{RAG_SERVER_URL}/generate\", json=payload,\n", + " timeout=aiohttp.ClientTimeout(total=120),\n", + " ) as resp:\n", + " text = await resp.text()\n", + " # Parse SSE response: extract content from each \"data: {...}\" line\n", + " chunks = []\n", + " for line in text.split(\"\\n\"):\n", + " if not line.startswith(\"data: \") or line[6:] == \"[DONE]\":\n", + " continue\n", + " try:\n", + " msg = json.loads(line[6:]).get(\"choices\", [{}])[0].get(\"message\", {})\n", + " if msg.get(\"content\"):\n", + " chunks.append(msg[\"content\"])\n", + " except json.JSONDecodeError:\n", + " pass\n", + " answer = \"\".join(chunks)\n", + " print(f\"Answer: {answer}\")\n", + " return answer\n", + " except aiohttp.ClientError as e:\n", + " print(f\"[ERROR] {e}\")\n", + " return None\n", + "\n", + "print(f\"[OK] Helpers loaded | Host IP: {get_host_ip()}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deploy NVIDIA RAG\n", + "\n", + "Deploy the NVIDIA RAG: NIMs (LLM, Embedding, Reranker), Milvus vector database, Ingestor server, and RAG server.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ngc_key = os.environ.get(\"NGC_API_KEY\")\n", + "if not ngc_key:\n", + " raise RuntimeError(\"NGC_API_KEY not set! Run the API keys cell first.\")\n", + "\n", + "os.chdir(RAG_REPO_DIR)\n", + "\n", + "# Set env vars needed by docker compose\n", + "os.environ[\"NGC_API_KEY\"] = ngc_key\n", + "os.environ[\"USERID\"] = f\"{os.getuid()}:{os.getgid()}\"\n", + "os.environ[\"COLLECTION_NAME\"] = MINIO_COLLECTION\n", + "\n", + "# Load RAG .env defaults (MODEL_DIRECTORY, etc.)\n", + "from dotenv import load_dotenv\n", + "env_file = os.path.join(RAG_REPO_DIR, \"deploy/compose/.env\")\n", + "if os.path.exists(env_file):\n", + " load_dotenv(env_file, override=False)\n", + "\n", + "# Login to nvcr.io\n", + "subprocess.run(f\"echo {ngc_key} | docker login nvcr.io -u '$oauthtoken' --password-stdin\",\n", + " shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + "\n", + "# Deploy components\n", + "for label, compose_file in [\n", + " (\"NIMs\", \"deploy/compose/nims.yaml\"),\n", + " (\"Vector DB\", \"deploy/compose/vectordb.yaml\"),\n", + "]:\n", + " print(f\"Deploying {label}...\")\n", + " run_command(f\"USERID=$(id -u) docker compose -f {compose_file} up -d\")\n", + "\n", + "print(\"Waiting 30s for Milvus...\")\n", + "time.sleep(30)\n", + "\n", + "for label, compose_file in [\n", + " (\"Ingestor\", \"deploy/compose/docker-compose-ingestor-server.yaml\"),\n", + " (\"RAG Server\", \"deploy/compose/docker-compose-rag-server.yaml\"),\n", + "]:\n", + " print(f\"Deploying {label}...\")\n", + " run_command(f\"docker compose -f {compose_file} up -d\")\n", + "\n", + "ip = get_host_ip()\n", + "print(f\"\\nRAG deployed: http://{ip}:8081 (server) | http://{ip}:8082 (ingestor) | http://{ip}:8090 (UI)\")\n", + "print(f\"COLLECTION_NAME: {MINIO_COLLECTION}\")\n", + "print(\"Wait ~10 minutes for NIMs to load models, then run the status check cell.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify RAG services are healthy. Wait ~10 minutes for NIMs to load models.\n", + "\n", + "The deployment status should be:\n", + "```\n", + "NAMES STATUS\n", + "rag-frontend Up About a minute\n", + "rag-server Up About a minute\n", + "ingestor-server Up About a minute\n", + "milvus-standalone Up 2 minutes (healthy)\n", + "milvus-etcd Up 2 minutes (healthy)\n", + "milvus-minio Up 2 minutes (healthy)\n", + "nim-llm-ms Up 2 minutes (healthy)\n", + "nemotron-embedding-ms Up 2 minutes (healthy)\n", + "nemotron-ranking-ms Up 2 minutes (healthy)\n", + "```\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check service status and print access URLs\n", + "print(\"Wait ~10 minutes for services to become healthy.\")\n", + "print(\"Run this cell again after waiting.\\n\")\n", + "\n", + "ip = get_host_ip()\n", + "for name, port, path in [\n", + " (\"RAG Server\", 8081, \"/health\"), (\"Ingestor\", 8082, \"/health\"),\n", + " (\"Frontend\", 8090, \"/\"), (\"Milvus\", 19530, \"/v1/vector/collections\"),\n", + "]:\n", + " try:\n", + " s = \"[OK]\" if requests.get(f\"http://localhost:{port}{path}\", timeout=10).status_code == 200 else \"[WARN]\"\n", + " except requests.ConnectionError:\n", + " s = \"[DOWN]\"\n", + " except requests.Timeout:\n", + " s = \"[TIMEOUT]\"\n", + " print(f\" {s} {name}: http://{ip}:{port}\")\n", + "run_command(\"docker ps --format 'table {{.Names}}\\t{{.Status}}' | grep -E '(rag|milvus|ingestor|nim|nemotron|NAMES)'\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deploy Continuous Ingestion from emulated object storage\n", + "\n", + "Deploy the Continuous Ingestion: Kafka message broker, MinIO object storage, and Kafka consumer for automated ingestion.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Deploy Services\n", + "\n", + "Deploy Kafka, MinIO, and the Kafka consumer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Verify prerequisites\n", + "net_check = subprocess.run(\"docker network inspect nvidia-rag\", shell=True, capture_output=True)\n", + "if net_check.returncode != 0:\n", + " raise RuntimeError(\"nvidia-rag network not found. Deploy RAG first.\")\n", + "\n", + "ngc_key = os.environ.get(\"NGC_API_KEY\", \"\")\n", + "if not ngc_key:\n", + " raise RuntimeError(\"NGC_API_KEY not set!\")\n", + "\n", + "host_ip = get_host_ip()\n", + "\n", + "# Set environment variables for docker compose\n", + "os.environ[\"HOST_IP\"] = host_ip\n", + "\n", + "# Login + pull + build\n", + "subprocess.run(f\"echo {ngc_key} | docker login nvcr.io -u '$oauthtoken' --password-stdin\",\n", + " shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + "\n", + "compose = f\"docker compose -f {AIDP_COMPOSE_FILE}\"\n", + "subprocess.run(f\"{compose} pull --ignore-pull-failures\", shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + "subprocess.run(f\"{compose} up -d --build\", shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + "\n", + "print(f\"Continuous Ingestion deployed:\")\n", + "print(f\" Kafka UI: http://{host_ip}:8080\")\n", + "print(f\" MinIO Console: http://{host_ip}:{MINIO_CONSOLE_PORT}\")\n", + "print(f\" Credentials: minioadmin / minioadmin\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify continuous ingestion services are running.\n", + "\n", + "The deployment status should be:\n", + "```\n", + "NAMES STATUS\n", + "kafka-consumer Up About a minute\n", + "aidp-kafka-ui Up About a minute\n", + "aidp-minio-mc Up About a minute\n", + "aidp-minio Up About a minute (healthy)\n", + "kafka Up About a minute (healthy)\n", + "```\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check service status and print access URLs\n", + "ip = get_host_ip()\n", + "for name, port, path in [\n", + " (\"Kafka UI\", 8080, \"/\"),\n", + " (\"MinIO Console\", MINIO_CONSOLE_PORT, \"/\"),\n", + "]:\n", + " try:\n", + " s = \"[OK]\" if requests.get(f\"http://localhost:{port}{path}\", timeout=10).status_code == 200 else \"[WARN]\"\n", + " except requests.ConnectionError:\n", + " s = \"[DOWN]\"\n", + " except requests.Timeout:\n", + " s = \"[TIMEOUT]\"\n", + " print(f\" {s} {name}: http://{ip}:{port}\")\n", + "\n", + "# Check kafka-consumer container status\n", + "result = subprocess.run(\"docker inspect -f '{{.State.Status}}' kafka-consumer 2>/dev/null\",\n", + " shell=True, capture_output=True, text=True)\n", + "status = result.stdout.strip()\n", + "s = \"[OK]\" if status == \"running\" else \"[DOWN]\"\n", + "print(f\" {s} Kafka Consumer: {status or 'not found'}\")\n", + "\n", + "run_command(\"docker ps --format 'table {{.Names}}\\t{{.Status}}' | grep -E '(kafka|minio|NAMES)'\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Testing\n", + "\n", + "Test the deployment by uploading documents, then querying via RAG.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Document Upload\n", + "\n", + "Upload a PDF document to MinIO, which triggers automatic ingestion via Kafka consumer.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.1 Upload to Storage\n", + "\n", + "Upload the document to MinIO object storage.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Sample documents are included in the repo under examples/rag_event_ingest/data/\n", + "pdf_path = os.path.join(DATA_DIR, \"documents\", \"Seahawks-Patriots in Super Bowl LX_ What We Learned from Seattle's 29-13 win.pdf\")\n", + "upload_file(pdf_path, \"Seahawks-Patriots_SuperBowl_LX_Analysis.pdf\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.2 Verify Document Ingestion" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check consumer logs to verify document processing status.\n", + "\n", + "The logs should show the document being picked up and successfully ingested:\n", + "```\n", + "services.document_indexer - INFO - Task ...: PENDING (0s)\n", + "services.document_indexer - INFO - Task ...: PENDING (5s)\n", + "handlers.base - INFO - [DocumentHandler] ✓ Seahawks-Patriots_SuperBowl_LX_Analysis.pdf → SUCCESS\n", + "consumer - INFO - ✓ SUMMARY: Seahawks-Patriots_SuperBowl_LX_Analysis.pdf | Collection: aidp_bucket | Duration: 12.76s | Status: SUCCESS\n", + "```\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Verify file landed in object storage\n", + "verify_file_in_storage(\"Seahawks-Patriots_SuperBowl_LX_Analysis.pdf\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.3 Verify Document Ingestion\n", + "\n", + "Check consumer logs to verify document processing status.\n", + "\n", + "The logs should show the document being picked up and successfully ingested:\n", + "```\n", + "services.document_indexer - INFO - Task ...: PENDING (0s)\n", + "services.document_indexer - INFO - Task ...: PENDING (5s)\n", + "handlers.base - INFO - [DocumentHandler] ✓ Seahawks-Patriots_SuperBowl_LX_Analysis.pdf → SUCCESS\n", + "consumer - INFO - ✓ SUMMARY: Seahawks-Patriots_SuperBowl_LX_Analysis.pdf | Collection: aidp_bucket | Duration: 12.76s | Status: SUCCESS\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check consumer logs for ingestion status\n", + "print(\"Waiting for document processing...\")\n", + "get_consumer_logs(50)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.4 Query Document via RAG\n", + "\n", + "You can query the ingested document either **programmatically** below or via the **RAG Frontend UI**.\n", + "\n", + "> **💡 RAG Frontend**: Open `http://:8090` in your browser for an interactive Q&A interface.\n", + "> Make sure to select the collection **`aidp_bucket`** in the UI.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Query the document\n", + "await query_rag(\"What was the final score and who won Super Bowl LX?\", MINIO_COLLECTION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ask another question about the document.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Query about key takeaways\n", + "await query_rag(\"What were the key lessons learned from Seattle's victory in Super Bowl LX?\", MINIO_COLLECTION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Clean Up\n", + "\n", + "Stop all services and clean up ingested data.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Stop RAG Deployment\n", + "\n", + "Stop all RAG services (NIMs, Milvus, Ingestor, RAG server).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "os.chdir(RAG_REPO_DIR)\n", + "for f in [\n", + " \"deploy/compose/docker-compose-rag-server.yaml\",\n", + " \"deploy/compose/docker-compose-ingestor-server.yaml\",\n", + " \"deploy/compose/vectordb.yaml\",\n", + " \"deploy/compose/nims.yaml\",\n", + "]:\n", + " run_command(f\"docker compose -f {f} down\")\n", + "print(\"[OK] RAG stopped\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Stop Continuous ingestion Deployment\n", + "\n", + "Stop Continuous ingestion services (Kafka, MinIO, Consumer).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "run_command(f\"docker compose -f {AIDP_COMPOSE_FILE} down\")\n", + "print(\"[OK] Continuous ingestion stopped\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 } From a9dc0f26c85d598510ba107331270139fd903da3 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Tue, 10 Mar 2026 12:01:04 -0500 Subject: [PATCH 37/81] added metadata fix --- src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py index 7dc7fa847..47e03f680 100644 --- a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py +++ b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py @@ -136,15 +136,17 @@ def similarity_search_by_vector_with_relevance_scores( results = cursor.fetchall() for result in results: - source = json.loads(result[2]) if result[2] else {} - metadata = {} + # parse source data + metadata = json.loads(result[2]) if result[2] else {} + # parse content_metadata + content_metadata = {} if isinstance(result[3], oracledb.LOB) and result[3]: - metadata = json.loads(self._get_clob_value(result[3])) + content_metadata = json.loads(self._get_clob_value(result[3])) elif isinstance(result[3], dict) and result[3]: - metadata = result[3] + content_metadata = result[3] # combining source data and metadata - metadata.update(source) + metadata['content_metadata'] = content_metadata # Apply filter if provided if filter: From affdc409bde65739ebb1283658bce422ceb202e6 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Tue, 10 Mar 2026 12:34:45 -0500 Subject: [PATCH 38/81] added versioning for main --- .github/workflows/build-push-main.yml | 10 ++++++---- uv.lock | 2 -- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-push-main.yml b/.github/workflows/build-push-main.yml index d3b671b2f..5f01ea666 100644 --- a/.github/workflows/build-push-main.yml +++ b/.github/workflows/build-push-main.yml @@ -17,6 +17,7 @@ jobs: OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }} # IMAGE VARS RAG_IMAGE_NAME: corrino-devops-repository/nvidia-rag-retrieval-oci + RAG_IMAGE_VERSION: 0.0.3 steps: - name: Checkout code uses: actions/checkout@v3 @@ -44,8 +45,8 @@ jobs: env: IMAGE_REMOTE: "iad.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.RAG_IMAGE_NAME }}" run: | - docker tag "${{ env.RAG_IMAGE_NAME }}":latest "${{ env.IMAGE_REMOTE }}:latest" - docker push "${{ env.IMAGE_REMOTE }}:latest" + docker tag "${{ env.RAG_IMAGE_NAME }}":"${{ env.RAG_IMAGE_VERSION }}" "${{ env.IMAGE_REMOTE }}":"${{ env.RAG_IMAGE_VERSION }}" + docker push "${{ env.IMAGE_REMOTE }}":"${{ env.RAG_IMAGE_VERSION }}" build-and-push-ingest: runs-on: ubuntu-latest @@ -58,6 +59,7 @@ jobs: OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }} # IMAGE VARS INGEST_IMAGE_NAME: corrino-devops-repository/nvidia-rag-ingestion-oci + INGEST_IMAGE_VERSION: 0.0.3 steps: - name: Checkout code uses: actions/checkout@v3 @@ -85,5 +87,5 @@ jobs: env: IMAGE_REMOTE: "iad.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.INGEST_IMAGE_NAME }}" run: | - docker tag "${{ env.INGEST_IMAGE_NAME }}":latest "${{ env.IMAGE_REMOTE }}:latest" - docker push "${{ env.IMAGE_REMOTE }}:latest" \ No newline at end of file + docker tag "${{ env.INGEST_IMAGE_NAME }}":"${{ env.INGEST_IMAGE_VERSION }}" "${{ env.IMAGE_REMOTE }}:"${{ env.INGEST_IMAGE_VERSION }}" + docker push "${{ env.IMAGE_REMOTE }}:"${{ env.INGEST_IMAGE_VERSION }}" \ No newline at end of file diff --git a/uv.lock b/uv.lock index 2a1f0d61d..2823fa268 100644 --- a/uv.lock +++ b/uv.lock @@ -1758,7 +1758,6 @@ dependencies = [ { name = "langchain-nvidia-ai-endpoints" }, { name = "lark" }, { name = "minio" }, - { name = "nv-ingest-client" }, { name = "oracledb" }, { name = "pdfplumber" }, { name = "protobuf" }, @@ -1880,7 +1879,6 @@ requires-dist = [ { name = "minio", specifier = ">=7.2,<8.0" }, { name = "nv-ingest-api", marker = "extra == 'all'", specifier = "==25.9.0" }, { name = "nv-ingest-api", marker = "extra == 'ingest'", specifier = "==25.9.0" }, - { name = "nv-ingest-client", specifier = "==25.9.0" }, { name = "nv-ingest-client", marker = "extra == 'all'", specifier = "==25.9.0" }, { name = "nv-ingest-client", marker = "extra == 'ingest'", specifier = "==25.9.0" }, { name = "opentelemetry-api", marker = "extra == 'all'", specifier = ">=1.29,<2.0" }, From 89276172dba05c05cc8b67736f21ead5b8e7caea Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Tue, 10 Mar 2026 12:53:22 -0500 Subject: [PATCH 39/81] fixing login ocir --- .github/workflows/build-push-pr.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build-push-pr.yml b/.github/workflows/build-push-pr.yml index 2a8b7d656..3e49edd99 100644 --- a/.github/workflows/build-push-pr.yml +++ b/.github/workflows/build-push-pr.yml @@ -31,8 +31,6 @@ jobs: id: login-ocir with: auth_token: ${{ secrets.OCI_AUTH_TOKEN }} - username: ${{ secrets.OCI_CLI_USER }} - tenancy_namespace: ${{ secrets.OCI_TENANCY_NAMESPACE }} - name: Compute short SHA id: short-sha @@ -74,8 +72,6 @@ jobs: id: login-ocir with: auth_token: ${{ secrets.OCI_AUTH_TOKEN }} - username: ${{ secrets.OCI_CLI_USER }} - tenancy_namespace: ${{ secrets.OCI_TENANCY_NAMESPACE }} - name: Compute short SHA id: short-sha From e629fd1e448c66f879aaf05bfaf81d250033779e Mon Sep 17 00:00:00 2001 From: Kurt Heiss Date: Tue, 10 Mar 2026 10:58:47 -0700 Subject: [PATCH 40/81] Kheiss/chunking topic (#417) * confirming presence of switcher text in conf.py file * Added chunking information --- docs/accuracy_perf.md | 2 +- docs/api-ingestor.md | 2 +- docs/api-rag.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/accuracy_perf.md b/docs/accuracy_perf.md index fa4f6b7a7..c24f0c9a5 100644 --- a/docs/accuracy_perf.md +++ b/docs/accuracy_perf.md @@ -14,7 +14,7 @@ Change the setting if you want different behavior. | Name | Default | Description | Advantages | Disadvantages | |----------------------|------------|---------------------|----------------------|--------------------------| | `APP_NVINGEST_CHUNKOVERLAP` | `150` | Increase overlap to ensure smooth transitions between chunks. | - Larger overlap provides smoother transitions between chunks.
| - Might increase processing overhead.
| -| `APP_NVINGEST_CHUNKSIZE` | `512` | Increase chunk size for more context. | - Larger chunks retain more context, improving coherence.
| - Larger chunks increase embedding size, slowing retrieval.
- Longer chunks might increase latency due to larger prompt size.
| +| `APP_NVINGEST_CHUNKSIZE` | `512` | Increase chunk size for more context. | - Larger chunks retain more context, improving coherence.
- Larger chunks increase compute time for embedding creation.
- Larger chunks can lead to longer retrieved context, increasing generation latency.
- Very large chunks may dilute semantic focus, reducing embedding precision.
| | `APP_NVINGEST_ENABLEPDFSPLITTER` | `true` | Set to `true` to perform chunk-based splitting of pdfs after the default page-level extraction occurs. Recommended for PDFs that are mostly text content. | - Provides more granular content segmentation.
| - Can increase the number of chunks and slow down the ingestion process.
| | `APP_NVINGEST_EXTRACTCHARTS` | `true` | Set to `true` to extract charts. | - Improves accuracy for documents that contain charts.
| - Increases ingestion time.
| | `APP_NVINGEST_EXTRACTIMAGES` | `false` | Set to `true` to enable image captioning during ingestion. For details, refer to [Image Captioning Support](image_captioning.md). | - Enhances multimodal retrieval accuracy for documents having images.
| - Increased processing time during ingestion.
- Requires additional GPU resources for VLM model deployment.
| diff --git a/docs/api-ingestor.md b/docs/api-ingestor.md index 443a521a6..548841574 100644 --- a/docs/api-ingestor.md +++ b/docs/api-ingestor.md @@ -8,7 +8,7 @@ This documentation contains the OpenAPI reference for the ingestor server. :::{tip} -To view this documentation on docs.nvidia.com, go to https://docs.nvidia.com/rag/latest/api-ingestor.html. +To view this documentation on docs.nvidia.com, browse to [https://docs.nvidia.com/rag/latest/api-ingestor](https://docs.nvidia.com/rag/latest/api-ingestor.html). ::: diff --git a/docs/api-rag.md b/docs/api-rag.md index 366d44b0f..518696304 100644 --- a/docs/api-rag.md +++ b/docs/api-rag.md @@ -8,7 +8,7 @@ This documentation contains the OpenAPI reference for the RAG server. :::{tip} -To view this documentation on docs.nvidia.com, go to https://docs.nvidia.com/rag/latest/api-rag.html. +To view this documentation on docs.nvidia.com, browse to [https://docs.nvidia.com/rag/latest/api-rag](https://docs.nvidia.com/rag/latest/api-rag.html). ::: From 0db1bf23608047e73ded3dd7ea6895c4c7687c30 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Tue, 10 Mar 2026 12:59:58 -0500 Subject: [PATCH 41/81] hard coded --- .github/workflows/build-push-pr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-push-pr.yml b/.github/workflows/build-push-pr.yml index 3e49edd99..b576cf59f 100644 --- a/.github/workflows/build-push-pr.yml +++ b/.github/workflows/build-push-pr.yml @@ -42,7 +42,7 @@ jobs: - name: Tag and Push RAG Docker Image env: - IMAGE_REMOTE: "iad.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.RAG_IMAGE_NAME }}" + IMAGE_REMOTE: "iad.ocir.io/iduyx1qnmway/${{ env.RAG_IMAGE_NAME }}" run: | docker tag "${{ env.RAG_IMAGE_NAME }}:pr-${{ steps.short-sha.outputs.value }}" "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" docker push "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" @@ -83,7 +83,7 @@ jobs: - name: Tag and Push Ingest Docker Image env: - IMAGE_REMOTE: "iad.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.INGEST_IMAGE_NAME }}" + IMAGE_REMOTE: "iad.ocir.io/iduyx1qnmway/${{ env.INGEST_IMAGE_NAME }}" run: | docker tag "${{ env.INGEST_IMAGE_NAME }}:pr-${{ steps.short-sha.outputs.value }}" "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" docker push "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" From 5e83bb827573d42e75404add0e7265a2865130e5 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Tue, 10 Mar 2026 14:02:31 -0500 Subject: [PATCH 42/81] changed regions --- .github/workflows/build-push-pr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-push-pr.yml b/.github/workflows/build-push-pr.yml index b576cf59f..8bf9ef332 100644 --- a/.github/workflows/build-push-pr.yml +++ b/.github/workflows/build-push-pr.yml @@ -42,7 +42,7 @@ jobs: - name: Tag and Push RAG Docker Image env: - IMAGE_REMOTE: "iad.ocir.io/iduyx1qnmway/${{ env.RAG_IMAGE_NAME }}" + IMAGE_REMOTE: "ord.ocir.io/iduyx1qnmway/${{ env.RAG_IMAGE_NAME }}" run: | docker tag "${{ env.RAG_IMAGE_NAME }}:pr-${{ steps.short-sha.outputs.value }}" "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" docker push "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" @@ -83,7 +83,7 @@ jobs: - name: Tag and Push Ingest Docker Image env: - IMAGE_REMOTE: "iad.ocir.io/iduyx1qnmway/${{ env.INGEST_IMAGE_NAME }}" + IMAGE_REMOTE: "ord.ocir.io/iduyx1qnmway/${{ env.INGEST_IMAGE_NAME }}" run: | docker tag "${{ env.INGEST_IMAGE_NAME }}:pr-${{ steps.short-sha.outputs.value }}" "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" docker push "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" From 206049644ee8cb761a10b72da30b5a287b4c3b50 Mon Sep 17 00:00:00 2001 From: Swapnil Masurekar Date: Wed, 11 Mar 2026 00:47:18 +0530 Subject: [PATCH 43/81] CI - Updated Nemotron endpoints for embedding, page-elements, graphic-elements, table-structure (#410) Signed-off-by: Swapnil Masurekar Co-authored-by: Shubhadeep Das <149712532+shubhadeepd@users.noreply.github.com> --- deploy/compose/nvdev.env | 8 ++++---- tests/integration/notebook_test_config.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/compose/nvdev.env b/deploy/compose/nvdev.env index b92e4500b..da3725dc7 100644 --- a/deploy/compose/nvdev.env +++ b/deploy/compose/nvdev.env @@ -20,7 +20,7 @@ export APP_LLM_MODELNAME=nvidia/llama-3.3-nemotron-super-49b-v1.5 # export APP_LLM_MODELNAME=nvidia/nemotron-3-nano-30b-a3b # Note: For locally deployed nemotron-3-nano, use: nvidia/nemotron-3-nano export APP_FILTEREXPRESSIONGENERATOR_MODELNAME=nvidia/llama-3.3-nemotron-super-49b-v1.5 -export APP_EMBEDDINGS_MODELNAME=nvdev/nvidia/llama-3.2-nv-embedqa-1b-v2 +export APP_EMBEDDINGS_MODELNAME=nvidia/llama-nemotron-embed-1b-v2 # For VLM Embedding Model (Nemoretriever-1b-vlm-embed-v1) # export APP_EMBEDDINGS_MODELNAME=nvdev/nvidia/llama-nemotron-embed-vl-1b-v2 export APP_RANKING_MODELNAME=nvidia/llama-3.2-nv-rerankqa-1b-v2 @@ -33,11 +33,11 @@ export APP_RANKING_SERVERURL="" export OCR_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr export OCR_INFER_PROTOCOL=http export OCR_MODEL_NAME=scene_text_ensemble -export YOLOX_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-page-elements-v3 +export YOLOX_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3 export YOLOX_INFER_PROTOCOL=http -export YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvdev/nvidia/nemoretriever-graphic-elements-v1 +export YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1 export YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL=http -export YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvdev/nvidia/nemoretriever-table-structure-v1 +export YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1 export YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL=http export SUMMARY_LLM="nvidia/llama-3.3-nemotron-super-49b-v1.5" export SUMMARY_LLM_SERVERURL="" diff --git a/tests/integration/notebook_test_config.yaml b/tests/integration/notebook_test_config.yaml index 83bc001d5..e0bcc3464 100644 --- a/tests/integration/notebook_test_config.yaml +++ b/tests/integration/notebook_test_config.yaml @@ -55,7 +55,7 @@ filter_expression_generator: # Embedding Configuration embeddings: - model_name: "nvidia/llama-3.2-nv-embedqa-1b-v2" # Model for generating text embeddings + model_name: "nvidia/llama-nemotron-embed-1b-v2" # Model for generating text embeddings dimensions: 2048 # Dimensionality of the embedding vectors server_url: "http://localhost:9080/v1" # URL endpoint for embedding service (on-prem NIM default) # api_key: "" # Optional: API key for embeddings (overrides NVIDIA_API_KEY environment variable) From 09109c147673cddeaaa9bd25bbee023f124c1d56 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Tue, 10 Mar 2026 14:21:00 -0500 Subject: [PATCH 44/81] combined GA jobs --- .github/workflows/build-push-main.yml | 56 +++++++-------------------- .github/workflows/build-push-pr.yml | 33 +--------------- 2 files changed, 15 insertions(+), 74 deletions(-) diff --git a/.github/workflows/build-push-main.yml b/.github/workflows/build-push-main.yml index 5f01ea666..9142ec590 100644 --- a/.github/workflows/build-push-main.yml +++ b/.github/workflows/build-push-main.yml @@ -16,8 +16,9 @@ jobs: OCI_CLI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }} OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }} # IMAGE VARS + INGEST_IMAGE_NAME: corrino-devops-repository/nvidia-rag-ingestion-oci RAG_IMAGE_NAME: corrino-devops-repository/nvidia-rag-retrieval-oci - RAG_IMAGE_VERSION: 0.0.3 + IMAGE_VERSION: 0.0.3 steps: - name: Checkout code uses: actions/checkout@v3 @@ -37,55 +38,24 @@ jobs: id: short-sha run: echo "value=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT - - name: Build Docker Image + - name: Build RAG Docker Image run: | docker build -t "${{ env.RAG_IMAGE_NAME }}":latest -f src/nvidia_rag/rag_server/Dockerfile . - - name: Tag and Push Docker Image + - name: Tag and Push RAG Docker Image env: - IMAGE_REMOTE: "iad.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.RAG_IMAGE_NAME }}" + IMAGE_REMOTE: "ord.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.RAG_IMAGE_NAME }}" run: | - docker tag "${{ env.RAG_IMAGE_NAME }}":"${{ env.RAG_IMAGE_VERSION }}" "${{ env.IMAGE_REMOTE }}":"${{ env.RAG_IMAGE_VERSION }}" - docker push "${{ env.IMAGE_REMOTE }}":"${{ env.RAG_IMAGE_VERSION }}" - - build-and-push-ingest: - runs-on: ubuntu-latest - env: - # OCI AUTH - OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }} - OCI_CLI_TENANCY: ${{ secrets.OCI_CLI_TENANCY }} - OCI_CLI_FINGERPRINT: ${{ secrets.OCI_CLI_FINGERPRINT }} - OCI_CLI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }} - OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }} - # IMAGE VARS - INGEST_IMAGE_NAME: corrino-devops-repository/nvidia-rag-ingestion-oci - INGEST_IMAGE_VERSION: 0.0.3 - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Log into OCIR - uses: oracle-actions/login-ocir@v1.3.0 - id: login-ocir - with: - auth_token: ${{ secrets.OCI_AUTH_TOKEN }} - - - name: Compute short SHA - id: short-sha - run: echo "value=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + docker tag "${{ env.RAG_IMAGE_NAME }}":"${{ env.IMAGE_VERSION }}" "${{ env.IMAGE_REMOTE }}":"${{ env.IMAGE_VERSION }}" + docker push "${{ env.IMAGE_REMOTE }}":"${{ env.IMAGE_VERSION }}" - - name: Build Docker Image + - name: Build Ingest Docker Image run: | - docker build -t "${{ env.INGEST_IMAGE_NAME }}":latest -f src/nvidia_rag/rag_server/Dockerfile . + docker build -t "${{ env.INGEST_IMAGE_NAME }}":latest -f src/nvidia_rag/ingestor_server/Dockerfile . - - name: Tag and Push Docker Image + - name: Tag and Push Ingest Docker Image env: - IMAGE_REMOTE: "iad.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.INGEST_IMAGE_NAME }}" + IMAGE_REMOTE: "ord.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.INGEST_IMAGE_NAME }}" run: | - docker tag "${{ env.INGEST_IMAGE_NAME }}":"${{ env.INGEST_IMAGE_VERSION }}" "${{ env.IMAGE_REMOTE }}:"${{ env.INGEST_IMAGE_VERSION }}" - docker push "${{ env.IMAGE_REMOTE }}:"${{ env.INGEST_IMAGE_VERSION }}" \ No newline at end of file + docker tag "${{ env.INGEST_IMAGE_NAME }}":"${{ env.IMAGE_VERSION }}" "${{ env.IMAGE_REMOTE }}:"${{ env.IMAGE_VERSION }}" + docker push "${{ env.IMAGE_REMOTE }}:"${{ env.IMAGE_VERSION }}" \ No newline at end of file diff --git a/.github/workflows/build-push-pr.yml b/.github/workflows/build-push-pr.yml index 8bf9ef332..c709a36ee 100644 --- a/.github/workflows/build-push-pr.yml +++ b/.github/workflows/build-push-pr.yml @@ -6,7 +6,7 @@ on: types: [synchronize] jobs: - build-and-push-rag: + build-and-push-images: runs-on: ubuntu-latesta env: # OCI AUTH @@ -17,6 +17,7 @@ jobs: OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }} # IMAGE VARS RAG_IMAGE_NAME: corrino-devops-repository/nvidia-rag-retrieval-oci + INGEST_IMAGE_NAME: corrino-devops-repository/nvidia-rag-ingestion-oci steps: - name: Checkout code uses: actions/checkout@v3 @@ -47,36 +48,6 @@ jobs: docker tag "${{ env.RAG_IMAGE_NAME }}:pr-${{ steps.short-sha.outputs.value }}" "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" docker push "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" - build-and-push-ingestion: - runs-on: ubuntu-latest - env: - # OCI AUTH - OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }} - OCI_CLI_TENANCY: ${{ secrets.OCI_CLI_TENANCY }} - OCI_CLI_FINGERPRINT: ${{ secrets.OCI_CLI_FINGERPRINT }} - OCI_CLI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }} - OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }} - # IMAGE VARS - INGEST_IMAGE_NAME: corrino-devops-repository/nvidia-rag-ingestion-oci - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Log into OCIR - uses: oracle-actions/login-ocir@v1.3.0 - id: login-ocir - with: - auth_token: ${{ secrets.OCI_AUTH_TOKEN }} - - - name: Compute short SHA - id: short-sha - run: echo "value=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT - - name: Build Ingest Docker Image run: | docker build -t "${{ env.INGEST_IMAGE_NAME }}":pr-${{ steps.short-sha.outputs.value }} -f src/nvidia_rag/ingestor_server/Dockerfile . From 4087f01dd20681a7eac99bfe52788dbc9b5671cf Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Tue, 10 Mar 2026 14:25:15 -0500 Subject: [PATCH 45/81] typo in latest --- .github/workflows/build-push-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-push-pr.yml b/.github/workflows/build-push-pr.yml index c709a36ee..ec516052c 100644 --- a/.github/workflows/build-push-pr.yml +++ b/.github/workflows/build-push-pr.yml @@ -7,7 +7,7 @@ on: jobs: build-and-push-images: - runs-on: ubuntu-latesta + runs-on: ubuntu-latest env: # OCI AUTH OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }} From 2cae7babc0a4cd60897d443b3e7e957ea40c480b Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Tue, 10 Mar 2026 14:43:45 -0500 Subject: [PATCH 46/81] improved versioning and fixed latest ref bug --- .github/workflows/build-push-main.yml | 17 ++++++++++------- .github/workflows/build-push-pr.yml | 6 +++--- VERSION | 1 + 3 files changed, 14 insertions(+), 10 deletions(-) create mode 100644 VERSION diff --git a/.github/workflows/build-push-main.yml b/.github/workflows/build-push-main.yml index 9142ec590..ba0b6e414 100644 --- a/.github/workflows/build-push-main.yml +++ b/.github/workflows/build-push-main.yml @@ -18,7 +18,6 @@ jobs: # IMAGE VARS INGEST_IMAGE_NAME: corrino-devops-repository/nvidia-rag-ingestion-oci RAG_IMAGE_NAME: corrino-devops-repository/nvidia-rag-retrieval-oci - IMAGE_VERSION: 0.0.3 steps: - name: Checkout code uses: actions/checkout@v3 @@ -38,24 +37,28 @@ jobs: id: short-sha run: echo "value=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + - name: Read version + id: version + run: echo "value=$(cat VERSION)" >> $GITHUB_OUTPUT + - name: Build RAG Docker Image run: | - docker build -t "${{ env.RAG_IMAGE_NAME }}":latest -f src/nvidia_rag/rag_server/Dockerfile . + docker build -t "${{ env.RAG_IMAGE_NAME }}":"${{ steps.version.outputs.value }}" -f src/nvidia_rag/rag_server/Dockerfile . - name: Tag and Push RAG Docker Image env: IMAGE_REMOTE: "ord.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.RAG_IMAGE_NAME }}" run: | - docker tag "${{ env.RAG_IMAGE_NAME }}":"${{ env.IMAGE_VERSION }}" "${{ env.IMAGE_REMOTE }}":"${{ env.IMAGE_VERSION }}" - docker push "${{ env.IMAGE_REMOTE }}":"${{ env.IMAGE_VERSION }}" + docker tag "${{ env.RAG_IMAGE_NAME }}":"${{ steps.version.outputs.value }}" "${{ env.IMAGE_REMOTE }}":"${{ steps.version.outputs.value }}" + docker push "${{ env.IMAGE_REMOTE }}":"${{ steps.version.outputs.value }}" - name: Build Ingest Docker Image run: | - docker build -t "${{ env.INGEST_IMAGE_NAME }}":latest -f src/nvidia_rag/ingestor_server/Dockerfile . + docker build -t "${{ env.INGEST_IMAGE_NAME }}":"${{ steps.version.outputs.value }}" -f src/nvidia_rag/ingestor_server/Dockerfile . - name: Tag and Push Ingest Docker Image env: IMAGE_REMOTE: "ord.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.INGEST_IMAGE_NAME }}" run: | - docker tag "${{ env.INGEST_IMAGE_NAME }}":"${{ env.IMAGE_VERSION }}" "${{ env.IMAGE_REMOTE }}:"${{ env.IMAGE_VERSION }}" - docker push "${{ env.IMAGE_REMOTE }}:"${{ env.IMAGE_VERSION }}" \ No newline at end of file + docker tag "${{ env.INGEST_IMAGE_NAME }}":"${{ steps.version.outputs.value }}" "${{ env.IMAGE_REMOTE }}":"${{ steps.version.outputs.value }}" + docker push "${{ env.IMAGE_REMOTE }}":"${{ steps.version.outputs.value }}" \ No newline at end of file diff --git a/.github/workflows/build-push-pr.yml b/.github/workflows/build-push-pr.yml index ec516052c..4ba07aa23 100644 --- a/.github/workflows/build-push-pr.yml +++ b/.github/workflows/build-push-pr.yml @@ -3,7 +3,7 @@ name: Build and Push on PR on: pull_request: - types: [synchronize] + types: [opened, synchronize, reopened] jobs: build-and-push-images: @@ -43,7 +43,7 @@ jobs: - name: Tag and Push RAG Docker Image env: - IMAGE_REMOTE: "ord.ocir.io/iduyx1qnmway/${{ env.RAG_IMAGE_NAME }}" + IMAGE_REMOTE: "ord.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.RAG_IMAGE_NAME }}" run: | docker tag "${{ env.RAG_IMAGE_NAME }}:pr-${{ steps.short-sha.outputs.value }}" "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" docker push "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" @@ -54,7 +54,7 @@ jobs: - name: Tag and Push Ingest Docker Image env: - IMAGE_REMOTE: "ord.ocir.io/iduyx1qnmway/${{ env.INGEST_IMAGE_NAME }}" + IMAGE_REMOTE: "ord.ocir.io/${{ secrets.OCI_TENANCY_NAMESPACE }}/${{ env.INGEST_IMAGE_NAME }}" run: | docker tag "${{ env.INGEST_IMAGE_NAME }}:pr-${{ steps.short-sha.outputs.value }}" "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" docker push "${{ env.IMAGE_REMOTE }}:pr-${{ steps.short-sha.outputs.value }}" diff --git a/VERSION b/VERSION new file mode 100644 index 000000000..f9cece58d --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +v0.0.3 From 51368c3c1562dffe3e794210c57f15ee3c1cabcb Mon Sep 17 00:00:00 2001 From: Shubhadeep Das <149712532+shubhadeepd@users.noreply.github.com> Date: Wed, 11 Mar 2026 01:59:02 +0530 Subject: [PATCH 47/81] Add notebook showcasing langchain connector for Nvidia RAG Retrieval (#419) * notebook: Add notebook showcasing langchain retriever connector * Update langchain connector version to 1.2.0 * Fix broken link in notebook doc --- docs/notebooks.md | 2 + examples/rag_react_agent/uv.lock | 9 +- notebooks/langchain_nvidia_retriever.ipynb | 277 +++++++++++++++++++++ pyproject.toml | 2 +- uv.lock | 9 +- 5 files changed, 290 insertions(+), 9 deletions(-) create mode 100644 notebooks/langchain_nvidia_retriever.ipynb diff --git a/docs/notebooks.md b/docs/notebooks.md index 7a7f50eee..6a56686a4 100644 --- a/docs/notebooks.md +++ b/docs/notebooks.md @@ -103,6 +103,8 @@ Use the following notebooks to learn comprehensive Python client usage, metadata - [rag_library_lite_usage.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/rag_library_lite_usage.ipynb) – Demonstrates containerless deployment of the NVIDIA RAG Python package in lite mode. Uses Milvus Lite (embedded vector database) and NV-Ingest subprocess mode for a simplified setup without Docker containers. Leverages NVIDIA cloud APIs for embeddings, ranking, and LLM inference. **Note**: This mode does not support image/table/chart citations or document summarization. +- [langchain_nvidia_retriever.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/langchain_nvidia_retriever.ipynb) – Showcases **LangChain integration** with the NVIDIA RAG Blueprint. Run [ingestion_api_usage.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/ingestion_api_usage.ipynb) first to ingest documents, then use `NVIDIARetriever` for retrieval (sync/async), custom parameters, error handling, and optional RAG chaining with `ChatNVIDIA`. + ## Advanced Notebooks diff --git a/examples/rag_react_agent/uv.lock b/examples/rag_react_agent/uv.lock index eee7b812b..16af3b48d 100644 --- a/examples/rag_react_agent/uv.lock +++ b/examples/rag_react_agent/uv.lock @@ -1384,16 +1384,17 @@ wheels = [ [[package]] name = "langchain-nvidia-ai-endpoints" -version = "1.1.0" +version = "1.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, { name = "filetype" }, { name = "langchain-core" }, + { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4e/d1/f72ec11097694e24d93268ab031c7ec56ab4bec1c43ef7814c659f3e2493/langchain_nvidia_ai_endpoints-1.1.0.tar.gz", hash = "sha256:048a3e6d7231365fdb9fff7bcff18ce6a516b25500681f51dcb69c39e82512a0", size = 47433, upload-time = "2026-02-25T21:48:16.87Z" } +sdist = { url = "https://files.pythonhosted.org/packages/47/4b/e417af1b2b7f861f37e26bf4fa4b05cda4052002e3f84a966f0735baf94f/langchain_nvidia_ai_endpoints-1.2.0.tar.gz", hash = "sha256:4bd63b812707ea348a86539001aa9a89b3cba3ee56ade7379247a955e4bfd3eb", size = 53851, upload-time = "2026-03-10T17:55:08.127Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/22/5f07957028f7fa8c3d695934af8e7309bfd5ab43f2a7a756d3c3d6ce44f3/langchain_nvidia_ai_endpoints-1.1.0-py3-none-any.whl", hash = "sha256:eb04251b2b21facf9d6f2e6e7fa593b89e4f5023ebe3af1e02813512d1cd9687", size = 51514, upload-time = "2026-02-25T21:48:15.695Z" }, + { url = "https://files.pythonhosted.org/packages/66/e4/186f1a99e4d30bd91c8438d024dc73a71c8f7e0657c7acb6e79658aa19cf/langchain_nvidia_ai_endpoints-1.2.0-py3-none-any.whl", hash = "sha256:c8e075d5b3d31216374af0cfa9e690ab28ada3ebbde34dd6d36fe16a26d883cc", size = 58269, upload-time = "2026-03-10T17:55:06.339Z" }, ] [[package]] @@ -2102,7 +2103,7 @@ requires-dist = [ { name = "langchain-elasticsearch", marker = "extra == 'all'", specifier = ">=0.3" }, { name = "langchain-elasticsearch", marker = "extra == 'elasticsearch'", specifier = ">=0.3" }, { name = "langchain-milvus", specifier = ">=0.3.0" }, - { name = "langchain-nvidia-ai-endpoints", specifier = ">=1.1.0" }, + { name = "langchain-nvidia-ai-endpoints", specifier = ">=1.2.0" }, { name = "langchain-openai", marker = "extra == 'all'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'ingest'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'rag'", specifier = ">=0.2" }, diff --git a/notebooks/langchain_nvidia_retriever.ipynb b/notebooks/langchain_nvidia_retriever.ipynb new file mode 100644 index 000000000..e3b57bdea --- /dev/null +++ b/notebooks/langchain_nvidia_retriever.ipynb @@ -0,0 +1,277 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# NVIDIARetriever Connector \u2013 LangChain Integration\n", + "\n", + "**Motivation:** This notebook showcases the **LangChain integration** with the NVIDIA RAG Blueprint. The `NVIDIARetriever` from `langchain-nvidia-ai-endpoints` connects to the NVIDIA RAG `/v1/search` endpoint and returns standard LangChain `Document` objects, enabling seamless use in chains, agents, and RAG pipelines without custom HTTP code.\n", + "\n", + "---\n", + "\n", + "## Prerequisite: Run Ingestion First\n", + "\n", + "**You must ingest documents before using this notebook.** Use the [ingestion_api_usage.ipynb](./ingestion_api_usage.ipynb) notebook:\n", + "\n", + "1. Open [ingestion_api_usage.ipynb](./ingestion_api_usage.ipynb).\n", + "2. Execute the following cells **in order** (top to bottom):\n", + " - **1. Install Dependencies** \u2013 `pip install aiohttp`\n", + " - **2. Setup Base Configuration** \u2013 ingestor URL (port 8082)\n", + " - **3. Health Check** \u2013 verify ingestor is running\n", + " - **4. Create collection** \u2013 creates `multimodal_data` collection\n", + " - **4. Upload Document** \u2013 FILEPATHS cell, then `upload_documents` cell\n", + " - **5. Get Task Status** \u2013 poll until state is `FINISHED`\n", + "3. When ingestion is complete, return here and run the cells below.\n", + "\n", + "Ensure the **RAG server** (port 8081) is running. See [Get Started](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/docs/deploy-docker-self-hosted.md)." + ], + "id": "303aa520" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Setup" + ], + "id": "c7a2a7dd" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "!pip install langchain-nvidia-ai-endpoints langchain-core\n", + "\n", + "import os\n", + "\n", + "# RAG server URL (use collection from ingestion_api_usage.ipynb)\n", + "RAG_IPADDRESS = (\n", + " \"rag-server\" if os.environ.get(\"AI_WORKBENCH\", \"false\") == \"true\" else \"localhost\"\n", + ")\n", + "RAG_BASE_URL = f\"http://{RAG_IPADDRESS}:8081\"\n", + "\n", + "# Collection from ingestion_api_usage.ipynb (default: multimodal_data)\n", + "COLLECTION_NAME = \"multimodal_data\"" + ], + "execution_count": null, + "outputs": [], + "id": "e6fe7153" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Retrieval with NVIDIARetriever\n", + "\n", + "The `NVIDIARetriever` from `langchain-nvidia-ai-endpoints` connects to the NVIDIA RAG Blueprint `/v1/search` endpoint and returns LangChain `Document` objects. Use `COLLECTION_NAME` to match the collection you created in [ingestion_api_usage.ipynb](./ingestion_api_usage.ipynb)." + ], + "id": "640eee93" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.1 Basic Sync Retrieval" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from langchain_nvidia_ai_endpoints import NVIDIARetriever\n", + "\n", + "retriever = NVIDIARetriever(\n", + " base_url=RAG_BASE_URL,\n", + " collection_names=[COLLECTION_NAME],\n", + " k=5,\n", + ")\n", + "\n", + "query = \"What are the main topics or products discussed?\"\n", + "docs = retriever.invoke(query)\n", + "\n", + "print(f\"Query: {query}\")\n", + "print(f\"Retrieved {len(docs)} documents:\\n\")\n", + "for i, doc in enumerate(docs, 1):\n", + " content_preview = (doc.page_content or \"\")[:300] + \"...\" if len(doc.page_content or \"\") > 300 else (doc.page_content or \"\")\n", + " score = doc.metadata.get(\"score\", \"N/A\")\n", + " source = doc.metadata.get(\"document_name\", \"N/A\")\n", + " print(f\"--- Document {i} ---\")\n", + " print(f\"Score: {score} | Source: {source}\")\n", + " print(f\"Content: {content_preview}\")\n", + " print()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.2 Custom Retrieval Parameters" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "retriever_custom = NVIDIARetriever(\n", + " base_url=RAG_BASE_URL,\n", + " collection_names=[COLLECTION_NAME],\n", + " k=3,\n", + " vdb_top_k=50,\n", + " enable_reranker=True,\n", + " enable_query_rewriting=False,\n", + ")\n", + "\n", + "docs = retriever_custom.invoke(\"Summarize key information\")\n", + "print(f\"Retrieved {len(docs)} documents\")\n", + "for i, doc in enumerate(docs, 1):\n", + " print(f\" {i}. {doc.metadata.get('document_name', 'N/A')} (score: {doc.metadata.get('score', 'N/A')})\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.3 Async Retrieval" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "docs = await retriever.ainvoke(\"What features or benefits are described?\")\n", + "print(f\"Async retrieval: {len(docs)} documents\")\n", + "for i, doc in enumerate(docs[:3], 1):\n", + " print(f\" {i}. {doc.metadata.get('document_name', 'N/A')}\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.4 Error Handling" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from langchain_nvidia_ai_endpoints.retrievers import (\n", + " NVIDIARAGConnectionError,\n", + " NVIDIARAGServerError,\n", + " NVIDIARAGValidationError,\n", + ")\n", + "\n", + "try:\n", + " bad_retriever = NVIDIARetriever(\n", + " base_url=\"http://invalid-host:8081\",\n", + " collection_names=[COLLECTION_NAME],\n", + " )\n", + " bad_retriever.invoke(\"test\")\n", + "except NVIDIARAGConnectionError as e:\n", + " print(f\"Connection error (expected): {e}\")\n", + "except NVIDIARAGValidationError as e:\n", + " print(f\"Validation error: {e}\")\n", + "except NVIDIARAGServerError as e:\n", + " print(f\"Server error ({e.status_code}): {e}\")\n", + "\n", + "print(\"\\nError handling works as expected.\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## RAG Chain (Optional)\n", + "\n", + "Chain `NVIDIARetriever` with `ChatNVIDIA` for end-to-end question answering. Requires `NVIDIA_API_KEY`." + ], + "id": "3b5824c6" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "if os.environ.get(\"NVIDIA_API_KEY\", \"\").startswith(\"nvapi-\"):\n", + " from langchain_core.output_parsers import StrOutputParser\n", + " from langchain_core.prompts import ChatPromptTemplate\n", + " from langchain_core.runnables import RunnablePassthrough\n", + "\n", + " from langchain_nvidia_ai_endpoints import ChatNVIDIA, NVIDIARetriever\n", + "\n", + " retriever = NVIDIARetriever(\n", + " base_url=RAG_BASE_URL,\n", + " collection_names=[COLLECTION_NAME],\n", + " k=4,\n", + " )\n", + "\n", + " def format_docs(docs):\n", + " return \"\\n\\n\".join(d.page_content for d in docs)\n", + "\n", + " prompt = ChatPromptTemplate.from_messages([\n", + " (\"system\", \"Answer based only on the context below.\\n\\n{context}\"),\n", + " (\"human\", \"{question}\"),\n", + " ])\n", + "\n", + " llm = ChatNVIDIA(model=\"meta/llama-3.1-8b-instruct\")\n", + " chain = (\n", + " {\"context\": retriever | format_docs, \"question\": RunnablePassthrough()}\n", + " | prompt\n", + " | llm\n", + " | StrOutputParser()\n", + " )\n", + "\n", + " answer = chain.invoke(\"What are the main topics or products?\")\n", + " print(answer)\n", + "else:\n", + " print(\"NVIDIA_API_KEY not set. Set it to run the RAG chain with ChatNVIDIA.\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## Cleanup (Optional)\n", + "\n", + "To remove the collection and documents, use the delete cells in [ingestion_api_usage.ipynb](./ingestion_api_usage.ipynb) (sections 7 and 9)." + ], + "id": "4c95fdc6" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Use ingestion_api_usage.ipynb sections 7 (Delete Documents) and 9 (Delete Collections)\n", + "# to remove the multimodal_data collection when finished." + ], + "execution_count": null, + "outputs": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 4b1faf01f..acdf4f53c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "langchain>=1.2.7", "langchain-community>=0.4", "langchain-milvus>=0.3.0", - "langchain-nvidia-ai-endpoints>=1.1.0", + "langchain-nvidia-ai-endpoints>=1.2.0", "minio>=7.2,<8.0", "pdfplumber>=0.11.9", "pydantic>=2.11,<3.0", diff --git a/uv.lock b/uv.lock index 3c72857d4..090443721 100644 --- a/uv.lock +++ b/uv.lock @@ -1309,16 +1309,17 @@ wheels = [ [[package]] name = "langchain-nvidia-ai-endpoints" -version = "1.1.0" +version = "1.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, { name = "filetype" }, { name = "langchain-core" }, + { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4e/d1/f72ec11097694e24d93268ab031c7ec56ab4bec1c43ef7814c659f3e2493/langchain_nvidia_ai_endpoints-1.1.0.tar.gz", hash = "sha256:048a3e6d7231365fdb9fff7bcff18ce6a516b25500681f51dcb69c39e82512a0", size = 47433, upload-time = "2026-02-25T21:48:16.87Z" } +sdist = { url = "https://files.pythonhosted.org/packages/47/4b/e417af1b2b7f861f37e26bf4fa4b05cda4052002e3f84a966f0735baf94f/langchain_nvidia_ai_endpoints-1.2.0.tar.gz", hash = "sha256:4bd63b812707ea348a86539001aa9a89b3cba3ee56ade7379247a955e4bfd3eb", size = 53851, upload-time = "2026-03-10T17:55:08.127Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/22/5f07957028f7fa8c3d695934af8e7309bfd5ab43f2a7a756d3c3d6ce44f3/langchain_nvidia_ai_endpoints-1.1.0-py3-none-any.whl", hash = "sha256:eb04251b2b21facf9d6f2e6e7fa593b89e4f5023ebe3af1e02813512d1cd9687", size = 51514, upload-time = "2026-02-25T21:48:15.695Z" }, + { url = "https://files.pythonhosted.org/packages/66/e4/186f1a99e4d30bd91c8438d024dc73a71c8f7e0657c7acb6e79658aa19cf/langchain_nvidia_ai_endpoints-1.2.0-py3-none-any.whl", hash = "sha256:c8e075d5b3d31216374af0cfa9e690ab28ada3ebbde34dd6d36fe16a26d883cc", size = 58269, upload-time = "2026-03-10T17:55:06.339Z" }, ] [[package]] @@ -1918,7 +1919,7 @@ requires-dist = [ { name = "langchain-elasticsearch", marker = "extra == 'all'", specifier = ">=0.3" }, { name = "langchain-elasticsearch", marker = "extra == 'elasticsearch'", specifier = ">=0.3" }, { name = "langchain-milvus", specifier = ">=0.3.0" }, - { name = "langchain-nvidia-ai-endpoints", specifier = ">=1.1.0" }, + { name = "langchain-nvidia-ai-endpoints", specifier = ">=1.2.0" }, { name = "langchain-openai", marker = "extra == 'all'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'ingest'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'rag'", specifier = ">=0.2" }, From da47ad0dbc4d8733a505a1e71f313e81c7992a3e Mon Sep 17 00:00:00 2001 From: Swapnil Masurekar Date: Wed, 11 Mar 2026 11:13:35 +0530 Subject: [PATCH 48/81] =?UTF-8?q?Revert=20"Added=20url=20for=20nvidia/llam?= =?UTF-8?q?a-nemotron-rerank-1b-v2=20model=20for=20cloud=20endp=E2=80=A6"?= =?UTF-8?q?=20and=20add=20ranker=20endpoint=20in=20nvdev=20(#421)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 1a11733169fb17e80537f7ded879472a1ada4d8c. Signed-off-by: Swapnil Masurekar --- deploy/compose/.env | 2 +- deploy/compose/nvdev.env | 4 ++-- docs/python-client.md | 4 ++-- docs/retrieval-only-deployment.md | 2 +- docs/text_only_ingest.md | 2 +- notebooks/building_rag_vdb_operator.ipynb | 2 +- notebooks/image_input.ipynb | 2 +- notebooks/rag_library_lite_usage.ipynb | 2 +- notebooks/rag_library_usage.ipynb | 4 ++-- notebooks/summarization.ipynb | 2 +- tests/unit/test_rag_server/test_rag_server_health.py | 2 +- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/deploy/compose/.env b/deploy/compose/.env index e9c4cd68d..cc80cfaf0 100644 --- a/deploy/compose/.env +++ b/deploy/compose/.env @@ -45,7 +45,7 @@ export YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL=grpc # export APP_FILTEREXPRESSIONGENERATOR_MODELNAME=nvidia/llama-3.3-nemotron-super-49b-v1.5 # export APP_FILTEREXPRESSIONGENERATOR_SERVERURL="" # export SUMMARY_LLM="nvidia/llama-3.3-nemotron-super-49b-v1.5" -# export APP_RANKING_SERVERURL="https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking" +# export APP_RANKING_SERVERURL="" # export SUMMARY_LLM_SERVERURL="" # export OCR_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr # export OCR_INFER_PROTOCOL=http diff --git a/deploy/compose/nvdev.env b/deploy/compose/nvdev.env index da3725dc7..d5a919153 100644 --- a/deploy/compose/nvdev.env +++ b/deploy/compose/nvdev.env @@ -23,13 +23,13 @@ export APP_FILTEREXPRESSIONGENERATOR_MODELNAME=nvidia/llama-3.3-nemotron-super-4 export APP_EMBEDDINGS_MODELNAME=nvidia/llama-nemotron-embed-1b-v2 # For VLM Embedding Model (Nemoretriever-1b-vlm-embed-v1) # export APP_EMBEDDINGS_MODELNAME=nvdev/nvidia/llama-nemotron-embed-vl-1b-v2 -export APP_RANKING_MODELNAME=nvidia/llama-3.2-nv-rerankqa-1b-v2 +export APP_RANKING_MODELNAME=nvidia/llama-nemotron-rerank-1b-v2 export ENABLE_RERANKER=True export APP_EMBEDDINGS_SERVERURL=https://integrate.api.nvidia.com/v1 export APP_LLM_SERVERURL="" export APP_FILTEREXPRESSIONGENERATOR_SERVERURL="" export APP_RANKING_SERVERURL="" -# export APP_RANKING_SERVERURL=https://ai.api.nvidia.com/v1/nvdev/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking/v1 +# export APP_RANKING_SERVERURL=https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking export OCR_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr export OCR_INFER_PROTOCOL=http export OCR_MODEL_NAME=scene_text_ensemble diff --git a/docs/python-client.md b/docs/python-client.md index e3d8b3119..e052b5e91 100644 --- a/docs/python-client.md +++ b/docs/python-client.md @@ -362,7 +362,7 @@ from nvidia_rag.utils.configuration import NvidiaRAGConfig # }, # "ranking": { # "model_name": "nvidia/llama-nemotron-rerank-1b-v2", -# "server_url": "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking", +# "server_url": "", # }, # }) @@ -371,7 +371,7 @@ config_rag = NvidiaRAGConfig.from_yaml("config.yaml") # Update config for cloud deployment if using Option 2 if DEPLOYMENT_MODE == "cloud": config_rag.embeddings.server_url = "https://integrate.api.nvidia.com/v1" - config_rag.ranking.server_url = "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking" + config_rag.ranking.server_url = "" # Empty uses NVIDIA API catalog config_rag.llm.server_url = "" # Empty uses NVIDIA API catalog # Initialize NvidiaRAG with config diff --git a/docs/retrieval-only-deployment.md b/docs/retrieval-only-deployment.md index 09d70c311..7f7f94475 100644 --- a/docs/retrieval-only-deployment.md +++ b/docs/retrieval-only-deployment.md @@ -116,7 +116,7 @@ For an even lighter deployment, use [NVIDIA-hosted NIMs](deploy-docker-nvidia-ho ```bash # Configure to use NVIDIA-hosted endpoints export APP_EMBEDDINGS_SERVERURL="" -export APP_RANKING_SERVERURL="https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking" +export APP_RANKING_SERVERURL="" ``` :::{note} diff --git a/docs/text_only_ingest.md b/docs/text_only_ingest.md index cf61f69ec..10cb6e9b1 100644 --- a/docs/text_only_ingest.md +++ b/docs/text_only_ingest.md @@ -69,7 +69,7 @@ In case you are [interacting with cloud hosted models](deploy-docker-nvidia-host ```bash export APP_EMBEDDINGS_SERVERURL="" export APP_LLM_SERVERURL="" - export APP_RANKING_SERVERURL="https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking" + export APP_RANKING_SERVERURL="" export YOLOX_HTTP_ENDPOINT="https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3" export YOLOX_INFER_PROTOCOL="http" ``` diff --git a/notebooks/building_rag_vdb_operator.ipynb b/notebooks/building_rag_vdb_operator.ipynb index 0cab35d6e..1852f1d9c 100644 --- a/notebooks/building_rag_vdb_operator.ipynb +++ b/notebooks/building_rag_vdb_operator.ipynb @@ -2092,7 +2092,7 @@ " config_ingestor.embeddings.server_url = \"https://integrate.api.nvidia.com/v1\"\n", " config_ingestor.llm.server_url = \"\" # Empty uses NVIDIA API catalog\n", " config_rag.embeddings.server_url = \"https://integrate.api.nvidia.com/v1\"\n", - " config_rag.ranking.server_url = \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking\"\n", + " config_rag.ranking.server_url = \"\" # Empty uses NVIDIA API catalog\n", " config_rag.llm.server_url = \"\" # Empty uses NVIDIA API catalog\n", "\n", "# Create embedding model using the configuration\n", diff --git a/notebooks/image_input.ipynb b/notebooks/image_input.ipynb index 9332f4144..4f778a2fa 100644 --- a/notebooks/image_input.ipynb +++ b/notebooks/image_input.ipynb @@ -360,7 +360,7 @@ "outputs": [], "source": [ "# Start the RAG server (accessible at localhost:8081)\n", - "os.environ[\"APP_RANKING_SERVERURL\"] = \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking\"\n", + "os.environ[\"APP_RANKING_SERVERURL\"] = \"\"\n", "! docker compose -f ../deploy/compose/docker-compose-rag-server.yaml up -d --build" ] }, diff --git a/notebooks/rag_library_lite_usage.ipynb b/notebooks/rag_library_lite_usage.ipynb index ded451558..915eb00ca 100644 --- a/notebooks/rag_library_lite_usage.ipynb +++ b/notebooks/rag_library_lite_usage.ipynb @@ -482,7 +482,7 @@ "\n", "# Set config for cloud API endpoints\n", "config_rag.embeddings.server_url = \"https://integrate.api.nvidia.com/v1\"\n", - "config_rag.ranking.server_url = \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking\"\n", + "config_rag.ranking.server_url = \"\" # Empty uses NVIDIA API catalog\n", "config_rag.llm.server_url = \"\" # Empty uses NVIDIA API catalog\n", "\n", "# Initialize NvidiaRAG with config\n", diff --git a/notebooks/rag_library_usage.ipynb b/notebooks/rag_library_usage.ipynb index 659f2f304..894e82620 100644 --- a/notebooks/rag_library_usage.ipynb +++ b/notebooks/rag_library_usage.ipynb @@ -630,7 +630,7 @@ "# },\n", "# \"ranking\": {\n", "# \"model_name\": \"nvidia/llama-nemotron-rerank-1b-v2\",\n", - "# \"server_url\": \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking\",\n", + "# \"server_url\": \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking/v1\",\n", "# },\n", "# }\n", "# )\n", @@ -640,7 +640,7 @@ "# Update config for cloud deployment if using Option 2\n", "if DEPLOYMENT_MODE == \"cloud\":\n", " config_rag.embeddings.server_url = \"https://integrate.api.nvidia.com/v1\"\n", - " config_rag.ranking.server_url = \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking\"\n", + " config_rag.ranking.server_url = \"\" # Empty uses NVIDIA API catalog\n", " config_rag.llm.server_url = \"\" # Empty uses NVIDIA API catalog\n", "\n", "# Initialize NvidiaRAG with config\n", diff --git a/notebooks/summarization.ipynb b/notebooks/summarization.ipynb index 63d39c5ac..d0c7ef285 100644 --- a/notebooks/summarization.ipynb +++ b/notebooks/summarization.ipynb @@ -545,7 +545,7 @@ "if DEPLOYMENT_MODE == \"cloud\":\n", " config.embeddings.server_url = \"https://integrate.api.nvidia.com/v1\"\n", " config.llm.server_url = \"\" # Empty uses NVIDIA API catalog\n", - " config.ranking.server_url = \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking\"\n", + " config.ranking.server_url = \"https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking/v1\"\n", " config.summarizer.server_url = \"\" # Empty uses NVIDIA API catalog\n", "else:\n", " config.embeddings.server_url = \"nemotron-embedding-ms:8000/v1\"\n", diff --git a/tests/unit/test_rag_server/test_rag_server_health.py b/tests/unit/test_rag_server/test_rag_server_health.py index fb8783dff..966a85e74 100644 --- a/tests/unit/test_rag_server/test_rag_server_health.py +++ b/tests/unit/test_rag_server/test_rag_server_health.py @@ -415,7 +415,7 @@ async def test_check_all_services_health_nvidia_api_catalog(self, mock_config): mock_config.query_rewriter.server_url = "https://ai.api.nvidia.com/v1" mock_config.embeddings.server_url = "https://api.nvcf.nvidia.com/v2" mock_config.ranking.server_url = ( - "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking" # Empty URL should be treated as API catalog + "" # Empty URL should be treated as API catalog ) mock_vdb_op = MagicMock() From cfb1dd9b23cb37e59c1c80fcfe9100b0dfa95597 Mon Sep 17 00:00:00 2001 From: Swapnil Masurekar Date: Wed, 11 Mar 2026 12:52:14 +0530 Subject: [PATCH 49/81] Nemoretriever OCR version 1.2.0 -> 1.2.1 in Helm (#422) --- deploy/helm/nvidia-blueprint-rag/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/helm/nvidia-blueprint-rag/values.yaml b/deploy/helm/nvidia-blueprint-rag/values.yaml index 3a39c1224..1afe008d1 100644 --- a/deploy/helm/nvidia-blueprint-rag/values.yaml +++ b/deploy/helm/nvidia-blueprint-rag/values.yaml @@ -1029,7 +1029,7 @@ nv-ingest: replicaCount: 1 image: repository: nvcr.io/nim/nvidia/nemoretriever-ocr-v1 - tag: "1.2.0" + tag: "1.2.1" imagePullSecrets: - name: ngc-secret env: From 7a0c8662b14cb23809c87140cfc49c89854d92d9 Mon Sep 17 00:00:00 2001 From: Shubhadeep Das <149712532+shubhadeepd@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:28:02 +0530 Subject: [PATCH 50/81] Fix name for rag langchain connector (#423) --- docs/notebooks.md | 2 +- notebooks/langchain_nvidia_retriever.ipynb | 588 +++++++++++---------- 2 files changed, 314 insertions(+), 276 deletions(-) diff --git a/docs/notebooks.md b/docs/notebooks.md index 6a56686a4..6146e4abf 100644 --- a/docs/notebooks.md +++ b/docs/notebooks.md @@ -103,7 +103,7 @@ Use the following notebooks to learn comprehensive Python client usage, metadata - [rag_library_lite_usage.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/rag_library_lite_usage.ipynb) – Demonstrates containerless deployment of the NVIDIA RAG Python package in lite mode. Uses Milvus Lite (embedded vector database) and NV-Ingest subprocess mode for a simplified setup without Docker containers. Leverages NVIDIA cloud APIs for embeddings, ranking, and LLM inference. **Note**: This mode does not support image/table/chart citations or document summarization. -- [langchain_nvidia_retriever.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/langchain_nvidia_retriever.ipynb) – Showcases **LangChain integration** with the NVIDIA RAG Blueprint. Run [ingestion_api_usage.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/ingestion_api_usage.ipynb) first to ingest documents, then use `NVIDIARetriever` for retrieval (sync/async), custom parameters, error handling, and optional RAG chaining with `ChatNVIDIA`. +- [langchain_nvidia_retriever.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/langchain_nvidia_retriever.ipynb) – Showcases **LangChain integration** with the NVIDIA RAG Blueprint. Run [ingestion_api_usage.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/ingestion_api_usage.ipynb) first to ingest documents, then use `NVIDIARAGRetriever` for retrieval (sync/async), custom parameters, error handling, and optional RAG chaining with `ChatNVIDIA`. diff --git a/notebooks/langchain_nvidia_retriever.ipynb b/notebooks/langchain_nvidia_retriever.ipynb index e3b57bdea..02b05b6db 100644 --- a/notebooks/langchain_nvidia_retriever.ipynb +++ b/notebooks/langchain_nvidia_retriever.ipynb @@ -1,277 +1,315 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# NVIDIARetriever Connector \u2013 LangChain Integration\n", - "\n", - "**Motivation:** This notebook showcases the **LangChain integration** with the NVIDIA RAG Blueprint. The `NVIDIARetriever` from `langchain-nvidia-ai-endpoints` connects to the NVIDIA RAG `/v1/search` endpoint and returns standard LangChain `Document` objects, enabling seamless use in chains, agents, and RAG pipelines without custom HTTP code.\n", - "\n", - "---\n", - "\n", - "## Prerequisite: Run Ingestion First\n", - "\n", - "**You must ingest documents before using this notebook.** Use the [ingestion_api_usage.ipynb](./ingestion_api_usage.ipynb) notebook:\n", - "\n", - "1. Open [ingestion_api_usage.ipynb](./ingestion_api_usage.ipynb).\n", - "2. Execute the following cells **in order** (top to bottom):\n", - " - **1. Install Dependencies** \u2013 `pip install aiohttp`\n", - " - **2. Setup Base Configuration** \u2013 ingestor URL (port 8082)\n", - " - **3. Health Check** \u2013 verify ingestor is running\n", - " - **4. Create collection** \u2013 creates `multimodal_data` collection\n", - " - **4. Upload Document** \u2013 FILEPATHS cell, then `upload_documents` cell\n", - " - **5. Get Task Status** \u2013 poll until state is `FINISHED`\n", - "3. When ingestion is complete, return here and run the cells below.\n", - "\n", - "Ensure the **RAG server** (port 8081) is running. See [Get Started](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/docs/deploy-docker-self-hosted.md)." - ], - "id": "303aa520" + "cells": [ + { + "cell_type": "markdown", + "id": "303aa520", + "metadata": {}, + "source": [ + "# NVIDIARAGRetriever Connector – LangChain Integration\n", + "\n", + "**Motivation:** This notebook showcases the **LangChain integration** with the NVIDIA RAG Blueprint. The `NVIDIARAGRetriever` from `langchain-nvidia-ai-endpoints` connects to the NVIDIA RAG `/v1/search` endpoint and returns standard LangChain `Document` objects, enabling seamless use in chains, agents, and RAG pipelines without custom HTTP code.\n", + "\n", + "---\n", + "\n", + "## Prerequisite: Run Ingestion First\n", + "\n", + "**You must ingest documents before using this notebook.** Use the [ingestion_api_usage.ipynb](./ingestion_api_usage.ipynb) notebook:\n", + "\n", + "1. Open [ingestion_api_usage.ipynb](./ingestion_api_usage.ipynb).\n", + "2. Execute the following cells **in order** (top to bottom):\n", + " - **1. Install Dependencies** – `pip install aiohttp`\n", + " - **2. Setup Base Configuration** – ingestor URL (port 8082)\n", + " - **3. Health Check** – verify ingestor is running\n", + " - **4. Create collection** – creates `multimodal_data` collection\n", + " - **4. Upload Document** – FILEPATHS cell, then `upload_documents` cell\n", + " - **5. Get Task Status** – poll until state is `FINISHED`\n", + "3. When ingestion is complete, return here and run the cells below.\n", + "\n", + "Ensure the **RAG server** (port 8081) is running. See [Get Started](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/docs/deploy-docker-self-hosted.md)." + ] + }, + { + "cell_type": "markdown", + "id": "c7a2a7dd", + "metadata": {}, + "source": [ + "---\n", + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e6fe7153", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install langchain-nvidia-ai-endpoints langchain-core\n", + "\n", + "import os\n", + "\n", + "# RAG server URL (use collection from ingestion_api_usage.ipynb)\n", + "RAG_IPADDRESS = (\n", + " \"rag-server\" if os.environ.get(\"AI_WORKBENCH\", \"false\") == \"true\" else \"localhost\"\n", + ")\n", + "RAG_BASE_URL = f\"http://{RAG_IPADDRESS}:8081\"\n", + "\n", + "# Collection from ingestion_api_usage.ipynb (default: multimodal_data)\n", + "COLLECTION_NAME = \"multimodal_data\"" + ] + }, + { + "cell_type": "markdown", + "id": "640eee93", + "metadata": {}, + "source": [ + "---\n", + "## Retrieval with NVIDIARAGRetriever\n", + "\n", + "The `NVIDIARAGRetriever` from `langchain-nvidia-ai-endpoints` connects to the NVIDIA RAG Blueprint `/v1/search` endpoint and returns LangChain `Document` objects. Use `COLLECTION_NAME` to match the collection you created in [ingestion_api_usage.ipynb](./ingestion_api_usage.ipynb)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.1 Basic Sync Retrieval" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b09138a", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_nvidia_ai_endpoints import NVIDIARAGRetriever\n", + "\n", + "retriever = NVIDIARAGRetriever(\n", + " base_url=RAG_BASE_URL,\n", + " collection_names=[COLLECTION_NAME],\n", + " k=5,\n", + ")\n", + "\n", + "query = \"What are the main topics or products discussed?\"\n", + "docs = retriever.invoke(query)\n", + "\n", + "print(f\"Query: {query}\")\n", + "print(f\"Retrieved {len(docs)} documents:\\n\")\n", + "for i, doc in enumerate(docs, 1):\n", + " content_preview = (doc.page_content or \"\")[:300] + \"...\" if len(doc.page_content or \"\") > 300 else (doc.page_content or \"\")\n", + " score = doc.metadata.get(\"score\", \"N/A\")\n", + " source = doc.metadata.get(\"document_name\", \"N/A\")\n", + " print(f\"--- Document {i} ---\")\n", + " print(f\"Score: {score} | Source: {source}\")\n", + " print(f\"Content: {content_preview}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.2 Custom Retrieval Parameters" + ] + }, + { + "cell_type": "markdown", + "id": "35d7f118", + "metadata": {}, + "source": [ + "For details on retrieval parameters, filter expressions, and metadata:\n", + "- [Custom metadata & filter expressions](../docs/custom-metadata.md) – `filter_expr` syntax (Milvus), metadata schema\n", + "- [Multi-turn & query rewriting](../docs/multiturn.md) – `enable_query_rewriting` for decontextualizing follow-up questions\n", + "- [Retriever API usage](./retriever_api_usage.ipynb) – Search endpoint payload parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6e5aa122", + "metadata": {}, + "outputs": [], + "source": [ + "retriever_custom = NVIDIARAGRetriever(\n", + " base_url=RAG_BASE_URL,\n", + " collection_names=[COLLECTION_NAME],\n", + " # Result counts\n", + " k=3, # Number of document chunks to return (0-25, maps to reranker_top_k)\n", + " vdb_top_k=50, # Top results from vector DB before reranking (0-400)\n", + " # Feature toggles\n", + " enable_reranker=True, # Rerank results for relevance\n", + " enable_query_rewriting=False, # Rewrite query for better retrieval\n", + " enable_filter_generator=False, # Auto-generate filters from query\n", + " enable_citations=True, # Include image/table/chart citations in metadata\n", + " # Filtering\n", + " confidence_threshold=0.0, # Min confidence (0.0-1.0, requires enable_reranker=True)\n", + " filter_expr=None, # Milvus filter expression, e.g. content_metadata['file_name'] == \"doc.pdf\"'\n", + " # Advanced\n", + " vdb_endpoint=\"http://milvus:19530\", # Vector DB endpoint (override if needed)\n", + " messages=[], # Conversation history for context-aware retrieval\n", + " timeout=60.0, # HTTP request timeout in seconds\n", + ")\n", + "\n", + "docs = retriever_custom.invoke(\"Summarize key information\")\n", + "print(f\"Retrieved {len(docs)} documents\")\n", + "for i, doc in enumerate(docs, 1):\n", + " print(f\" {i}. {doc.metadata.get('document_name', 'N/A')} (score: {doc.metadata.get('score', 'N/A')})\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.3 Async Retrieval" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "docs = await retriever.ainvoke(\"What features or benefits are described?\")\n", + "print(f\"Async retrieval: {len(docs)} documents\")\n", + "for i, doc in enumerate(docs[:3], 1):\n", + " print(f\" {i}. {doc.metadata.get('document_name', 'N/A')}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.4 Error Handling" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bfcc9f22", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_nvidia_ai_endpoints.retrievers import (\n", + " NVIDIARAGConnectionError,\n", + " NVIDIARAGServerError,\n", + " NVIDIARAGValidationError,\n", + ")\n", + "\n", + "try:\n", + " bad_retriever = NVIDIARAGRetriever(\n", + " base_url=\"http://invalid-host:8081\",\n", + " collection_names=[COLLECTION_NAME],\n", + " )\n", + " bad_retriever.invoke(\"test\")\n", + "except NVIDIARAGConnectionError as e:\n", + " print(f\"Connection error (expected): {e}\")\n", + "except NVIDIARAGValidationError as e:\n", + " print(f\"Validation error: {e}\")\n", + "except NVIDIARAGServerError as e:\n", + " print(f\"Server error ({e.status_code}): {e}\")\n", + "\n", + "print(\"\\nError handling works as expected.\")" + ] + }, + { + "cell_type": "markdown", + "id": "3b5824c6", + "metadata": {}, + "source": [ + "---\n", + "## RAG Chain (Optional)\n", + "\n", + "Chain `NVIDIARAGRetriever` with `ChatNVIDIA` for end-to-end question answering. Requires `NVIDIA_API_KEY` to call the NVIDIA API Catalog.\n", + "\n", + "**Get an API key:** See [Get an API Key](../docs/api-key.md) for instructions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d82a5d54", + "metadata": {}, + "outputs": [], + "source": [ + "# Set NVIDIA_API_KEY if not already set (see ../docs/api-key.md to get a key)\n", + "if not os.environ.get(\"NVIDIA_API_KEY\", \"\").startswith(\"nvapi-\"):\n", + " import getpass\n", + " key = getpass.getpass(\"Enter your NVIDIA API key (nvapi-...): \")\n", + " if key.startswith(\"nvapi-\"):\n", + " os.environ[\"NVIDIA_API_KEY\"] = key\n", + " else:\n", + " print(\"NVIDIA_API_KEY not set. Set it to run the RAG chain. See [Get an API Key](../docs/api-key.md)\")\n", + "\n", + "if os.environ.get(\"NVIDIA_API_KEY\", \"\").startswith(\"nvapi-\"):\n", + " from langchain_core.output_parsers import StrOutputParser\n", + " from langchain_core.prompts import ChatPromptTemplate\n", + " from langchain_core.runnables import RunnablePassthrough\n", + "\n", + " from langchain_nvidia_ai_endpoints import ChatNVIDIA, NVIDIARAGRetriever\n", + "\n", + " retriever = NVIDIARAGRetriever(\n", + " base_url=RAG_BASE_URL,\n", + " collection_names=[COLLECTION_NAME],\n", + " k=4,\n", + " )\n", + "\n", + " def format_docs(docs):\n", + " return \"\\n\\n\".join(d.page_content for d in docs)\n", + "\n", + " prompt = ChatPromptTemplate.from_messages([\n", + " (\"system\", \"Answer based only on the context below.\\n\\n{context}\"),\n", + " (\"human\", \"{question}\"),\n", + " ])\n", + "\n", + " # Model aligned with rag-server default (nvidia/llama-3.3-nemotron-super-49b-v1.5)\n", + " llm = ChatNVIDIA(model=\"nvidia/llama-3.3-nemotron-super-49b-v1.5\")\n", + " chain = (\n", + " {\"context\": retriever | format_docs, \"question\": RunnablePassthrough()}\n", + " | prompt\n", + " | llm\n", + " | StrOutputParser()\n", + " )\n", + "\n", + " answer = chain.invoke(\"What are the main topics or products?\")\n", + " print(answer)\n", + "else:\n", + " print(\"NVIDIA_API_KEY not set. Set it (e.g. os.environ['NVIDIA_API_KEY'] = 'nvapi-...') or run this cell again to be prompted. See [Get an API Key](../docs/api-key.md)\")" + ] + }, + { + "cell_type": "markdown", + "id": "4c95fdc6", + "metadata": {}, + "source": [ + "---\n", + "## Cleanup (Optional)\n", + "\n", + "To remove the collection and documents, use the delete cells in [ingestion_api_usage.ipynb](./ingestion_api_usage.ipynb) (sections 7 and 9)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Use ingestion_api_usage.ipynb sections 7 (Delete Documents) and 9 (Delete Collections)\n", + "# to remove the multimodal_data collection when finished." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.12" + } }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "## Setup" - ], - "id": "c7a2a7dd" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "!pip install langchain-nvidia-ai-endpoints langchain-core\n", - "\n", - "import os\n", - "\n", - "# RAG server URL (use collection from ingestion_api_usage.ipynb)\n", - "RAG_IPADDRESS = (\n", - " \"rag-server\" if os.environ.get(\"AI_WORKBENCH\", \"false\") == \"true\" else \"localhost\"\n", - ")\n", - "RAG_BASE_URL = f\"http://{RAG_IPADDRESS}:8081\"\n", - "\n", - "# Collection from ingestion_api_usage.ipynb (default: multimodal_data)\n", - "COLLECTION_NAME = \"multimodal_data\"" - ], - "execution_count": null, - "outputs": [], - "id": "e6fe7153" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "## Retrieval with NVIDIARetriever\n", - "\n", - "The `NVIDIARetriever` from `langchain-nvidia-ai-endpoints` connects to the NVIDIA RAG Blueprint `/v1/search` endpoint and returns LangChain `Document` objects. Use `COLLECTION_NAME` to match the collection you created in [ingestion_api_usage.ipynb](./ingestion_api_usage.ipynb)." - ], - "id": "640eee93" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.1 Basic Sync Retrieval" - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from langchain_nvidia_ai_endpoints import NVIDIARetriever\n", - "\n", - "retriever = NVIDIARetriever(\n", - " base_url=RAG_BASE_URL,\n", - " collection_names=[COLLECTION_NAME],\n", - " k=5,\n", - ")\n", - "\n", - "query = \"What are the main topics or products discussed?\"\n", - "docs = retriever.invoke(query)\n", - "\n", - "print(f\"Query: {query}\")\n", - "print(f\"Retrieved {len(docs)} documents:\\n\")\n", - "for i, doc in enumerate(docs, 1):\n", - " content_preview = (doc.page_content or \"\")[:300] + \"...\" if len(doc.page_content or \"\") > 300 else (doc.page_content or \"\")\n", - " score = doc.metadata.get(\"score\", \"N/A\")\n", - " source = doc.metadata.get(\"document_name\", \"N/A\")\n", - " print(f\"--- Document {i} ---\")\n", - " print(f\"Score: {score} | Source: {source}\")\n", - " print(f\"Content: {content_preview}\")\n", - " print()" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.2 Custom Retrieval Parameters" - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "retriever_custom = NVIDIARetriever(\n", - " base_url=RAG_BASE_URL,\n", - " collection_names=[COLLECTION_NAME],\n", - " k=3,\n", - " vdb_top_k=50,\n", - " enable_reranker=True,\n", - " enable_query_rewriting=False,\n", - ")\n", - "\n", - "docs = retriever_custom.invoke(\"Summarize key information\")\n", - "print(f\"Retrieved {len(docs)} documents\")\n", - "for i, doc in enumerate(docs, 1):\n", - " print(f\" {i}. {doc.metadata.get('document_name', 'N/A')} (score: {doc.metadata.get('score', 'N/A')})\")" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.3 Async Retrieval" - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "docs = await retriever.ainvoke(\"What features or benefits are described?\")\n", - "print(f\"Async retrieval: {len(docs)} documents\")\n", - "for i, doc in enumerate(docs[:3], 1):\n", - " print(f\" {i}. {doc.metadata.get('document_name', 'N/A')}\")" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.4 Error Handling" - ] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from langchain_nvidia_ai_endpoints.retrievers import (\n", - " NVIDIARAGConnectionError,\n", - " NVIDIARAGServerError,\n", - " NVIDIARAGValidationError,\n", - ")\n", - "\n", - "try:\n", - " bad_retriever = NVIDIARetriever(\n", - " base_url=\"http://invalid-host:8081\",\n", - " collection_names=[COLLECTION_NAME],\n", - " )\n", - " bad_retriever.invoke(\"test\")\n", - "except NVIDIARAGConnectionError as e:\n", - " print(f\"Connection error (expected): {e}\")\n", - "except NVIDIARAGValidationError as e:\n", - " print(f\"Validation error: {e}\")\n", - "except NVIDIARAGServerError as e:\n", - " print(f\"Server error ({e.status_code}): {e}\")\n", - "\n", - "print(\"\\nError handling works as expected.\")" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "## RAG Chain (Optional)\n", - "\n", - "Chain `NVIDIARetriever` with `ChatNVIDIA` for end-to-end question answering. Requires `NVIDIA_API_KEY`." - ], - "id": "3b5824c6" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "if os.environ.get(\"NVIDIA_API_KEY\", \"\").startswith(\"nvapi-\"):\n", - " from langchain_core.output_parsers import StrOutputParser\n", - " from langchain_core.prompts import ChatPromptTemplate\n", - " from langchain_core.runnables import RunnablePassthrough\n", - "\n", - " from langchain_nvidia_ai_endpoints import ChatNVIDIA, NVIDIARetriever\n", - "\n", - " retriever = NVIDIARetriever(\n", - " base_url=RAG_BASE_URL,\n", - " collection_names=[COLLECTION_NAME],\n", - " k=4,\n", - " )\n", - "\n", - " def format_docs(docs):\n", - " return \"\\n\\n\".join(d.page_content for d in docs)\n", - "\n", - " prompt = ChatPromptTemplate.from_messages([\n", - " (\"system\", \"Answer based only on the context below.\\n\\n{context}\"),\n", - " (\"human\", \"{question}\"),\n", - " ])\n", - "\n", - " llm = ChatNVIDIA(model=\"meta/llama-3.1-8b-instruct\")\n", - " chain = (\n", - " {\"context\": retriever | format_docs, \"question\": RunnablePassthrough()}\n", - " | prompt\n", - " | llm\n", - " | StrOutputParser()\n", - " )\n", - "\n", - " answer = chain.invoke(\"What are the main topics or products?\")\n", - " print(answer)\n", - "else:\n", - " print(\"NVIDIA_API_KEY not set. Set it to run the RAG chain with ChatNVIDIA.\")" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "## Cleanup (Optional)\n", - "\n", - "To remove the collection and documents, use the delete cells in [ingestion_api_usage.ipynb](./ingestion_api_usage.ipynb) (sections 7 and 9)." - ], - "id": "4c95fdc6" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# Use ingestion_api_usage.ipynb sections 7 (Delete Documents) and 9 (Delete Collections)\n", - "# to remove the multimodal_data collection when finished." - ], - "execution_count": null, - "outputs": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.0" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file + "nbformat": 4, + "nbformat_minor": 5 +} From 4e2ff47680225927b5279808f220d08431f0fe3a Mon Sep 17 00:00:00 2001 From: niyatisingal Date: Wed, 11 Mar 2026 14:50:13 +0530 Subject: [PATCH 51/81] Print reasoning tokens if DEBUG logging is enabled (#424) * Print reasoning tokens if DEBUG logging is enabled (#378) * Print reasoning tokens if VERBOSE logging is enabled Signed-off-by: Niyati Singal * fix(observability): use hardcoded GenAI attribute keys instead of deprecated SpanAttributes (#377) Signed-off-by: Swapnil Masurekar --------- Signed-off-by: Niyati Singal Signed-off-by: Swapnil Masurekar Co-authored-by: Swapnil Masurekar * Print reasoning tokens if DEBUG logging is enabled Signed-off-by: Niyati Singal --------- Signed-off-by: Niyati Singal Signed-off-by: Swapnil Masurekar Co-authored-by: Swapnil Masurekar --- src/nvidia_rag/utils/llm.py | 52 +++++++++++++++++++++++++++++++ tests/unit/test_utils/test_llm.py | 4 +-- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/src/nvidia_rag/utils/llm.py b/src/nvidia_rag/utils/llm.py index 0148f3ce2..e1f226655 100644 --- a/src/nvidia_rag/utils/llm.py +++ b/src/nvidia_rag/utils/llm.py @@ -440,6 +440,9 @@ def streaming_filter_think(chunks: Iterable[str]) -> Iterable[str]: This generator filters content between think tags in streaming LLM responses. It handles both complete tags in a single chunk and tags split across multiple tokens. + When DEBUG logging is enabled (i.e. LOGLEVEL=DEBUG), reasoning tokens are + logged from block content or reasoning_content field. + Args: chunks (Iterable[str]): Chunks from a streaming LLM response @@ -464,6 +467,8 @@ def streaming_filter_think(chunks: Iterable[str]) -> Iterable[str]: match_position = 0 buffer = "" output_buffer = "" + think_accumulator = "" + reasoning_content_accumulator = "" chunk_count = 0 for chunk in chunks: @@ -471,6 +476,10 @@ def streaming_filter_think(chunks: Iterable[str]) -> Iterable[str]: content = content or reasoning chunk_count += 1 + # Accumulate reasoning tokens when DEBUG logging is enabled (e.g. reasoning_content from nemotron-3-nano) + if reasoning and logger.isEnabledFor(logging.DEBUG): + reasoning_content_accumulator += reasoning + # Let's first check for full tags - this is the most reliable approach buffer += content @@ -487,6 +496,10 @@ def streaming_filter_think(chunks: Iterable[str]) -> Iterable[str]: while state == IN_THINK and FULL_END_TAG in buffer: end_idx = buffer.find(FULL_END_TAG) + if logger.isEnabledFor(logging.DEBUG): + think_content = buffer[:end_idx] + if think_content: + think_accumulator += think_content + "\n" # Discard everything up to and including end tag buffer = buffer[end_idx + len(FULL_END_TAG) :] content = buffer @@ -534,10 +547,14 @@ def streaming_filter_think(chunks: Iterable[str]) -> Iterable[str]: elif state == IN_THINK: if content_stripped == END_TAG_PARTS[0].strip(): + # Accumulate think content before the end tag start + think_accumulator += buffer[: -len(content)] if content else buffer state = MATCHING_END match_position = 1 buffer = content # Keep this token in buffer else: + if logger.isEnabledFor(logging.DEBUG): + think_accumulator += buffer buffer = "" # Discard content inside think block elif state == MATCHING_END: @@ -546,11 +563,15 @@ def streaming_filter_think(chunks: Iterable[str]) -> Iterable[str]: match_position += 1 if match_position >= len(END_TAG_PARTS): # Complete end tag matched + if think_accumulator and logger.isEnabledFor(logging.DEBUG): + think_accumulator += "\n" state = NORMAL match_position = 0 buffer = "" # Clear buffer else: # False match, revert to IN_THINK + if logger.isEnabledFor(logging.DEBUG): + think_accumulator += buffer state = IN_THINK buffer = "" # Discard content @@ -572,6 +593,11 @@ def streaming_filter_think(chunks: Iterable[str]) -> Iterable[str]: if output_buffer: yield output_buffer + if think_accumulator and logger.isEnabledFor(logging.DEBUG): + logger.debug("Reasoning tokens (think): %s", think_accumulator.rstrip()) + if reasoning_content_accumulator and logger.isEnabledFor(logging.DEBUG): + logger.debug("Reasoning tokens: %s", reasoning_content_accumulator) + logger.info( "Finished streaming_filter_think processing after %d chunks", chunk_count ) @@ -608,6 +634,9 @@ async def streaming_filter_think_async(chunks, enable_thinking: bool = False): This async generator filters content between think tags in streaming LLM responses. It handles both complete tags in a single chunk and tags split across multiple tokens. + When DEBUG logging is enabled (i.e. LOGLEVEL=DEBUG), reasoning tokens are + logged from block content or reasoning_content field. + When enable_thinking is True and the model uses a separate reasoning_content field (e.g. Nemotron 3), reasoning tokens are dropped and only content is forwarded. The tag filter still runs to handle models that embed reasoning in content. @@ -638,6 +667,8 @@ async def streaming_filter_think_async(chunks, enable_thinking: bool = False): match_position = 0 buffer = "" output_buffer = "" + think_accumulator = "" + reasoning_content_accumulator = "" chunk_count = 0 async for chunk in chunks: @@ -645,6 +676,10 @@ async def streaming_filter_think_async(chunks, enable_thinking: bool = False): content = content if enable_thinking else (content or reasoning) chunk_count += 1 + # Accumulate reasoning when DEBUG logging is enabled (e.g. reasoning_content from nemotron-3-nano) + if reasoning and logger.isEnabledFor(logging.DEBUG): + reasoning_content_accumulator += reasoning + # Let's first check for full tags - this is the most reliable approach buffer += content @@ -661,6 +696,10 @@ async def streaming_filter_think_async(chunks, enable_thinking: bool = False): while state == IN_THINK and FULL_END_TAG in buffer: end_idx = buffer.find(FULL_END_TAG) + if logger.isEnabledFor(logging.DEBUG): + think_content = buffer[:end_idx] + if think_content: + think_accumulator += think_content + "\n" # Discard everything up to and including end tag buffer = buffer[end_idx + len(FULL_END_TAG) :] content = buffer @@ -708,10 +747,14 @@ async def streaming_filter_think_async(chunks, enable_thinking: bool = False): elif state == IN_THINK: if content_stripped == END_TAG_PARTS[0].strip(): + # Accumulate think content before the end tag start + think_accumulator += buffer[: -len(content)] if content else buffer state = MATCHING_END match_position = 1 buffer = content # Keep this token in buffer else: + if logger.isEnabledFor(logging.DEBUG): + think_accumulator += buffer buffer = "" # Discard content inside think block elif state == MATCHING_END: @@ -720,11 +763,15 @@ async def streaming_filter_think_async(chunks, enable_thinking: bool = False): match_position += 1 if match_position >= len(END_TAG_PARTS): # Complete end tag matched + if think_accumulator and logger.isEnabledFor(logging.DEBUG): + think_accumulator += "\n" state = NORMAL match_position = 0 buffer = "" # Clear buffer else: # False match, revert to IN_THINK + if logger.isEnabledFor(logging.DEBUG): + think_accumulator += buffer state = IN_THINK buffer = "" # Discard content @@ -746,6 +793,11 @@ async def streaming_filter_think_async(chunks, enable_thinking: bool = False): if output_buffer: yield output_buffer + if think_accumulator and logger.isEnabledFor(logging.DEBUG): + logger.debug("Reasoning tokens: %s", think_accumulator.rstrip()) + if reasoning_content_accumulator and logger.isEnabledFor(logging.DEBUG): + logger.debug("Reasoning tokens: %s", reasoning_content_accumulator) + logger.info( "Finished streaming_filter_think_async processing after %d chunks", chunk_count ) diff --git a/tests/unit/test_utils/test_llm.py b/tests/unit/test_utils/test_llm.py index 377f0b68d..c59cabf43 100644 --- a/tests/unit/test_utils/test_llm.py +++ b/tests/unit/test_utils/test_llm.py @@ -425,7 +425,7 @@ class TestStreamingFilterThink: """Test cases for streaming_filter_think function.""" def create_mock_chunk(self, content): - """Helper to create mock chunk with content attribute.""" + """Helper to create mock chunk with content and additional_kwargs (so 'in' works).""" chunk = Mock() chunk.content = content chunk.additional_kwargs = {} @@ -822,7 +822,7 @@ def test_streaming_filter_complete_workflow(self): assert result == expected def create_mock_chunk(self, content): - """Helper to create mock chunk with content attribute.""" + """Helper to create mock chunk with content and additional_kwargs (so 'in' works).""" chunk = Mock() chunk.content = content chunk.additional_kwargs = {} From aa965864af0c3ef391b7d00449d55d4a5ca7c10d Mon Sep 17 00:00:00 2001 From: Shubhadeep Das <149712532+shubhadeepd@users.noreply.github.com> Date: Wed, 11 Mar 2026 17:39:02 +0530 Subject: [PATCH 52/81] CI: Update helm packaging and selective publish support (#425) --- .github/workflows/publish-artifacts.yml | 114 ++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/.github/workflows/publish-artifacts.yml b/.github/workflows/publish-artifacts.yml index 2be3979e1..659d4ed04 100644 --- a/.github/workflows/publish-artifacts.yml +++ b/.github/workflows/publish-artifacts.yml @@ -7,6 +7,16 @@ on: - cron: '30 18 * * *' workflow_dispatch: inputs: + JOBS_TO_RUN: + description: 'Jobs to run (manual trigger only)' + required: true + default: 'all' + type: choice + options: + - all + - wheel-only + - containers-only + - helm-chart-only CONTAINER_TAG: description: 'Custom tag for containers (optional)' required: false @@ -15,6 +25,26 @@ on: description: 'Artifactory version (optional, defaults to auto-generated from get_version.sh)' required: false default: '' + HELM_CHART_VERSION: + description: 'Helm chart version for NGC (optional, defaults to auto-generated from get_version.sh)' + required: false + default: '' + # Container-level selection (applies when JOBS_TO_RUN is 'all' or 'containers-only') + PUBLISH_RAG_SERVER: + description: 'Publish rag-server container' + required: false + default: true + type: boolean + PUBLISH_INGESTOR_SERVER: + description: 'Publish ingestor-server container' + required: false + default: true + type: boolean + PUBLISH_RAG_FRONTEND: + description: 'Publish rag-frontend container' + required: false + default: true + type: boolean env: RELEASE_TYPE: dev @@ -26,6 +56,7 @@ jobs: publish-wheel: name: Build and Publish Python Wheel runs-on: ubuntu-latest + if: github.event_name != 'workflow_dispatch' || github.event.inputs.JOBS_TO_RUN == 'all' || github.event.inputs.JOBS_TO_RUN == 'wheel-only' container: image: python:3.10 steps: @@ -106,6 +137,7 @@ jobs: publish-rag-server: name: Build and Publish RAG Server Container runs-on: ubuntu-latest + if: github.event_name != 'workflow_dispatch' || ((github.event.inputs.JOBS_TO_RUN == 'all' || github.event.inputs.JOBS_TO_RUN == 'containers-only') && github.event.inputs.PUBLISH_RAG_SERVER != 'false') steps: - name: Checkout code uses: actions/checkout@v4 @@ -164,6 +196,7 @@ jobs: publish-ingestor-server: name: Build and Publish Ingestor Server Container runs-on: ubuntu-latest + if: github.event_name != 'workflow_dispatch' || ((github.event.inputs.JOBS_TO_RUN == 'all' || github.event.inputs.JOBS_TO_RUN == 'containers-only') && github.event.inputs.PUBLISH_INGESTOR_SERVER != 'false') steps: - name: Checkout code uses: actions/checkout@v4 @@ -222,6 +255,7 @@ jobs: publish-rag-frontend: name: Build and Publish RAG Frontend Container runs-on: ubuntu-latest + if: github.event_name != 'workflow_dispatch' || ((github.event.inputs.JOBS_TO_RUN == 'all' || github.event.inputs.JOBS_TO_RUN == 'containers-only') && github.event.inputs.PUBLISH_RAG_FRONTEND != 'false') steps: - name: Checkout code uses: actions/checkout@v4 @@ -274,3 +308,83 @@ jobs: docker images | grep "rag-frontend" | awk '{print $3}' | xargs -r docker rmi -f || echo "No rag-frontend images to delete" docker system prune -f || true + # ============================================================================ + # PUBLISH HELM CHART TO NGC + # ============================================================================ + publish-helm-chart: + name: Build and Publish Helm Chart to NGC + runs-on: ubuntu-latest + if: github.event_name != 'workflow_dispatch' || github.event.inputs.JOBS_TO_RUN == 'all' || github.event.inputs.JOBS_TO_RUN == 'helm-chart-only' + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install Helm + uses: azure/setup-helm@v4 + with: + version: 'v3.17.0' + + - name: Install NGC CLI + env: + NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} + run: | + echo "Installing NGC CLI..." + wget --content-disposition https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/4.9.10/files/ngccli_linux.zip -O ngccli_linux.zip + unzip -o ngccli_linux.zip + chmod u+x ngc-cli/ngc + echo "$(pwd)/ngc-cli" >> $GITHUB_PATH + echo "NGC CLI installed successfully" + + - name: Determine Helm chart version + id: helm_version + run: | + if [ -n "${{ github.event.inputs.HELM_CHART_VERSION }}" ]; then + echo "Using custom Helm chart version: ${{ github.event.inputs.HELM_CHART_VERSION }}" + VERSION="${{ github.event.inputs.HELM_CHART_VERSION }}" + else + echo "Using auto-generated version from get_version.sh" + chmod +x ./ci/get_version.sh + VERSION=$(./ci/get_version.sh) + echo "Generated version: $VERSION" + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "HELM_CHART_VERSION=$VERSION" >> $GITHUB_ENV + + - name: Add Helm repositories + env: + NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} + run: | + cd deploy/helm + helm repo add nvidia-nim https://helm.ngc.nvidia.com/nim/nvidia/ --username='$oauthtoken' --password="$NGC_API_KEY" + helm repo add nim https://helm.ngc.nvidia.com/nim/ --username='$oauthtoken' --password="$NGC_API_KEY" + helm repo add nemo-microservices https://helm.ngc.nvidia.com/nvidia/nemo-microservices --username='$oauthtoken' --password="$NGC_API_KEY" + helm repo add baidu-nim https://helm.ngc.nvidia.com/nim/baidu --username='$oauthtoken' --password="$NGC_API_KEY" + helm repo add bitnami https://charts.bitnami.com/bitnami + helm repo add elastic https://helm.elastic.co + helm repo add otel https://open-telemetry.github.io/opentelemetry-helm-charts + helm repo add zipkin https://zipkin.io/zipkin-helm + helm repo add prometheus https://prometheus-community.github.io/helm-charts + helm repo update + + - name: Package Helm chart + env: + NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} + run: | + cd deploy/helm + helm dependency update nvidia-blueprint-rag + helm package nvidia-blueprint-rag/ --version "${{ env.HELM_CHART_VERSION }}" + CHART_TGZ=$(ls nvidia-blueprint-rag-*.tgz) + echo "Created: $CHART_TGZ" + + - name: Push Helm chart to NGC + env: + NGC_API_KEY: ${{ secrets.CI_NVSTAGING_BLUEPRINT_KEY }} + run: | + cd deploy/helm + CHART_TGZ="nvidia-blueprint-rag-${{ env.HELM_CHART_VERSION }}.tgz" + TARGET="nvstaging/blueprint/nvidia-blueprint-rag:${{ env.HELM_CHART_VERSION }}" + # Remove existing version to overwrite (ignore error if version does not exist) + ngc registry chart remove "$TARGET" --org nvstaging -y 2>/dev/null || true + ngc registry chart push "$TARGET" --source "$CHART_TGZ" --org nvstaging + echo "Helm chart published to NGC: $TARGET" + From b68c69fd21e38bcc2190a9838382c5c60be1d096 Mon Sep 17 00:00:00 2001 From: kumar-punit Date: Wed, 11 Mar 2026 21:43:03 +0530 Subject: [PATCH 53/81] Add rag-blueprint agent skill with CLAUDE.md and project config (#407) * Add rag-blueprint agent skill with CLAUDE.md and project config * Update query-and-conversation.md * Update notebooks.md Updated with NVIDIA style guide recommendations --------- Co-authored-by: Kurt Heiss --- .gitignore | 5 + AGENTS.md | 86 +++++++++++ CLAUDE.md | 84 ++++++++++ README.md | 23 +++ .../.agents/skills/rag-blueprint/SKILL.md | 136 ++++++++++++++++ .../references/configure/api-reference.md | 29 ++++ .../references/configure/data-catalog.md | 36 +++++ .../references/configure/evaluation.md | 26 ++++ .../references/configure/guardrails.md | 30 ++++ .../references/configure/ingestion.md | 53 +++++++ .../rag-blueprint/references/configure/mcp.md | 26 ++++ .../references/configure/migration.md | 35 +++++ .../configure/models-and-infrastructure.md | 68 ++++++++ .../references/configure/multimodal-query.md | 35 +++++ .../references/configure/notebooks.md | 50 ++++++ .../references/configure/observability.md | 29 ++++ .../configure/query-and-conversation.md | 82 ++++++++++ .../configure/reasoning-and-generation.md | 57 +++++++ .../configure/search-and-retrieval.md | 67 ++++++++ .../references/configure/summarization.md | 40 +++++ .../references/configure/user-interface.md | 27 ++++ .../rag-blueprint/references/configure/vlm.md | 56 +++++++ .../skills/rag-blueprint/references/deploy.md | 119 ++++++++++++++ .../references/deploy/docker-nvidia-hosted.md | 38 +++++ .../deploy/docker-retrieval-only.md | 37 +++++ .../references/deploy/docker-self-hosted.md | 49 ++++++ .../rag-blueprint/references/deploy/docker.md | 88 +++++++++++ .../references/deploy/helm-mig.md | 38 +++++ .../references/deploy/helm-standard.md | 51 ++++++ .../rag-blueprint/references/deploy/helm.md | 103 ++++++++++++ .../references/deploy/library-full.md | 43 ++++++ .../references/deploy/library-lite.md | 37 +++++ .../references/deploy/library.md | 54 +++++++ .../rag-blueprint/references/shutdown.md | 128 +++++++++++++++ .../rag-blueprint/references/troubleshoot.md | 146 ++++++++++++++++++ skill-source/README.md | 111 +++++++++++++ 36 files changed, 2122 insertions(+) create mode 100644 AGENTS.md create mode 100644 CLAUDE.md create mode 100644 skill-source/.agents/skills/rag-blueprint/SKILL.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/api-reference.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/data-catalog.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/evaluation.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/guardrails.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/ingestion.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/mcp.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/migration.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/models-and-infrastructure.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/multimodal-query.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/notebooks.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/observability.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/query-and-conversation.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/reasoning-and-generation.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/search-and-retrieval.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/summarization.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/user-interface.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/configure/vlm.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/deploy.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/deploy/docker-nvidia-hosted.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/deploy/docker-retrieval-only.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/deploy/docker-self-hosted.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/deploy/docker.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/deploy/helm-mig.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/deploy/helm-standard.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/deploy/helm.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/deploy/library-full.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/deploy/library-lite.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/deploy/library.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/shutdown.md create mode 100644 skill-source/.agents/skills/rag-blueprint/references/troubleshoot.md create mode 100644 skill-source/README.md diff --git a/.gitignore b/.gitignore index 9dded62bf..3611412e3 100644 --- a/.gitignore +++ b/.gitignore @@ -80,4 +80,9 @@ coverage/ cover/ *.log tests/data/ +# Agent skills (installed via npx skills add) +/.agents/ +/.claude/ +skills-lock.json + # Workbench Project Layout \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..183677906 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,86 @@ +# NVIDIA RAG Blueprint + +Reference implementation for a Retrieval Augmented Generation pipeline. Python 3.11+ backend (FastAPI + LangChain), React/TypeScript frontend, deployable via Docker Compose or Helm. + +## Project structure + +``` +src/nvidia_rag/ +├── rag_server/ # RAG query/response server (FastAPI) +├── ingestor_server/ # Document ingestion server (FastAPI) +└── utils/ # Shared utilities +frontend/ # React + TypeScript UI (pnpm) +deploy/ +├── compose/ # Docker Compose files and env configs +└── helm/ # Helm charts (standard + MIG-slicing) +docs/ # User-facing documentation (Sphinx, RST/MD) +tests/ +├── unit/ # No network calls allowed +└── integration/ # Network calls permitted +notebooks/ # Jupyter notebooks for evaluation and examples +``` + +## Development commands + +### Backend (Python) + +```bash +uv sync # Install all deps +uv run pytest tests/unit/ # Unit tests +uv run pytest tests/integration/ # Integration tests +ruff check --fix src/ # Lint + autofix +ruff format src/ # Format +pre-commit run --all-files # Run all pre-commit hooks +``` + +### Frontend (TypeScript) + +```bash +cd frontend +pnpm install +pnpm run dev # Dev server +pnpm run lint # ESLint +pnpm exec tsc --noEmit # Type check +pnpm run test:run # Tests +``` + +## Code conventions + +- **Python**: Ruff for linting and formatting (line-length 88, double quotes, space indent). Config in `pyproject.toml`. +- **Type hints**: Required on all function signatures. +- **Imports**: Sorted by isort via Ruff. No in-function imports. +- **Tests**: Mirror source tree (`src/nvidia_rag/rag_server/server.py` → `tests/unit/rag_server/test_server.py`). +- **Frontend**: ESLint + TypeScript strict mode. Function components with hooks. +- **Env files**: `deploy/compose/nvdev.env` (NVIDIA-hosted NIMs) and `deploy/compose/.env` (self-hosted). These are the source of truth for Docker deployments — shell-only exports are lost on restart. + +## Deployment modes + +1. **Docker Compose** — `deploy/compose/` with env-file configs. Multiple profiles: standard, retrieval-only, NVIDIA-hosted. +2. **Helm** — `deploy/helm/nvidia-blueprint-rag/` chart with `values.yaml`. Supports MIG GPU slicing via `deploy/helm/mig-slicing/`. +3. **Library** — Import `nvidia_rag` as a Python package for custom pipelines. + +## Key files + +- `pyproject.toml` — All Python deps, ruff config, project metadata +- `deploy/compose/nvdev.env` — Default env file for NVIDIA API Catalog deployments +- `src/nvidia_rag/rag_server/prompt.yaml` — System prompt templates +- `docs/support-matrix.md` — GPU requirements per deployment mode +- `docs/service-port-gpu-reference.md` — Port mappings and GPU assignments + +## PR and commit guidelines + +- Target the `develop` branch, never `main`. +- All commits must be signed off (DCO). +- Run `pre-commit run --all-files` before submitting. +- See `CONTRIBUTING.md` for full workflow. + +## Operations — `rag-blueprint` skill + +For any operational task — deploying, configuring, troubleshooting, or shutting down the RAG Blueprint — read and follow the skill at `.agents/skills/rag-blueprint/SKILL.md`. + +The skill handles: + +- **Deploy** — Docker Compose (standard, retrieval-only, NVIDIA-hosted), Helm, MIG-slicing, library mode +- **Configure** — VLM, guardrails, query rewriting, ingestion, search & retrieval, models, observability, summarization, multimodal, MCP, evaluation, notebooks, UI, and more +- **Troubleshoot** — Debug unhealthy services, container errors, GPU issues, connectivity failures +- **Shutdown** — Stop, tear down, and clean up services diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..e16f0c9d6 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,84 @@ +# NVIDIA RAG Blueprint + +Reference implementation for a Retrieval Augmented Generation pipeline. Python 3.11+ backend (FastAPI + LangChain), React/TypeScript frontend, deployable via Docker Compose or Helm. + +## Project structure + +``` +src/nvidia_rag/ +├── rag_server/ # RAG query/response server (FastAPI) +├── ingestor_server/ # Document ingestion server (FastAPI) +└── utils/ # Shared utilities +frontend/ # React + TypeScript UI (pnpm) +deploy/ +├── compose/ # Docker Compose files and env configs +└── helm/ # Helm charts (standard + MIG-slicing) +docs/ # User-facing documentation (Sphinx, RST/MD) +tests/ +├── unit/ # No network calls allowed +└── integration/ # Network calls permitted +notebooks/ # Jupyter notebooks for evaluation and examples +``` + +## Development commands + +### Backend (Python) + +```bash +uv sync # Install all deps +uv run pytest tests/unit/ # Unit tests +uv run pytest tests/integration/ # Integration tests +ruff check --fix src/ # Lint + autofix +ruff format src/ # Format +pre-commit run --all-files # Run all pre-commit hooks +``` + +### Frontend (TypeScript) + +```bash +cd frontend +pnpm install +pnpm run dev # Dev server +pnpm run lint # ESLint +pnpm exec tsc --noEmit # Type check +pnpm run test:run # Tests +``` + +## Code conventions + +- **Python**: Ruff for linting and formatting (line-length 88, double quotes, space indent). Config in `pyproject.toml`. +- **Type hints**: Required on all function signatures. +- **Imports**: Sorted by isort via Ruff. No in-function imports. +- **Tests**: Mirror source tree (`src/nvidia_rag/rag_server/server.py` → `tests/unit/rag_server/test_server.py`). +- **Frontend**: ESLint + TypeScript strict mode. Function components with hooks. +- **Env files**: `deploy/compose/nvdev.env` (NVIDIA-hosted NIMs) and `deploy/compose/.env` (self-hosted). These are the source of truth for Docker deployments — shell-only exports are lost on restart. + +## Deployment modes + +1. **Docker Compose** — `deploy/compose/` with env-file configs. Multiple profiles: standard, retrieval-only, NVIDIA-hosted. +2. **Helm** — `deploy/helm/nvidia-blueprint-rag/` chart with `values.yaml`. Supports MIG GPU slicing via `deploy/helm/mig-slicing/`. +3. **Library** — Import `nvidia_rag` as a Python package for custom pipelines. + +## Key files + +- `pyproject.toml` — All Python deps, ruff config, project metadata +- `deploy/compose/nvdev.env` — Default env file for NVIDIA API Catalog deployments +- `src/nvidia_rag/rag_server/prompt.yaml` — System prompt templates +- `docs/support-matrix.md` — GPU requirements per deployment mode +- `docs/service-port-gpu-reference.md` — Port mappings and GPU assignments + +## PR and commit guidelines + +- Target the `develop` branch, never `main`. +- All commits must be signed off (DCO). +- Run `pre-commit run --all-files` before submitting. +- See `CONTRIBUTING.md` for full workflow. + +## Operations — `/rag-blueprint` skill + +For any operational task, use the `rag-blueprint` skill (`.agents/skills/rag-blueprint/`). + +- **Deploy** — Docker Compose (standard, retrieval-only, NVIDIA-hosted), Helm, MIG-slicing, library mode +- **Configure** — VLM, guardrails, query rewriting, ingestion, search & retrieval, models, observability, summarization, multimodal, MCP, evaluation, notebooks, UI, and more +- **Troubleshoot** — Debug unhealthy services, container errors, GPU issues, connectivity failures +- **Shutdown** — Stop, tear down, and clean up services diff --git a/README.md b/README.md index 72c5ca5a3..c400dd410 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,29 @@ The following is a step-by-step explanation of the workflow from the end-user pe +## AI Agent Skill + +An agent skill is included that enables AI coding assistants (Claude Code, Cursor, etc.) to deploy, configure, troubleshoot, and manage the RAG Blueprint autonomously. + +### Install + +```bash +npx skills add . +``` + +This installs the `rag-blueprint` skill from `skill-source/`. After installation, the agent handles requests like: + +- *"Deploy RAG on Docker with NVIDIA-hosted models"* +- *"Enable VLM image captioning and restart the ingestor"* +- *"Ingestion failed for 3 files, can you check why?"* +- *"Switch from Docker to library mode"* +- *"Shut down all RAG services"* + +> **Note:** If the agent doesn't pick up the skill automatically (e.g., for short or ambiguous queries), invoke it explicitly with `/rag-blueprint `. + +For skill architecture details, see [`skill-source/README.md`](skill-source/README.md). + + ## Get Started With NVIDIA RAG Blueprint The recommended way to get started is to deploy the NVIDIA RAG Blueprint diff --git a/skill-source/.agents/skills/rag-blueprint/SKILL.md b/skill-source/.agents/skills/rag-blueprint/SKILL.md new file mode 100644 index 000000000..8f48d2858 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/SKILL.md @@ -0,0 +1,136 @@ +--- +name: rag-blueprint +description: "NVIDIA RAG Blueprint — deploy, configure, troubleshoot, and manage. Handles any RAG action: deploy, install, start, enable, disable, toggle, change, configure, troubleshoot, debug, fix, shutdown, stop, or tear down any RAG feature or service (VLM, guardrails, query rewriting, models, search, ingestion, observability, summarization, and more)." +argument-hint: deploy RAG | enable feature | disable feature | configure | troubleshoot | shutdown +allowed-tools: Bash(echo *), Bash(nvidia-smi *), Bash(curl *), Bash(docker ps *), Bash(docker exec *), Bash(docker info *), Bash(docker --version *), Bash(docker compose version *), Bash(docker logs *), Bash(docker system *), Bash(kubectl get *), Bash(kubectl describe *), Bash(kubectl version *), Bash(kubectl logs *), Bash(helm version *), Bash(helm list *), Bash(git rev-parse *), Bash(git describe *), Bash(git status *), Bash(python3 --version *), Bash(pip3 show *), Bash(df *), Bash(du *), Bash(cat /proc/*), Bash(cat /etc/os-release *), Bash(ss *), Bash(netstat *), Bash(ls *), Bash(grep *), Bash(lsof *), Bash(ps aux *), Read, Grep, Glob +license: Apache-2.0 +metadata: + author: nvidia-rag-team + version: "1.0" +--- + +# NVIDIA RAG Blueprint + +## Autonomy Principles + +- Auto-detect everything: GPU, VRAM, drivers, Docker, CUDA, disk, OS, ports, existing services, NGC key, repo state. +- If it can be checked with a command, check it — don't ask the user. +- Ask only when user action is required: providing an API key, confirming data deletion, or choosing between equally valid options. +- Once analysis is done, route to the correct workflow and execute. + +## Intent Detection + +Determine what the user wants and route immediately: + +| User Intent | Action | +|-------------|--------| +| Deploy, install, set up, start RAG | Read and follow `references/deploy.md` | +| Configure, enable, change, toggle a feature | Use the **Configure** section below | +| Troubleshoot, debug, fix, error, unhealthy | Read and follow `references/troubleshoot.md` | +| Stop, shutdown, tear down, clean up | Read and follow `references/shutdown.md` | + +If the intent is ambiguous, infer from context (e.g., "RAG isn't working" → troubleshoot; "get RAG running" → deploy). Only ask if genuinely unclear. + +--- + +## Configure + +Requires a running RAG deployment. If services are not running, deploy first via `references/deploy.md`. + +Match the user's request to a reference file, then read and follow it: + +| Feature Keywords | Reference | +|-----------------|-----------| +| VLM, VLM embeddings, image captioning | `references/configure/vlm.md` | +| NeMo Guardrails | `references/configure/guardrails.md` | +| Query rewriting, decomposition, multi-turn | `references/configure/query-and-conversation.md` | +| Ingestion (text-only, audio, Nemotron Parse, OCR, batch CLI, NV-Ingest, volume mount, performance) | `references/configure/ingestion.md` | +| Search, retrieval, hybrid search, multi-collection, metadata, filters, reranker, topK, accuracy/performance | `references/configure/search-and-retrieval.md` | +| LLM/embedding/ranking model changes, vector DB, Milvus/Elasticsearch auth, service keys, model profiles, ports/GPU | `references/configure/models-and-infrastructure.md` | +| Reasoning, self-reflection, prompts, generation params (tokens, temperature, citations), per-request LLM params | `references/configure/reasoning-and-generation.md` | +| Summarization | `references/configure/summarization.md` | +| Observability (tracing, Zipkin, Grafana, Prometheus) | `references/configure/observability.md` | +| Multimodal query (image + text) | `references/configure/multimodal-query.md` | +| Data catalog (collection/document metadata) | `references/configure/data-catalog.md` | +| User interface (UI settings) | `references/configure/user-interface.md` | +| API reference (endpoints, schemas) | `references/configure/api-reference.md` | +| Evaluation (RAGAS metrics) | `references/configure/evaluation.md` | +| MCP server & client, agent toolkit | `references/configure/mcp.md` | +| Migration (version upgrades) | `references/configure/migration.md` | +| Notebooks (setup and catalog) | `references/configure/notebooks.md` | + +### Configure Flow + +1. Match the user's request to a reference file from the table above. + +2. Detect what's running: + ```bash + echo "=== NIM ===" && docker ps --format '{{.Names}}' 2>/dev/null | grep -iE '(nim-llm|nemoretriever-embedding|nemoretriever-ranking|nemo-vlm|nemotron-vlm)' || echo "NO_LOCAL_NIMS"; echo "=== RAG ===" && docker ps --format '{{.Names}}' 2>/dev/null | grep -iE '(rag-server|ingestor-server|milvus)' || echo "NO_DOCKER_RAG"; echo "=== K8S ===" && kubectl get pods -n rag 2>/dev/null | head -5 || echo "NO_K8S"; echo "=== LIBRARY ===" && ps aux 2>/dev/null | grep -E '(nvidia_rag|uvicorn.*rag)' | grep -v grep || echo "NO_LIBRARY" + ``` + +3. Use this table to determine platform, deployment type, and where config lives: + + | Local NIMs running? | RAG services running? | Deployment Type | Config Location | + |---------------------|-----------------------|-----------------|-----------------| + | Yes (Docker) | Any | Self-hosted | `deploy/compose/.env` | + | No | Yes (Docker) | NVIDIA-hosted | `deploy/compose/nvdev.env` | + | Yes (K8s pods) | Any | Self-hosted | `values.yaml` (NIM sections) | + | No | Yes (K8s pods) | NVIDIA-hosted | `values.yaml` (envVars) | + | — | Library processes | Library mode | `notebooks/config.yaml` | + | No | No | Not running | Deploy first via `references/deploy.md` | + + Tell the user what you detected and ask to confirm. Example: "I see local NIM containers running (nim-llm-ms, nemoretriever-embedding-ms) — this is a self-hosted deployment. Config file is `deploy/compose/.env`. Correct?" + +4. Check current feature state before changing anything — read the config location from step 3, then cross-check the live service: + - Docker: `docker exec rag-server env 2>/dev/null | grep -E ""` + - Helm: `kubectl get pod -n rag -l app=rag-server -o jsonpath='{.items[0].spec.containers[0].env}' 2>/dev/null` + + If the config file and live service disagree, tell the user the service has stale config and will need a restart. + +5. If the feature needs extra GPUs, check availability against hardware restrictions (see below): + ```bash + nvidia-smi --query-gpu=index,name,memory.total,memory.used --format=csv,noheader 2>/dev/null || echo "NO_GPU" + ``` + +6. Read the reference file and apply changes: + - **Docker**: edit the env file (uncomment to enable, re-comment to disable — the env file is the source of truth). Then restart the affected service: + ``` + source && docker compose -f deploy/compose/ up -d + ``` + | Service | Compose File | + |---------|-------------| + | rag-server | `docker-compose-rag-server.yaml` | + | ingestor-server | `docker-compose-ingestor-server.yaml` | + | milvus, etcd, minio | `vectordb.yaml` | + | NIM containers (LLM, embedding, ranking, VLM, OCR) | `nims.yaml` | + | guardrails | `docker-compose-nemo-guardrails.yaml` | + | observability (Grafana, Prometheus, Zipkin) | `observability.yaml` | + - **Helm**: edit `values.yaml`, then upgrade: `helm upgrade rag -n rag -f values.yaml` + - **Library**: edit `notebooks/config.yaml`, then restart the Python process + +7. Verify: + - Docker: `docker ps --format "table {{.Names}}\t{{.Status}}" | head -20; curl -s http://localhost:8081/v1/health?check_dependencies=true 2>/dev/null | head -1` + - Helm: `kubectl get pods -n rag; kubectl rollout status deployment/rag-server -n rag --timeout=120s` + - Library: `curl -s http://localhost:8081/v1/health 2>/dev/null | head -1` + +8. If restart fails, read `references/troubleshoot.md`. If multiple features requested, repeat from step 1 for each. + +### When User Says "Configure" Without Specifics + +Run steps 2–3 above, then read the identified config file to list what's currently enabled: +```bash +grep -E "^(export )?(ENABLE_|APP_)" 2>/dev/null | sort +``` +Summarize what's running and enabled, then ask which feature to change. + +--- + +## Hardware Restrictions + +Read `docs/support-matrix.md` for current GPU requirements per deployment mode. +Read `docs/service-port-gpu-reference.md` for port mappings and GPU assignments. + +| GPU | Feature Restrictions | +|-----|---------------------| +| B200 | No VLM, No Guardrails, No Nemotron Parse. May need multi-GPU LLM (`LLM_MS_GPU_ID`). | +| RTX PRO 6000 | No Nemotron Parse. No Audio on Helm. | diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/api-reference.md b/skill-source/.agents/skills/rag-blueprint/references/configure/api-reference.md new file mode 100644 index 000000000..056d814ba --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/api-reference.md @@ -0,0 +1,29 @@ +# API Reference + +## When to Use +- User needs to call RAG or Ingestor APIs directly +- User asks about endpoints, request/response formats, or task status tracking + +## Process +1. Read `docs/api-rag.md` for RAG server endpoints (port 8081) +2. Read `docs/api-ingestor.md` for Ingestor server endpoints (port 8082) +3. Consult OpenAPI schemas for exact request/response shapes + +## Agent-Specific Notes +- RAG Server runs on port 8081: `/v1/generate`, `/v1/search`, `/v1/health`, `/v1/configuration`, `/v1/metrics`, `/v1/summary` +- Ingestor Server runs on port 8082: `/v1/documents`, `/v1/collection`, `/v1/collections`, `/v1/status` +- `POST /v1/documents` returns a `task_id` — poll `GET /v1/status?task_id=` for progress +- Task states: `PENDING` → `FINISHED` or `FAILED` (also `UNKNOWN` if not found) +- NV-Ingest extraction states: `not_started` → `submitted` → `processing` → `completed` or `failed` +- Max file size: 400 MB per document +- Full health check: `GET /v1/health?check_dependencies=true` + +## Notebooks +- `notebooks/ingestion_api_usage.ipynb` — ingestion API usage examples +- `notebooks/retriever_api_usage.ipynb` — RAG retriever API: search and query examples + +## Source Documentation +- `docs/api-rag.md` -- RAG server API details +- `docs/api-ingestor.md` -- Ingestor server API details +- `docs/api_reference/openapi_schema_rag_server.json` -- RAG server OpenAPI schema +- `docs/api_reference/openapi_schema_ingestor_server.json` -- Ingestor server OpenAPI schema diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/data-catalog.md b/skill-source/.agents/skills/rag-blueprint/references/configure/data-catalog.md new file mode 100644 index 000000000..7b2d6c106 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/data-catalog.md @@ -0,0 +1,36 @@ +# Data Catalog + +## When to Use +- User wants to manage collection or document metadata for governance +- User asks about tagging, ownership, or lifecycle status of collections +- User wants to list or update collection metadata + +## Restrictions +- None — available automatically after deployment, no additional configuration needed +- Works with both Milvus and Elasticsearch (full feature parity) + +## Process +1. Read `docs/data-catalog.md` for full API reference, field definitions, and examples +2. All endpoints are on the ingestor server (port `8082`) +3. Use PATCH endpoints for updates (merge updates — only provided fields change) + +## Decision Table + +| Goal | Source Doc | Key Action | +|------|-----------|------------| +| Add governance metadata | `docs/data-catalog.md` | POST `/v1/collection` with description, tags, owner | +| Update lifecycle status | `docs/data-catalog.md` | PATCH with `status: "Archived"` | +| Track content types | `docs/data-catalog.md` | Read auto-populated `has_tables`, `has_images` metrics | +| Filter during retrieval | See custom metadata docs | Use `metadata_schema` + `filter_expr` (not data catalog) | + +## Agent-Specific Notes +- Auto-populated metrics (`number_of_files`, `last_indexed`, `has_tables`, etc.) are system-set — not user-editable +- `date_created` and `last_updated` timestamps are automatic +- PATCH is a merge update — omitted fields keep current values +- Different from custom metadata: catalog = governance/discovery, custom metadata = retrieval filtering + +## Notebooks +- `notebooks/ingestion_api_usage.ipynb` — ingestion and collection management examples + +## Source Documentation +- `docs/data-catalog.md` — full API reference, catalog fields, auto-populated metrics, Python client examples diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/evaluation.md b/skill-source/.agents/skills/rag-blueprint/references/configure/evaluation.md new file mode 100644 index 000000000..90f9c6206 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/evaluation.md @@ -0,0 +1,26 @@ +# Evaluation + +## When to Use +- User wants to measure RAG pipeline quality +- User asks about accuracy, relevancy, groundedness, or recall metrics + +## Process +1. Read `docs/evaluate.md` for full evaluation methodology and setup +2. Choose the appropriate notebook based on metrics needed +3. Run evaluation against the deployed RAG pipeline + +## Agent-Specific Notes +- Uses RAGAS framework for all metrics +- Answer Accuracy, Context Relevancy, and Groundedness are covered in one notebook +- Recall is measured separately at top-k cutoffs (1, 3, 5, 10) + +## Notebooks +| Notebook | Metrics | +|----------|---------| +| `notebooks/evaluation_01_ragas.ipynb` | Answer Accuracy, Context Relevancy, Groundedness | +| `notebooks/evaluation_02_recall.ipynb` | Recall at top-k cutoffs | + +## Source Documentation +- `docs/evaluate.md` -- full evaluation guide and metric definitions +- [RAGAS documentation](https://docs.ragas.io/en/stable/) +- [NVIDIA RAGAS metrics](https://docs.ragas.io/en/stable/concepts/metrics/available_metrics/nvidia_metrics/) diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/guardrails.md b/skill-source/.agents/skills/rag-blueprint/references/configure/guardrails.md new file mode 100644 index 000000000..309a18611 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/guardrails.md @@ -0,0 +1,30 @@ +# NeMo Guardrails + +## When to Use +- User wants content safety, topic control, or jailbreak prevention +- User asks to enable/disable guardrails + +## Restrictions +- Not available on B200 GPUs +- Requires 2 extra GPUs with 48GB+ each (H100, A100 SXM 80GB, or RTX PRO 6000) +- Not supported in library mode or Helm deployments +- Jailbreak detection model not yet available out-of-the-box + +## Process + +1. Detect the deployment mode (guardrails are Docker-only — not supported on Helm or library mode). Edit the active env file for Docker +2. Read `docs/nemo-guardrails.md` for full setup and configuration +3. Choose deployment mode: self-hosted (local NIMs) or cloud-hosted (NVIDIA API) +4. For self-hosted: assign GPU IDs — read `docs/service-port-gpu-reference.md` for default GPU assignments and adjust for your system +5. Verify all three services healthy: `nemo-guardrails-microservice`, content-safety NIM, topic-control NIM +6. Enable in UI: Settings > Output Preferences > Guardrails toggle + +## Agent-Specific Notes +- Cloud mode (`nemoguard_cloud` config) skips local NIM containers — only the microservice is needed +- Per-request toggle via `enable_guardrails` in `/generate` body requires server-level `ENABLE_GUARDRAILS=true` first +- Override guardrails URL with `NEMO_GUARDRAILS_URL` if running on a different host +- Content-safety and topic-control models are trained on single-turn data — multi-turn conversations may get inconsistent safety classifications +- Current guardrails only produce simple refusal responses ("I'm sorry. I can't respond to that.") + +## Source Documentation +- `docs/nemo-guardrails.md` -- full setup, configuration, and customization of guardrail rules diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/ingestion.md b/skill-source/.agents/skills/rag-blueprint/references/configure/ingestion.md new file mode 100644 index 000000000..ec5e6251f --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/ingestion.md @@ -0,0 +1,53 @@ +# Ingestion: Text-Only, Audio, Nemotron Parse, OCR & Batch + +## When to Use +User wants to configure ingestion mode (text-only, audio, Nemotron Parse), switch OCR engines, save extraction results to disk, use standalone NV-Ingest, tune ingestion performance, or run batch ingestion. + +## Restrictions +- Nemotron Parse: not available on B200 or RTX PRO 6000 GPUs (requires H100 or A100 SXM 80GB) +- Audio on Helm: not supported on RTX PRO 6000 +- Nemotron Parse GPU conflict: read `docs/service-port-gpu-reference.md` for default GPU assignments. Nemotron Parse defaults to the same GPU as LLM — reassign on limited-GPU systems + +## Process + +1. Detect the deployment mode (Docker self-hosted / NVIDIA-hosted / Helm / Library). Docker: edit the active env file. Helm: edit `values.yaml`. Library: edit `notebooks/config.yaml` +2. Read the relevant source doc for detailed configuration +3. Apply the required env vars to the active config, restart ingestor (and NIM services if enabling new profiles) +4. Verify: upload a test document and check ingestion status + +## Decision Table + +| Goal | Source Doc | Key Action | +|------|-----------|------------| +| Text-only ingestion | `docs/text_only_ingest.md` | Set extract vars to False, set `COMPONENTS_TO_READY_CHECK=""` | +| Audio ingestion | `docs/audio_ingestion.md` | Start audio NIM (`--profile audio`), set `AUDIO_MS_GPU_ID` | +| Nemotron Parse | `docs/nemotron-parse-extraction.md` | `APP_NVINGEST_PDFEXTRACTMETHOD=nemotron_parse`, start NIM | +| OCR config/switch | `docs/nemoretriever-ocr.md` | Switch between NeMo Retriever OCR and Paddle OCR | +| Save to disk | `docs/mount-ingestor-volume.md` | `APP_NVINGEST_SAVETODISK=True`, mount volume | +| Standalone NV-Ingest | `docs/nv-ingest-standalone.md` | Direct Python client, no full ingestor server | +| Batch ingestion | See `scripts/batch_ingestion.py` | `python scripts/batch_ingestion.py --folder ... --collection-name ...` | +| Tune performance | `docs/accuracy_perf.md` | Adjust chunk size, overlap, batch settings | +| Summarization at ingest | `references/configure/summarization.md` | `generate_summary: true` in upload payload | + +## Agent-Specific Notes + +- Text-only mode: set `COMPONENTS_TO_READY_CHECK=""` in the active env file so NV-Ingest does not wait for disabled extraction services. If the compose file hardcodes `COMPONENTS_TO_READY_CHECK=ALL`, update it to `${COMPONENTS_TO_READY_CHECK:-ALL}` so the env var takes effect +- Use `--profile rag` with nims.yaml to skip OCR/detection NIMs in text-only mode +- Audio formats supported: `.mp3`, `.wav`, `.mp4`, `.avi`, `.mov`, `.mkv` +- Riva ASR requires ~8GB VRAM +- NeMo Retriever OCR is 2x+ faster than Paddle OCR but needs 8GB vs 3GB VRAM +- Batch CLI: `pip install -r scripts/requirements.txt` first; idempotent (skips already-ingested files) +- MIG deployments: reduce batch sizes for large bulk ingestion jobs + +## Notebooks +- `notebooks/ingestion_api_usage.ipynb` — Ingestor API: collections, uploads, document management + +## Source Documentation +- `docs/text_only_ingest.md` — Text-only ingestion (skip OCR/detection) +- `docs/audio_ingestion.md` — Audio/video ingestion via ASR +- `docs/nemotron-parse-extraction.md` — Nemotron Parse PDF extraction +- `docs/nemoretriever-ocr.md` — OCR configuration and switching +- `docs/mount-ingestor-volume.md` — Volume mount for extraction results +- `docs/nv-ingest-standalone.md` — Standalone NV-Ingest without ingestor server +- `docs/accuracy_perf.md` — Ingestion tuning settings (chunk size, overlap, batch params) +- `docs/service-port-gpu-reference.md` — OCR port mappings and GPU assignments diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/mcp.md b/skill-source/.agents/skills/rag-blueprint/references/configure/mcp.md new file mode 100644 index 000000000..0fc9516e8 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/mcp.md @@ -0,0 +1,26 @@ +# MCP Server & Client + +## When to Use +- User wants to expose RAG APIs as MCP tools for agentic workflows +- User asks about MCP transport modes, NeMo Agent Toolkit integration, or ReAct agents + +## Process +1. Read `docs/mcp.md` for full MCP server/client setup and configuration +2. Choose transport mode: `sse`, `streamable_http`, or `stdio` +3. Run MCP server from `examples/nvidia_rag_mcp/mcp_server.py` +4. For agentic RAG, see ReAct agent example in `examples/rag_react_agent/` + +## Agent-Specific Notes +- MCP wraps both RAG tools (`generate`, `search`, `get_summary`) and Ingestor tools (`create_collection`, `upload_documents`, etc.) via FastMCP +- `stdio` transport does not require a running server — client spawns it directly +- ReAct agent requires: Python 3.11+, `NVIDIA_API_KEY`, and data already ingested into Milvus +- Configure Milvus endpoint in `examples/rag_react_agent/src/rag_react_agent/configs/config.yml` or via `APP_VECTORSTORE_URL` + +## Notebooks +| Notebook | Description | +|----------|-------------| +| `notebooks/mcp_server_usage.ipynb` | End-to-end MCP workflow: collection creation, upload, RAG queries | +| `notebooks/nat_mcp_integration.ipynb` | NeMo Agent Toolkit integration with RAG MCP server | + +## Source Documentation +- `docs/mcp.md` -- full MCP server/client documentation and transport configuration diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/migration.md b/skill-source/.agents/skills/rag-blueprint/references/configure/migration.md new file mode 100644 index 000000000..c56e020ee --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/migration.md @@ -0,0 +1,35 @@ +# Migration Guide + +## When to Use +- User is upgrading between RAG Blueprint versions +- User encounters breaking API changes or deprecated endpoints after an update + +## Process +1. Read `docs/migration_guide.md` for full version-by-version migration details +2. Identify the user's current and target versions +3. Apply changes sequentially for each version gap + +## Agent-Specific Notes + +### v2.2.0 → v2.3.0 +- New `confidence_threshold` field in `/generate` and `/search` (0.0–1.0, default 0.0) +- New `summary_options` parameter with `page_filter`, `shallow_summary`, `summarization_strategy` +- `SUMMARY_LLM_MAX_CHUNK_LENGTH` and `SUMMARY_CHUNK_OVERLAP` changed from character-based to token-based — divide old values by ~4 + +### v2.1.0 → v2.2.0 +- Added `generate_summary` to `/documents`, new `GET /summary` endpoint +- `POST /collection` (singular) replaces `POST /collections` for single collection creation +- `collection_names: List[str]` replaces `collection_name: str` in `/generate` and `/search` + +### v2.0.0 → v2.1.0 +- `POST /documents` gained `blocking: bool` (default `True`); use `false` + `GET /status` for async + +### v1.0.0 → v2.0.0 (Breaking) +- Single server split into RAG Server (port 8081) and Ingestion Server (port 8082) +- Collections must be explicitly created before uploading documents +- Default changed from cloud-hosted to on-prem models + +## Source Documentation +- `docs/migration_guide.md` — Full migration guide with examples and env var changes +- `docs/release-notes.md` — Release notes and version history +- `docs/query-to-answer-pipeline.md` — Query-to-answer pipeline architecture overview diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/models-and-infrastructure.md b/skill-source/.agents/skills/rag-blueprint/references/configure/models-and-infrastructure.md new file mode 100644 index 000000000..68add08bc --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/models-and-infrastructure.md @@ -0,0 +1,68 @@ +# Models, Vector DB & Service API Keys + +## When to Use +User wants to change LLM, embedding, or ranking models; switch vector DB (Milvus/Elasticsearch); configure Milvus auth, GPU mode, or custom endpoints; set service-specific API keys; or build a custom VDB operator. + +## Process + +Detect the deployment mode before making changes. Docker: edit the active env file. Helm: edit `values.yaml` under `nimOperator` and `envVars` sections. Library: edit `notebooks/config.yaml`. + +### Change Models (LLM, Embedding, Ranking) +1. Read `docs/change-model.md` for full model change instructions +2. Read `docs/model-profiles.md` for NIM profile selection and GPU-specific profiles +3. Key env vars: `APP_LLM_MODELNAME`, `APP_EMBEDDINGS_MODELNAME`, `APP_RANKING_MODELNAME` +4. Embedding model change requires re-ingesting all documents — update `APP_EMBEDDINGS_DIMENSIONS` to match +5. Restart affected services (RAG server + ingestor for embedding changes) +6. Verify via health endpoint + +### Switch Vector DB (Milvus to Elasticsearch) +1. Read `docs/change-vectordb.md` for full setup (Docker and Helm) +2. Key env vars: `APP_VECTORSTORE_URL`, `APP_VECTORSTORE_NAME` +3. Data is not migrated — re-ingest all documents after switching +4. Elasticsearch requires port 9200; check for conflicts + +### Milvus Configuration +1. Read `docs/milvus-configuration.md` for indexing, GPU, auth, and tuning +2. Read `docs/milvus-schema.md` for collection schema requirements +3. CPU mode: set `APP_VECTORSTORE_ENABLEGPUSEARCH=False`, `APP_VECTORSTORE_ENABLEGPUINDEX=False`, change Milvus image to non-GPU +4. Auth: download milvus.yaml, enable `authorizationEnabled`, set password before first deployment + +### API Keys +1. Read `docs/api-key.md` for NGC API key setup and per-service keys +2. Fallback order: service-specific key > `NVIDIA_API_KEY` > `NGC_API_KEY` +3. Per-service keys: `APP_LLM_APIKEY`, `APP_EMBEDDINGS_APIKEY`, `APP_RANKING_APIKEY`, `APP_VLM_APIKEY`, etc. + +## Decision Table + +| Goal | Source Doc | Key Action | +|------|-----------|------------| +| Change LLM | `docs/change-model.md` | Set `APP_LLM_MODELNAME`, restart RAG server | +| Change embedding | `docs/change-model.md` | Set `APP_EMBEDDINGS_MODELNAME` + `APP_EMBEDDINGS_DIMENSIONS`, re-ingest | +| Change reranker | `docs/change-model.md` | Set `APP_RANKING_MODELNAME`, restart RAG server | +| Switch to Elasticsearch | `docs/change-vectordb.md` | Create data dir, start ES profile, set env vars, re-ingest | +| Milvus auth | `docs/milvus-configuration.md` | Download config, enable auth, mount volume | +| Milvus CPU mode | `docs/milvus-configuration.md` | Change image, disable GPU env vars | +| Custom VDB | `docs/change-vectordb.md` | Implement `VDBRag`, register in `__init__.py` | +| NIM profiles | `docs/model-profiles.md` | List profiles, set `NIM_MODEL_PROFILE` | +| Service API keys | `docs/api-key.md` | Set per-service `*_APIKEY` vars | +| Collection schema | `docs/milvus-schema.md` | Required fields: pk, vector, text, source, content_metadata | + +## Agent-Specific Notes + +- Nemotron-3-Nano naming: `nvidia/nemotron-3-nano-30b-a3b` (NVIDIA-hosted) vs `nvidia/nemotron-3-nano` (self-hosted NIM) — same model, different names +- Helm model changes go in `values.yaml` under `nimOperator` and `envVars` sections +- Custom VDB operator requires implementing `VDBRag` base class — see `docs/change-vectordb.md` "Custom Vector Database Operator" section +- VDB auth tokens can be passed per-request via `Authorization: Bearer ` header +- Milvus password persists in etcd volume — to change after deployment, must delete volumes (destroys data) + +## Notebooks +- `notebooks/building_rag_vdb_operator.ipynb` — Custom VDB operator implementation (OpenSearch example) + +## Source Documentation +- `docs/change-model.md` — Model changes (LLM, embedding, ranking, NIM images) +- `docs/change-vectordb.md` — Vector DB switching, Elasticsearch setup, custom VDB operator +- `docs/milvus-configuration.md` — Milvus indexing, GPU config, auth, tuning +- `docs/milvus-schema.md` — Collection schema fields and requirements +- `docs/model-profiles.md` — NIM profile definitions and selection +- `docs/api-key.md` — NGC API key setup, per-service keys, fallback order +- `docs/service-port-gpu-reference.md` — Port mappings and GPU assignments for all services diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/multimodal-query.md b/skill-source/.agents/skills/rag-blueprint/references/configure/multimodal-query.md new file mode 100644 index 000000000..783b6c209 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/multimodal-query.md @@ -0,0 +1,35 @@ +# Multimodal Query (Image + Text) + +## When to Use +- User wants to query knowledge base with images and text together +- User asks about VLM (Vision Language Model) deployment for RAG +- User wants image-based document understanding or visual Q&A + +## Restrictions +- Not available with Elasticsearch — Milvus only +- Reranker must be disabled (`ENABLE_RERANKER=false`) +- Summarization not supported (VLM replaces LLM) +- On-prem: requires NVIDIA H100 or A100 SXM 80GB GPU +- Single-page retrieval only — image queries return content from one page per document + +## Process +1. Detect the deployment mode (Docker / Helm / Library). Docker: edit the active env file. Helm: edit `values.yaml`. Library: edit `notebooks/config.yaml` +2. Read `docs/multimodal-query.md` for full env var configuration and commands +3. Choose variant: self-hosted (Docker), NVIDIA-hosted (cloud), or Helm +4. Deploy VLM + VLM Embedding NIMs per source doc instructions +5. Set VLM env vars in the active config and switch embedding model to VLM embedding +6. Restart ingestor + RAG server (Docker: add `--build` flag) and verify + +## Agent-Specific Notes +- Must select a collection before querying — queries without collection return no results +- First VLM deployment: model downloads take 10–20 min (~10GB+) +- `VLM_MS_GPU_ID` — read `docs/service-port-gpu-reference.md` for the default GPU assignment and override if needed +- Cloud rate limits apply for ingestion of >10 files +- For Helm with MIG: ensure dedicated MIG slice is assigned to VLM +- Image extraction must be enabled: `APP_NVINGEST_EXTRACTIMAGES=True`, `APP_NVINGEST_IMAGE_ELEMENTS_MODALITY=image` + +## Notebooks +- `notebooks/image_input.ipynb` — end-to-end multimodal query examples, image upload, VLM querying + +## Source Documentation +- `docs/multimodal-query.md` — full Docker/cloud/Helm configuration, env vars, API usage, limitations diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/notebooks.md b/skill-source/.agents/skills/rag-blueprint/references/configure/notebooks.md new file mode 100644 index 000000000..544de03bb --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/notebooks.md @@ -0,0 +1,50 @@ +# Notebooks + +## When to Use +- Hands-on examples of NVIDIA RAG Blueprint features are needed +- There are questions about Jupyter notebooks, tutorials, or code samples + +## Process +1. Read `docs/notebooks.md` for full notebook descriptions and prerequisites. +2. Set up the environment: virtualenv, `jupyterlab`, and `git lfs pull` for test data. +3. Open JupyterLab at `http://:8889`. + +## Agent-Specific Notes +- Git LFS is required because several notebooks rely on large data files (`git lfs install && git lfs pull`). +- In Docker mode, deploy NVIDIA RAG Blueprint first, then run notebooks against the running services. +- In library mode, use `rag_library_usage.ipynb` (full) or `rag_library_lite_usage.ipynb` (containerless). +- The custom VDB operator notebook requires Docker for OpenSearch services. + +## Notebook Catalog + +### Beginner +| Notebook | Topic | +|-----------------------------|-------------------------------------| +| `ingestion_api_usage.ipynb` | Document ingestion through the API | +| `retriever_api_usage.ipynb` | Search and retrieval API | +| `image_input.ipynb` | Image upload and multimodal queries | + +### Intermediate +| Notebook | Topic | +|--------------------------------|----------------------------------------| +| `summarization.ipynb` | Document summarization strategies | +| `evaluation_01_ragas.ipynb` | RAGAS accuracy, relevancy, groundedness| +| `evaluation_02_recall.ipynb` | Recall at top-k cutoffs | +| `nb_metadata.ipynb` | Custom metadata and filtered retrieval | +| `rag_library_usage.ipynb` | Full library mode end-to-end | +| `rag_library_lite_usage.ipynb` | Lite, containerless library mode | + +### Advanced +| Notebook | Topic | +|-----------------------------------|-------------------------------------| +| `building_rag_vdb_operator.ipynb` | Custom OpenSearch VDB operator | +| `mcp_server_usage.ipynb` | MCP server with transport modes | +| `nat_mcp_integration.ipynb` | NeMo Agent Toolkit plus MCP | + +### Deployment +| Notebook | Topic | +|--------------------|-----------------------| +| `launchable.ipynb` | Brev cloud deployment | + +## Source Documentation +- `docs/notebooks.md` — full notebook descriptions, setup, and prerequisites. diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/observability.md b/skill-source/.agents/skills/rag-blueprint/references/configure/observability.md new file mode 100644 index 000000000..5b291d339 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/observability.md @@ -0,0 +1,29 @@ +# Observability + +## When to Use +- User wants tracing, metrics, or monitoring for the RAG pipeline +- User asks about latency debugging, Zipkin, Grafana, or Prometheus + +## Process +1. Detect the deployment mode. Docker: edit the active env file. Helm: edit `values.yaml`. Library: edit `notebooks/config.yaml` +2. Read `docs/observability.md` for full setup (Docker and Helm) +3. Set `OPENTELEMETRY_CONFIG_FILE` and `APP_TRACING_ENABLED=True` in the active config +4. Start observability stack and restart RAG server +5. Import Grafana dashboard from `deploy/config/rag-metrics-dashboard.json` + +## Agent-Specific Notes +- Library mode: set `OPENTELEMETRY_CONFIG_FILE` in the environment for tracing; the Docker-based Prometheus/Grafana stack is independent +- Helm: Prometheus Operator CRDs must be installed before deploying with observability enabled +- Default Grafana credentials: `admin` / `admin` +- Zipkin spans cover: `query-rewriter`, `retriever`, `context-reranker`, `llm-stream` +- Span I/O visible via `traceloop.entity.input` / `traceloop.entity.output` fields + +### Quick Latency Triage +| Symptom | Check | +|---------|-------| +| Slow first token | `rag_ttft_ms` — compare retriever and reranker spans | +| Slow full response | `llm_generation_time_ms` / `llm-stream` span | +| Retrieval heavy | Compare `retrieval_time_ms` vs `context_reranker_time_ms` | + +## Source Documentation +- `docs/observability.md` -- full Docker/Helm setup, env vars, metrics reference, and dashboard import diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/query-and-conversation.md b/skill-source/.agents/skills/rag-blueprint/references/configure/query-and-conversation.md new file mode 100644 index 000000000..2f092cd97 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/query-and-conversation.md @@ -0,0 +1,82 @@ +```markdown +# Query Rewriting, Query Decomposition, and Multi-Turn + +Use these features when you want the system to understand follow-up questions, rewrite queries for better retrieval, or break complex questions into smaller parts. + +## When to use + +Use these settings when: + +- You want to enable multi-turn conversations or support follow-up questions. +- You want query rewriting to improve retrieval accuracy. +- You need complex multi-hop query decomposition. +- You are configuring or debugging conversation history behavior.[file:1] + +## Restrictions + +- Query rewriting and multi-turn both require `CONVERSATION_HISTORY > 0`. If it is set to 0, query rewriting has no effect.[file:1] +- Query decomposition works only when `use_knowledge_base=true` and with a single collection.[file:1] +- On Helm, query rewriting is supported only with an on-prem LLM, not with cloud-hosted models.[file:1] + +## Dependencies + +`CONVERSATION_HISTORY` is shared by query rewriting and multi-turn, so changing it affects both behaviors. + +| Setting | Depends on | Side effect when changed | +|-------------------------|----------------------------|-----------------------------------------------------------| +| `ENABLE_QUERYREWRITER` | `CONVERSATION_HISTORY > 0` | Enabling requires conversation history; disabling has no side effects | +| `CONVERSATION_HISTORY` | — | Setting to `0` also effectively disables query rewriting |[file:1] + +## Process + +First detect the deployment mode. +- Docker: edit the active environment file. +- Helm: edit `values.yaml`. +- Library: edit `notebooks/config.yaml`.[file:1] + +### Query rewriting + +1. Review `docs/multiturn.md` for full configuration details.[file:1] +2. To enable, set `ENABLE_QUERYREWRITER=True`. If `CONVERSATION_HISTORY` is `0`, set it to `5` or another positive value.[file:1] +3. To disable, unset or comment out `ENABLE_QUERYREWRITER`.[file:1] +4. Restart the RAG server.[file:1] + +### Multi-turn + +1. Review `docs/multiturn.md` for configuration, retrieval strategies, and API usage.[file:1] +2. To enable, set `CONVERSATION_HISTORY > 0` and choose the retrieval strategy you want to use.[file:1] +3. To disable, set `CONVERSATION_HISTORY=0`.[file:1] +4. Restart the RAG server.[file:1] + +### Query decomposition + +1. Review `docs/query_decomposition.md` for the decomposition algorithm, limitations, and examples.[file:1] +2. Set `ENABLE_QUERY_DECOMPOSITION=true` and `MAX_RECURSION_DEPTH=3` (or a different depth that fits your use case).[file:1] +3. Restart the RAG server.[file:1] + +## Decision table + +| Goal | Source doc | Key settings | +|-------------------------------|----------------------------|-----------------------------------------------------------| +| Multi-turn with best accuracy | `docs/multiturn.md` | `CONVERSATION_HISTORY=5`, `ENABLE_QUERYREWRITER=True` | +| Multi-turn with low latency | `docs/multiturn.md` | `CONVERSATION_HISTORY=5`, `MULTITURN_RETRIEVER_SIMPLE=True` | +| Complex multi-hop queries | `docs/query_decomposition.md` | `ENABLE_QUERY_DECOMPOSITION=true`, `MAX_RECURSION_DEPTH=3` | +| Disable multi-turn (default) | — | `CONVERSATION_HISTORY=0` |[file:1] + +## Agent-specific notes + +- `MULTITURN_RETRIEVER_SIMPLE` only applies when query rewriting is disabled. If both are configured, query rewriting takes precedence.[file:1] +- You can toggle query rewriting per request by setting `enable_query_rewriting: true` in `POST /generate`, but `CONVERSATION_HISTORY` must still be greater than 0.[file:1] +- By default, multi-turn is disabled with `CONVERSATION_HISTORY=0`.[file:1] +- Query decomposition adds latency and is most useful for multi-hop queries that involve multiple entities or steps.[file:1] +- In library mode, configure these settings in `notebooks/config.yaml` instead of using environment variables.[file:1] + +## Notebooks + +- `notebooks/retriever_api_usage.ipynb`: RAG retriever API usage with search and end-to-end query examples.[file:1] + +## Source documentation + +- `docs/query_decomposition.md`: Decomposition algorithm details, when to use it, and recursion depth guidance.[file:1] +- `docs/multiturn.md`: Conversation history behavior, retrieval strategies, API usage, and Helm configuration.[file:1] +``` diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/reasoning-and-generation.md b/skill-source/.agents/skills/rag-blueprint/references/configure/reasoning-and-generation.md new file mode 100644 index 000000000..d5bb24114 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/reasoning-and-generation.md @@ -0,0 +1,57 @@ +# Reasoning, Self-Reflection & Prompt Customization + +## When to Use +User wants to enable reasoning/thinking mode, configure self-reflection, customize prompts, adjust generation parameters (max tokens, temperature, citations), or understand thinking budget options. + +## Process +1. Detect the deployment mode (Docker / Helm / Library). Docker: edit the active env file. Helm: edit `values.yaml`. Library: edit `notebooks/config.yaml` +2. Read the relevant source doc for the specific feature +3. Apply env vars to the active config or edit prompt files, restart RAG server +4. Prompt changes require `--build` flag (Docker); env var changes only need restart +5. Verify: test with a query and check for reasoning output or changed behavior + +## Decision Table + +| Goal | Source Doc | Key Action | +|------|-----------|------------| +| Enable reasoning (Nemotron 1.5) | `docs/enable-nemotron-thinking.md` | Edit `prompt.yaml`: `/no_think` → `/think`, set temperature | +| Enable reasoning (Nano 30B) | `docs/enable-nemotron-thinking.md` | `ENABLE_NEMOTRON_3_NANO_THINKING=true` | +| Self-reflection | `docs/self-reflection.md` | `ENABLE_REFLECTION=true`, set thresholds | +| Prompt customization | `docs/prompt-customization.md` | `PROMPT_CONFIG_FILE=/path/to/custom.yaml` or edit prompt.yaml | +| Generation parameters | `docs/llm-params.md` | `LLM_MAX_TOKENS`, `LLM_TEMPERATURE`, `ENABLE_CITATIONS` | +| Per-request overrides | `docs/llm-params.md` | `temperature`, `top_p`, `max_tokens`, `stop` in API payload | + +## Agent-Specific Notes + +- Prompt changes need `--build` flag on restart; env var changes do not +- Self-reflection: streaming not supported during groundedness checks +- Self-reflection uses same LLM by default; override with `REFLECTION_LLM`, `REFLECTION_LLM_SERVERURL`, `REFLECTION_LLM_APIKEY` +- Helm: only on-premises reflection is supported +- GPU requirements for reflection: see `docs/self-reflection.md` for optimal GPU configurations +- Debug reflection: set `LOGLEVEL=INFO` to observe iteration counts +- `FILTER_THINK_TOKENS=false` to see full reasoning output (filtered by default) +- 18 prompt templates available in `prompt.yaml` — custom file only overrides specified keys + +### Reasoning Model Comparison + +| Model | Control | Thinking Budget | Output Format | +|-------|---------|-----------------|---------------| +| Nemotron 1.5 | System prompt (`/think`) | None | `` tags (filtered by default) | +| Nemotron-3-Nano 9B | System prompt (`/think`) | `min_thinking_tokens` + `max_thinking_tokens` | `reasoning_content` field | +| Nemotron-3-Nano 30B | `ENABLE_NEMOTRON_3_NANO_THINKING` env var | `max_thinking_tokens` only | `reasoning_content` field | + +### Thinking Budget Recommendations + +| Range | Use Case | +|-------|----------| +| 1024–4096 | Faster responses for simpler questions | +| 8192–16384 | More thorough reasoning for complex queries | + +## Notebooks +- `notebooks/retriever_api_usage.ipynb` — end-to-end query examples showing generation behavior + +## Source Documentation +- `docs/enable-nemotron-thinking.md` — Reasoning mode for all Nemotron models +- `docs/self-reflection.md` — Self-reflection configuration and thresholds +- `docs/prompt-customization.md` — Prompt template catalog and customization +- `docs/llm-params.md` — Generation parameters (temperature, max tokens, etc.) diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/search-and-retrieval.md b/skill-source/.agents/skills/rag-blueprint/references/configure/search-and-retrieval.md new file mode 100644 index 000000000..b311c958d --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/search-and-retrieval.md @@ -0,0 +1,67 @@ +# Search & Retrieval: Hybrid Search, Multi-Collection, Metadata & Profiles + +## When to Use +User wants to enable hybrid search, query multiple collections, add custom metadata/filters, tune retrieval performance, configure reranker, enable natural language filter generation, or switch accuracy/performance profiles. + +## Process + +1. Detect the deployment mode (Docker / Helm / Library). Docker: edit the active env file. Helm: edit `values.yaml`. Library: edit `notebooks/config.yaml` +2. Read the relevant source doc for detailed configuration +3. Apply the required env vars to the active config and restart affected services +4. Verify via search/generate API call + +## Decision Table + +| Goal | Source Doc | Key Env Vars | +|------|-----------|-------------| +| Hybrid search | `docs/hybrid_search.md` | `APP_VECTORSTORE_SEARCHTYPE=hybrid` | +| Multi-collection | `docs/multi-collection-retrieval.md` | `enable_reranker: True` in API payload | +| Custom metadata | `docs/custom-metadata.md` | Metadata in upload payload, `vdb_filter_expression` in query | +| Accuracy profile | `docs/accuracy_perf.md` | Copy values from `deploy/compose/accuracy_profile.env` into the active env file | +| Performance profile | `docs/accuracy_perf.md` | Copy values from `deploy/compose/perf_profile.env` into the active env file | +| Filter generation | `docs/custom-metadata.md` | `ENABLE_FILTER_GENERATOR=True` | + +## Agent-Specific Notes + +- Hybrid search requires re-ingesting — existing collections created with `dense` must be re-created +- Multi-collection: limited to 5 collections per query; reranker is mandatory +- Multi-collection not supported when `ENABLE_QUERY_DECOMPOSITION=true` +- Elasticsearch RRF not supported in open-source version — must use `weighted` ranker +- Ingestor must be restarted alongside RAG server when enabling hybrid search +- `RERANKER_CONFIDENCE_THRESHOLD` is a legacy alias for `RERANKER_SCORE_THRESHOLD` +- Recommended `RERANKER_SCORE_THRESHOLD` range: 0.3–0.5 (too high filters out too many chunks) + +### Advanced Tuning (not fully documented elsewhere) + +| Variable | Default | Description | +|----------|---------|-------------| +| `APP_VECTORSTORE_INDEXTYPE` | `GPU_CAGRA` | Vector index type | +| `APP_VECTORSTORE_EF` | `100` | Search accuracy/speed trade-off (must be >= `VECTOR_DB_TOPK`) | +| `VECTOR_DB_TOPK` | `100` | Candidates from vector DB (input to reranker) | +| `APP_RETRIEVER_TOPK` | `10` | Chunks sent to LLM prompt (after reranking) | +| `ENABLE_RERANKER` | `True` | Toggle reranking model | +| `RERANKER_SCORE_THRESHOLD` | `0.0` | Minimum reranker score (0.0–1.0) | +| `COLLECTION_NAME` | `multimodal_data` | Default collection name | + +### Partial Filtering +- Strict (default): fails if any collection doesn't support the filter +- Flexible (`allow_partial_filtering: true` in config.yaml): succeeds if at least one collection supports it + +### VDB Filter Support + +| Feature | Milvus | Elasticsearch | +|---------|--------|---------------| +| NL filter generation | LLM-powered | Not supported (manual DSL) | +| Filter syntax | String expressions | List of dicts (ES Query DSL) | +| UI support | Full filtering interface | API only | + +## Notebooks +- `notebooks/retriever_api_usage.ipynb` — RAG retriever API: search and end-to-end queries +- `notebooks/nb_metadata.ipynb` — Metadata ingestion, filtering, and extraction from queries + +## Source Documentation +- `docs/hybrid_search.md` — Hybrid dense + sparse search configuration +- `docs/multi-collection-retrieval.md` — Multi-collection querying +- `docs/custom-metadata.md` — Custom metadata schema, filtering expressions, filter generation +- `docs/accuracy_perf.md` — Best practices for tuning ingestion/retrieval/generation settings +- `docs/python-client.md` — Python library API for search and filtering diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/summarization.md b/skill-source/.agents/skills/rag-blueprint/references/configure/summarization.md new file mode 100644 index 000000000..299c41e89 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/summarization.md @@ -0,0 +1,40 @@ +# Document Summarization + +## When to Use +- User wants to generate summaries during document ingestion +- User asks about summarization strategies or options +- User wants to check summary status or progress + +## Restrictions +- Not supported in lite mode (containerless/library-only deployment) +- Requires Redis for status tracking and rate limiting +- Collection must exist before uploading with `generate_summary: true` + +## Process +1. Detect the deployment mode. Docker: edit the active env file. Helm: configure under `ingestor-server.envVars` in `values.yaml`. Library: use the upload API parameters directly (no env vars needed) +2. Read `docs/summarization.md` for full configuration, env vars, and prompt customization +3. Set `generate_summary: true` in the upload payload (per-request, no global toggle) +4. Optionally configure `summary_options`: strategy, shallow mode, page filter +5. Retrieve summary via `GET /v1/summary?collection_name=...&file_name=...` + +## Decision Table + +| Goal | Strategy | Notes | +|------|----------|-------| +| Fastest overview | `"single"` + `shallow_summary=true` + `page_filter` | Quick text-only extraction | +| Best quality | `null` (iterative, default) + `shallow_summary=false` | Sequential refinement | +| Balanced | `"hierarchical"` + `shallow_summary=true` | Parallel tree-based | + +## Agent-Specific Notes +- `CONVERSATION_HISTORY` prerequisite does not apply — that's for query rewriting only +- `SUMMARY_LLM_SERVERURL=""` (empty) routes to NVIDIA cloud; `"nim-llm:8000"` for self-hosted +- `SUMMARY_LLM_MAX_CHUNK_LENGTH` should be below the model's context window to leave room for prompt + output +- Redis semaphore auto-resets on ingestor startup (prevents stale values from crashes) +- If Redis is unavailable, summaries still generate but no real-time status tracking +- Status entries have 24-hour TTL in Redis + +## Notebooks +- `notebooks/summarization.ipynb` — complete examples for all strategies, status polling, library mode usage + +## Source Documentation +- `docs/summarization.md` — env var reference, prompt customization, rate limiting, chunking details diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/user-interface.md b/skill-source/.agents/skills/rag-blueprint/references/configure/user-interface.md new file mode 100644 index 000000000..7fbad8165 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/user-interface.md @@ -0,0 +1,27 @@ +# User Interface + +## When to Use +- User asks about the RAG UI, uploading documents, settings, or metadata filtering +- User wants to configure features via the web interface + +## Restrictions +- Sample/experimentation UI — not intended for production +- 100-file limit per upload batch; use multiple batches or API for bulk uploads +- 10 MB max per image attachment + +## Process +1. Read `docs/user-interface.md` for full UI documentation +2. Access at `http://localhost:8090` (or `http://:8090` for remote) +3. Configure RAG settings and feature toggles via Settings panel +4. Use Filter Bar above chat input for metadata-filtered queries + +## Agent-Specific Notes +- VLM Inference must be enabled in Settings > Feature Toggles before image attachments work +- ECONNRESET errors on multi-file uploads — recommend API for bulk operations +- Document summaries generate asynchronously; UI shows "Generating summary..." until complete +- Document count in UI may lag slightly after ingestion +- Metadata filtering supports AND/OR logic between filters (toggle via logic button) +- Custom metadata schema is set during collection creation via the Metadata Schema Editor + +## Source Documentation +- `docs/user-interface.md` -- full UI documentation including settings, file types, metadata, and health monitoring diff --git a/skill-source/.agents/skills/rag-blueprint/references/configure/vlm.md b/skill-source/.agents/skills/rag-blueprint/references/configure/vlm.md new file mode 100644 index 000000000..d573cd0c2 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/configure/vlm.md @@ -0,0 +1,56 @@ +# VLM, VLM Embeddings & Image Captioning + +## When to Use +User wants image understanding, visual content analysis, VLM inference, multimodal embeddings, or image captioning during ingestion. + +## Restrictions +- Not available on B200 GPUs — use H100, A100 SXM 80GB, or RTX PRO 6000 +- Requires extra GPU (GPU 1+ for 2-GPU systems, GPU 2+ for 3+ GPUs with fallback) +- VLM embeddings: experimental, PDF-only, no summarization, no citations with page-as-image +- Image captioning on Helm: on-prem only (modify `values.yaml` to enable) + +## Process +1. Detect the deployment mode (Docker / Helm / Library). Docker: edit the active env file. Helm: edit `values.yaml`. Library: edit `notebooks/config.yaml` +2. Read the relevant source doc for detailed steps: + - VLM generation: `docs/vlm.md` + - VLM embeddings: `docs/vlm-embed.md` + - Image captioning: `docs/image_captioning.md` +3. Start VLM NIM (self-hosted) or configure cloud endpoint (NVIDIA-hosted) +4. Set the required variables in the active config: + - Enabling: `ENABLE_VLM_INFERENCE=true` and `APP_NVINGEST_EXTRACTIMAGES=True` + - Disabling: re-comment those variables in the env file +5. Restart affected services and verify with a health check + image-containing document query + +## Decision Table + +| Goal | Source Doc | Docker Profile | Notes | +|------|-----------|---------------|-------| +| VLM replaces LLM | `docs/vlm.md` | `--profile vlm-generation` | LLM not started; set `VLM_TO_LLM_FALLBACK=false` | +| VLM + LLM fallback | `docs/vlm.md` | `--profile vlm-only` | Needs 3+ GPUs; both VLM and LLM running | +| VLM embeddings | `docs/vlm-embed.md` | `--profile vlm-embed` | Experimental; requires re-ingestion | +| Image captioning | `docs/image_captioning.md` | `--profile vlm-only` | Requires VLM NIM; Helm: on-prem only | +| Multimodal query | `docs/multimodal-query.md` | (depends on VLM mode) | Image + text querying | + +## Agent-Specific Notes + +- `--profile vlm-generation` skips the LLM entirely — use `--profile vlm-only` for fallback mode +- `VLM_TO_LLM_FALLBACK` defaults to `true`, but `vlm-generation` profile does not start LLM +- Helm VLM: disable `nim-llm` and enable `nim-vlm` (VLM uses LLM's GPU allocation) +- Helm fallback: keep both `nim-vlm` and `nim-llm` enabled, set `VLM_TO_LLM_FALLBACK: "true"` +- VLM context window is limited — keep queries self-contained +- Image captioning known issue: files without graphs/charts/tables/plots fail to ingest when captioning is enabled + +### Key Env Vars (always needed) +- `ENABLE_VLM_INFERENCE=true` — master toggle +- `APP_NVINGEST_EXTRACTIMAGES=True` — extract images during ingestion +- `VLM_MS_GPU_ID=` — self-hosted GPU assignment + +## Notebooks +- `notebooks/image_input.ipynb` — Multimodal queries with VLM (text + image) + +## Source Documentation +- `docs/vlm.md` — VLM generation (self-hosted, NVIDIA-hosted, Helm, Library) +- `docs/vlm-embed.md` — VLM embeddings (experimental) +- `docs/image_captioning.md` — Image captioning during ingestion +- `docs/multimodal-query.md` — Image + text querying +- `docs/service-port-gpu-reference.md` — default GPU assignments for VLM and other NIMs diff --git a/skill-source/.agents/skills/rag-blueprint/references/deploy.md b/skill-source/.agents/skills/rag-blueprint/references/deploy.md new file mode 100644 index 000000000..96d712892 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/deploy.md @@ -0,0 +1,119 @@ +# RAG Blueprint Deployment + +## Phase 1: Environment Analysis + +Run this single command to collect all environment information at once: + +```bash +echo "=== GPU ===" && nvidia-smi --query-gpu=index,name,memory.total --format=csv,noheader 2>/dev/null || echo "NO_GPU"; echo "=== VRAM ===" && nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | awk '{s+=$1} END {print s "MB total"}' || echo "0MB total"; echo "=== DRIVER ===" && cat /proc/driver/nvidia/version 2>/dev/null | head -1 || echo "NO_DRIVER"; echo "=== CUDA ===" && nvcc --version 2>/dev/null | grep "release" || echo "NO_CUDA_TOOLKIT"; echo "=== DOCKER ===" && docker --version 2>/dev/null || echo "NO_DOCKER"; echo "=== COMPOSE ===" && docker compose version 2>/dev/null || echo "NO_COMPOSE"; echo "=== NVIDIA_TOOLKIT ===" && docker info 2>/dev/null | grep -i "runtimes.*nvidia" || echo "NO_NVIDIA_TOOLKIT"; echo "=== PYTHON ===" && python3 --version 2>/dev/null || echo "NO_PYTHON"; echo "=== DISK ===" && df -h --output=avail / | tail -1; echo "=== OS ===" && cat /etc/os-release 2>/dev/null | grep -E "^(NAME|VERSION)="; echo "=== NGC_KEY ===" && if [ -n "$NGC_API_KEY" ]; then echo "NGC_KEY_SET"; elif [ -n "$NVIDIA_API_KEY" ]; then echo "NVIDIA_KEY_SET"; elif grep -qr "NGC_API_KEY=" deploy/compose/.env deploy/compose/nvdev.env 2>/dev/null | grep -qv "nvapi-your-key"; then echo "DOTENV_SET"; else echo "NOT_SET"; fi; echo "=== RUNNING ===" && docker ps --format "{{.Names}}" 2>/dev/null | grep -E "(rag-server|ingestor-server|nim-llm|milvus)" | head -10 || echo "NO_RUNNING_SERVICES"; echo "=== PORTS ===" && (ss -tlnp 2>/dev/null || netstat -tlnp 2>/dev/null) | grep -E ":(8081|8082|8090|19530) " || echo "PORTS_FREE"; echo "=== REPO ===" && git rev-parse --show-toplevel 2>/dev/null && git describe --tags 2>/dev/null || echo "NO_GIT_REPO"; echo "=== CACHE ===" && du -sh ~/.cache/model-cache/ 2>/dev/null || echo "NO_CACHE" +``` + +Present a summary table: + +| Check | Result | +|-------|--------| +| GPU(s) | (list with VRAM, or NO_GPU) | +| Total VRAM | (sum in MB/GB) | +| NVIDIA Driver | (version or NO_DRIVER) | +| CUDA Toolkit | (version or NO_CUDA_TOOLKIT) | +| Docker | (version or NO_DOCKER) | +| Docker Compose | (version or NO_COMPOSE) | +| NVIDIA Container Toolkit | (detected or NO_NVIDIA_TOOLKIT) | +| Python | (version or NO_PYTHON) | +| Free disk | (value) | +| OS | (name + version) | +| NGC_API_KEY | ENV_SET / DOTENV_SET / NOT_SET | +| Existing services | (list or none) | +| Port availability | (free or list conflicts) | +| Repo | (tag/branch or NO_GIT_REPO) | +| Model cache | (size or empty) | + +### Existing Services Warning + +If RAG services are already running, tell the user briefly: "Existing RAG services detected (list). Proceeding will restart them." Continue unless the user objects. + +If the user wants to **switch deployment modes** (e.g., NVIDIA-hosted → self-hosted, or Docker → library), shut down the existing deployment first via `references/shutdown.md`, then proceed with the new mode. + +If ports are occupied by non-RAG processes, tell the user which ports conflict and suggest stopping the conflicting process. This is a blocker. + +## Phase 2: NGC_API_KEY Handling + +Check in this order: + +1. If `NGC_API_KEY` is set in the shell environment → proceed. +2. If `NVIDIA_API_KEY` is set (common in library mode) → proceed silently. +3. If `NGC_API_KEY` is in `deploy/compose/.env` or `deploy/compose/nvdev.env` (and not the placeholder `nvapi-your-key`) → load it and proceed. +4. If none found → tell the user: "NGC_API_KEY is required. Get one from https://org.ngc.nvidia.com/setup/api-keys and run: `export NGC_API_KEY=\"nvapi-...\"` — then tell me when done." +5. After user confirms → re-check silently. If still not set, write placeholder to `.env` and tell the user to edit it. + +## Phase 3: Blocker Checks + +Automatically check and report all blockers at once (don't stop at the first one): + +Read `docs/support-matrix.md` for current minimum versions and disk requirements, then check: + +- **Docker Compose below minimum**: "Upgrade Docker Compose. See https://docs.docker.com/compose/install/linux/" +- **NVIDIA Driver below minimum** (if self-hosted): "Upgrade NVIDIA driver. See `docs/support-matrix.md` for required version." +- **NVIDIA Container Toolkit missing** (and self-hosted needed): "Install NVIDIA Container Toolkit. See https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html" +- **Insufficient disk**: "Check `docs/support-matrix.md` for disk requirements per deployment mode." +- **No Docker and no Python 3.11+**: "Install Docker or Python 3.11+ to proceed." + +List all blockers together so the user can fix them in one pass — don't make them fix one, re-run, fix another. + +## Phase 4: Route to Deployment Mode + +### User explicitly requests a mode +- "library mode" / "lite mode" / "no docker" / "python mode" → read and follow `deploy/library.md` +- "docker" / "self-hosted" / "local" → read and follow `deploy/docker.md` with mode **self-hosted** +- "cloud" / "nvidia-hosted" / "hosted" → read and follow `deploy/docker.md` with mode **nvidia-hosted** +- "retrieval only" / "search only" / "no LLM" → read and follow `deploy/docker.md` with mode **retrieval-only** +- "kubernetes" / "k8s" / "helm" → read and follow `deploy/helm.md` +- "workbench" / "ai workbench" → tell user to follow `deploy/workbench/README.md` (AI Workbench uses its own UI-driven workflow) + +### Docker is available (Docker + Compose detected) + +**Self-hosted eligible** — read `docs/support-matrix.md` ("Hardware Requirements (Docker)" section) for current GPU requirements. All of the following must also be true: +- GPU count and type matches the Docker self-hosted requirements from the support matrix +- ≥200 GB free disk (per `docs/support-matrix.md` "Disk Space Requirements") +- NVIDIA Container Toolkit detected +- NVIDIA driver meets minimum version from `docs/support-matrix.md` ("Driver Versions") + +If self-hosted eligible → read and follow `deploy/docker.md` with mode **self-hosted** + +**Otherwise with Docker** → read and follow `deploy/docker.md` with mode **nvidia-hosted** + +Tell the user WHY if they have some GPU but not enough: +- "You have [X GPU] with [Y GB] VRAM. Self-hosted requires [requirements from docs/support-matrix.md]. Deploying with NVIDIA-hosted cloud NIMs instead — faster startup, no model download." + +### Docker is available but Compose is not + +Tell the user: "Docker is installed but Docker Compose is below the minimum version (see `docs/support-matrix.md`). Install it: https://docs.docker.com/compose/install/linux/ — or use library mode instead." + +If user chooses library mode → read and follow `deploy/library.md` + +### Docker is not available + +- Python 3.11+ available → read and follow `deploy/library.md` with mode **lite** +- No Python → tell user to install Python 3.11+ or Docker + +## After Deployment + +Once deployment completes, verify health: + +```bash +echo "=== RAG Server ===" && curl -s http://localhost:8081/v1/health?check_dependencies=true 2>/dev/null || echo "RAG_SERVER_NOT_READY"; echo "=== Ingestor ===" && curl -s http://localhost:8082/v1/health?check_dependencies=true 2>/dev/null || echo "INGESTOR_NOT_READY" +``` + +If healthy, tell the user: +- "RAG Blueprint is running and healthy." +- "Ask me to configure features like VLM, query rewriting, guardrails, etc." +- "Ask me to shutdown when you're done." + +If unhealthy, read `references/troubleshoot.md` and diagnose. Match error output against known issues, fix, and retry. Escalate to the user only if the fix requires their action (API key, data deletion). + +## Notebooks +- `notebooks/launchable.ipynb` — Cloud deployment via Brev (alternative to local deployment) + +## Source Documentation +- `docs/support-matrix.md` — GPU requirements, driver versions, disk space, supported platforms +- `docs/service-port-gpu-reference.md` — port mappings and GPU assignments for all services diff --git a/skill-source/.agents/skills/rag-blueprint/references/deploy/docker-nvidia-hosted.md b/skill-source/.agents/skills/rag-blueprint/references/deploy/docker-nvidia-hosted.md new file mode 100644 index 000000000..f4c6ede07 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/deploy/docker-nvidia-hosted.md @@ -0,0 +1,38 @@ +# Docker Deployment (NVIDIA-Hosted NIMs) + +## When to Use +- User wants fast deployment without local model downloads +- User has no GPU or limited GPU +- User asks about cloud-hosted or NVIDIA API deployment +- User wants to avoid 15–30 min NIM startup time + +## Restrictions +- Requires internet access (calls NVIDIA cloud APIs) +- NVIDIA-hosted endpoints have rate limits — large ingestions (>10 files) may hit 429 errors +- NGC_API_KEY required for cloud API access +- Docker and Compose minimum versions per `docs/support-matrix.md` + +## Process +1. Read `docs/deploy-docker-nvidia-hosted.md` for full commands and env configuration +2. Use `deploy/compose/nvdev.env` — pre-configured for cloud endpoints. Source it before compose commands: `source deploy/compose/nvdev.env` +3. Start vector DB → ingestor → RAG server + frontend (no NIM startup needed) +4. Verify: `docker ps` shows containers; UI at `http://localhost:8090` + +## Decision Table + +| Goal | Key Action | +|------|------------| +| Standard cloud deployment | Use `nvdev.env` (pre-configured for cloud) | +| Zero-GPU (no Milvus GPU) | Also switch Milvus image to CPU-only | +| Large file ingestion | Reduce batch/concurrency settings to avoid 429s | +| Maximum throughput | Use self-hosted deployment instead | + +## Agent-Specific Notes +- First run: 5–10 min (image pulls only); subsequent: 1–2 min +- No `nims.yaml` startup — all model inference is cloud-hosted +- All subsequent configure/restart operations should source the same env file used for the initial deploy (`deploy/compose/nvdev.env`) +- For zero-GPU: switch Milvus to CPU-only by changing the GPU image tag to the equivalent non-GPU tag and setting `APP_VECTORSTORE_ENABLEGPUSEARCH=False`. See `docs/deploy-docker-nvidia-hosted.md` for the current image tags +- Rate limit mitigation for large ingestions: reduce `NV_INGEST_FILES_PER_BATCH`, `NV_INGEST_CONCURRENT_BATCHES`, `MAX_INGEST_PROCESS_WORKERS`, `NV_INGEST_MAX_UTIL` to minimum values + +## Source Documentation +- `docs/deploy-docker-nvidia-hosted.md` — full step-by-step commands, env var blocks, CPU Milvus setup diff --git a/skill-source/.agents/skills/rag-blueprint/references/deploy/docker-retrieval-only.md b/skill-source/.agents/skills/rag-blueprint/references/deploy/docker-retrieval-only.md new file mode 100644 index 000000000..fb4935cf8 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/deploy/docker-retrieval-only.md @@ -0,0 +1,37 @@ +# Retrieval-Only Deployment + +## When to Use +- User wants search/retrieval without LLM generation +- User asks to deploy only embedding + reranking services +- User wants `/search` endpoint with an external LLM +- User wants a lightweight, low-GPU deployment + +## Restrictions +- `/generate` endpoint returns an error — no LLM is deployed +- Self-hosted: 1 GPU, ~24 GB memory +- NVIDIA-hosted: 0 GPUs (cloud embedding + reranking) + +## Process +1. Read `docs/retrieval-only-deployment.md` for full commands, env vars, and API examples +2. Choose variant: self-hosted (local NIMs), NVIDIA-hosted (cloud), or Helm +3. For self-hosted: start only embedding + ranking NIMs, skip LLM +4. For NVIDIA-hosted: set embedding/ranking server URLs to empty, skip NIM startup entirely +5. For Helm: set `nimOperator.nim-llm.enabled=false` +6. Start vector DB → ingestor → RAG server +7. Verify health: `GET http://localhost:8081/v1/health?check_dependencies=true` + +## Decision Table + +| Goal | Variant | Key Difference | +|------|---------|----------------| +| Minimal GPU usage with local models | Self-hosted | 1 GPU, ~24 GB | +| Zero GPU, cloud APIs | NVIDIA-hosted | Set server URLs to empty, skip NIM startup | +| Kubernetes | Helm | Disable `nim-llm` in values.yaml | + +## Agent-Specific Notes +- Permission errors on model cache → try `USERID=0` or `chmod -R 755 ~/.cache/model-cache` +- Empty search results → verify documents ingested: `GET http://localhost:8082/v1/documents?collection_name=` +- Users can send `/search` results to their own external LLM for generation + +## Source Documentation +- `docs/retrieval-only-deployment.md` — full deployment commands, API examples, search payload options diff --git a/skill-source/.agents/skills/rag-blueprint/references/deploy/docker-self-hosted.md b/skill-source/.agents/skills/rag-blueprint/references/deploy/docker-self-hosted.md new file mode 100644 index 000000000..dc18ef649 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/deploy/docker-self-hosted.md @@ -0,0 +1,49 @@ +# Docker Deployment (Self-Hosted NIMs) + +## When to Use +- User wants full on-premises deployment with local NIM containers +- User has supported GPUs and wants models running locally +- User asks to deploy RAG Blueprint with Docker + +## Restrictions + +Read `docs/support-matrix.md` for current GPU requirements. Feature restrictions per GPU type: + +| GPU | Cannot Use | +|-----|------------| +| B200 | VLM, Guardrails, Nemotron Parse | +| RTX PRO 6000 | Nemotron Parse | + +- Read `docs/support-matrix.md` for current minimum NVIDIA Driver, CUDA, Docker, and Compose versions +- NVIDIA Container Toolkit required (`docker info` shows nvidia runtime) +- Disk space per `docs/support-matrix.md` ("Disk Space Requirements") +- If any prerequisite is missing, tell the user what to install before proceeding + +## Process +1. Read `docs/deploy-docker-self-hosted.md` for full commands and env configuration +2. Read `docs/support-matrix.md` for GPU compatibility and supported model combinations +3. Verify container toolkit, prepare model cache directory, source `.env` +4. Apply GPU-specific config per source docs +5. Start NIMs → wait for healthy → start remaining services +6. Verify: `docker ps` shows all containers healthy; UI at `http://localhost:8090` + +## Decision Table + +| Goal | Profile Flag | Notes | +|------|-------------|-------| +| Full deployment (default) | (none) | LLM + embedding + ranking + OCR + detection | +| Text-only RAG (lighter) | `--profile rag` | Skip OCR/detection NIMs | +| Ingestion workload only | `--profile ingest` | Embedding + OCR + detection | +| VLM replaces LLM | `--profile vlm-generation` | Not on B200 | +| Advanced PDF extraction | `--profile nemotron-parse` | Not on B200 or RTX PRO 6000 | + +## Agent-Specific Notes +- First run: 15–30 min (model downloads ~100–150 GB, no progress bar); subsequent: 2–5 min +- Monitor download progress: `du -sh ~/.cache/model-cache/` +- Permission error on model cache → try `USERID=0` instead of `USERID=$(id -u)` +- Cloud NIM section in `deploy/compose/.env` must be commented out for self-hosted +- Rebuild after code changes: add `--build` flag to compose up commands + +## Source Documentation +- `docs/deploy-docker-self-hosted.md` — full step-by-step commands, env vars, GPU assignments +- `docs/support-matrix.md` — GPU compatibility, supported models, hardware requirements diff --git a/skill-source/.agents/skills/rag-blueprint/references/deploy/docker.md b/skill-source/.agents/skills/rag-blueprint/references/deploy/docker.md new file mode 100644 index 000000000..eb31fb1a6 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/deploy/docker.md @@ -0,0 +1,88 @@ +# RAG Docker Deployment + +## Determine Mode + +If routed here from the deploy workflow, the mode (self-hosted, nvidia-hosted, or retrieval-only) was already decided. Use it. + +If invoked directly without a mode, auto-detect: + +```bash +echo "=== COMPOSE ===" && docker compose version 2>/dev/null || echo "NO_COMPOSE"; echo "=== GPU ===" && nvidia-smi --query-gpu=name,memory.total --format=csv,noheader 2>/dev/null || echo "NO_GPU"; echo "=== DISK ===" && df -h --output=avail / | tail -1; echo "=== RUNNING ===" && docker ps --format "{{.Names}}" 2>/dev/null | grep -E "(rag-server|ingestor-server|nim-llm|milvus)" | head -10 || echo "NONE_RUNNING" +``` + +If NO_COMPOSE: stop and tell the user to install Docker Compose (see `docs/support-matrix.md` for minimum version). + +Read `docs/support-matrix.md` ("Hardware Requirements (Docker)" section) for current GPU requirements, then: +- GPU count/type meets self-hosted requirements from the support matrix, and 200+ GB free disk → **self-hosted** +- Any GPU or no GPU with ≥50 GB free disk → **nvidia-hosted** +- User explicitly says "retrieval only" / "no LLM" / "search only" → **retrieval-only** + +Auto-route based on hardware. Only ask if two modes are equally valid and the user's intent is ambiguous. + +## Verify NGC_API_KEY + +Auto-check all possible locations before asking: + +```bash +[ -n "$NGC_API_KEY" ] && echo "ENV_SET" || (grep -qr "NGC_API_KEY=" deploy/compose/.env deploy/compose/nvdev.env 2>/dev/null | grep -qv "nvapi-your-key" && echo "DOTENV_SET" || echo "NOT_SET") +``` + +- **ENV_SET**: proceed silently. +- **DOTENV_SET**: load the env file that contains the key and proceed. +- **NOT_SET**: ask the user to provide it. This is the only thing to ask for. + +## Docker Login + +Auto-check if already logged in: + +```bash +grep -q "nvcr.io" ~/.docker/config.json 2>/dev/null && echo "ALREADY_LOGGED_IN" || echo "NOT_LOGGED_IN" +``` + +If already logged in → proceed silently. + +If not logged in → tell the user to run this themselves (the key gets expanded in agent logs): + +> Please run in your terminal: `echo "${NGC_API_KEY}" | docker login nvcr.io -u '$oauthtoken' --password-stdin` + +Wait for confirmation only if login was needed. + +## Deploy + +Based on the mode, read and follow the appropriate reference: + +- **Self-hosted**: read and follow `docker-self-hosted.md` +- **NVIDIA-hosted**: read and follow `docker-nvidia-hosted.md` +- **Retrieval-only**: read and follow `docker-retrieval-only.md` + +## Post-Deploy Verification + +Run health checks: + +```bash +sleep 5; echo "=== RAG ===" && curl -s http://localhost:8081/v1/health?check_dependencies=true 2>/dev/null || echo "RAG_NOT_READY"; echo "=== INGESTOR ===" && curl -s http://localhost:8082/v1/health?check_dependencies=true 2>/dev/null || echo "INGESTOR_NOT_READY"; echo "=== CONTAINERS ===" && docker ps --format "table {{.Names}}\t{{.Status}}" 2>/dev/null | grep -E "(rag|milvus|nim|ingest)" | head -15 +``` + +If services are still initializing, automatically poll every 30 seconds: +- **NVIDIA-hosted**: poll until healthy or 5 minutes elapsed (no model downloads needed). +- **Self-hosted**: poll until healthy or 15 minutes elapsed (model downloads on first run). +- **Retrieval-only**: poll until healthy or 5 minutes elapsed. + +Show progress to the user during polling. + +## On Success + +Tell the user: +- "RAG Blueprint is running and healthy. Open http://localhost:8090 to use the UI." (skip for retrieval-only) +- "Ask me to configure features (VLM, query rewriting, guardrails, etc.)" +- "Ask me to shutdown when you're done." + +## On Error + +1. Read the error output from the failed command. +2. Read `references/troubleshoot.md` to match against common issues (port conflict, disk full, NGC auth, GPU OOM). +3. Apply the fix and retry. +4. If still failing, report the specific error to the user with the fix that was attempted. + +## Source Documentation +- `docs/support-matrix.md` — GPU requirements, hardware compatibility, disk space diff --git a/skill-source/.agents/skills/rag-blueprint/references/deploy/helm-mig.md b/skill-source/.agents/skills/rag-blueprint/references/deploy/helm-mig.md new file mode 100644 index 000000000..dd3dd8ce3 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/deploy/helm-mig.md @@ -0,0 +1,38 @@ +# MIG GPU Deployment + +## When to Use +- User wants fine-grained GPU allocation on Kubernetes using MIG slices +- User has H100 GPUs and wants to share them across RAG services +- User asks about Multi-Instance GPU deployment + +## Restrictions +- Requires H100 80GB HBM3 GPUs (MIG-compatible) +- MIG profiles in this guide are specific to H100 80GB — other GPUs need different profiles +- Requires cloned repository (MIG config files in `deploy/helm/`) +- All standard Helm prerequisites apply (GPU Operator, NIM Operator, StorageClass) +- Ingestion profile is scaled down with MIG — large bulk ingestion jobs may fail + +## Process +1. Read `docs/mig-deployment.md` for full configuration, commands, and MIG slice definitions +2. Enable MIG with mixed strategy on ClusterPolicy +3. Apply MIG ConfigMap and label the node +4. Verify node labels show `mig.config.state: "success"` before proceeding +5. Install Helm chart with `-f mig-slicing/values-mig.yaml` + +## Decision Table + +| Goal | Source Doc | Key Action | +|------|-----------|------------| +| Standard MIG on H100 | `docs/mig-deployment.md` | Apply MIG config, label node, install chart | +| RTX PRO 6000 with MIG | `docs/mig-deployment.md` | Also uncomment model section in values.yaml | +| Custom MIG profiles | NVIDIA MIG User Guide | Modify `mig-config.yaml` for different GPU types | + +## Agent-Specific Notes +- Must wait for `mig.config.state: "success"` on the node before Helm install — if not present, wait and re-check +- Default H100 MIG layout (see `docs/mig-deployment.md` for current GPU count and slice definitions): GPU 0 → small slices, GPU 1 → mixed slices, GPU 2 → full-GPU slice +- LLM gets the largest slice (`7g.80gb`); embedding/Milvus/ingest share small slices +- RTX PRO 6000 variant: uncomment model section in values.yaml, then use both `-f values.yaml -f mig-slicing/values-mig.yaml` +- Uninstall follows standard Helm procedure (see Helm deployment docs) + +## Source Documentation +- `docs/mig-deployment.md` — full MIG config, ClusterPolicy patches, node labeling, verification, Helm install commands diff --git a/skill-source/.agents/skills/rag-blueprint/references/deploy/helm-standard.md b/skill-source/.agents/skills/rag-blueprint/references/deploy/helm-standard.md new file mode 100644 index 000000000..df2630660 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/deploy/helm-standard.md @@ -0,0 +1,51 @@ +# Helm Deployment + +## When to Use +- User wants to deploy RAG Blueprint on Kubernetes +- User asks about Helm chart installation (from NGC or local repo) +- User mentions Kubernetes, k8s, or Helm in deployment context + +## Restrictions + +Read `docs/support-matrix.md` for current Kubernetes, Helm, and OS version requirements. + +- Requires GPU Operator + NIM Operator pre-installed +- Default StorageClass must be configured for PVC provisioning +- Disk space per `docs/support-matrix.md` +- NeMo Guardrails not available in Helm deployment +- Image captioning: on-prem only (requires `values.yaml` changes; see `docs/image_captioning.md`) + +## Process + +### Option A: Deploy from NGC (Remote Chart) +1. Read `docs/deploy-helm.md` for full commands and values +2. Ensure prerequisites: GPU Operator, NIM Operator, StorageClass, NGC_API_KEY +3. Install chart, monitor pods, port-forward frontend + +### Option B: Deploy from Repository (Local Chart) +1. Read `docs/deploy-helm-from-repo.md` for full commands and repo setup +2. Add required Helm repos, run `helm dependency update`, install from local path + +### RTX PRO 6000 Variant +1. Uncomment model section under `nimOperator.nim-llm.model` in `values.yaml` +2. See source docs for engine/precision/GPU settings + +## Decision Table + +| Goal | Option | Key Action | +|------|--------|------------| +| Quick deploy from published chart | NGC (Option A) | `helm upgrade --install` with NGC URL | +| Customized chart | Local repo (Option B) | Clone, modify values, `helm dependency update` | +| RTX PRO 6000 GPUs | Either option | Uncomment model section in values.yaml | +| Retrieval-only (no LLM) | Either option | `--set nimOperator.nim-llm.enabled=false` | + +## Agent-Specific Notes +- First deployment: 60–70 min (model cache download); subsequent: 10–15 min +- Pods in `ContainerCreating`/`Init` for extended time is normal during cache download +- PVCs are not removed by `helm uninstall` — delete manually: `kubectl delete nimcache --all -n rag && kubectl delete pvc --all -n rag` +- Port-forwarding may timeout for large file ingestion — not suitable for bulk uploads +- All configurable endpoints documented in `deploy/helm/nvidia-blueprint-rag/endpoints.md` + +## Source Documentation +- `docs/deploy-helm.md` — NGC remote chart deployment, prerequisites, monitoring +- `docs/deploy-helm-from-repo.md` — local chart deployment, repo setup, dependency management diff --git a/skill-source/.agents/skills/rag-blueprint/references/deploy/helm.md b/skill-source/.agents/skills/rag-blueprint/references/deploy/helm.md new file mode 100644 index 000000000..b381f4dcc --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/deploy/helm.md @@ -0,0 +1,103 @@ +# RAG Helm Deployment + +If routed here from the deploy workflow, proceed directly to Phase 1. + +## Phase 1: Prerequisites Check + +Run all checks at once: + +```bash +echo "=== KUBECTL ===" && kubectl version --client 2>/dev/null || echo "NO_KUBECTL"; echo "=== HELM ===" && helm version --short 2>/dev/null || echo "NO_HELM"; echo "=== STORAGECLASS ===" && kubectl get storageclass 2>/dev/null || echo "NO_STORAGECLASS"; echo "=== NODES ===" && kubectl get nodes -o wide 2>/dev/null || echo "NO_CLUSTER_ACCESS"; echo "=== GPU_OPERATOR ===" && kubectl get pods -n gpu-operator 2>/dev/null | grep -i running || echo "NO_GPU_OPERATOR"; echo "=== NIM_OPERATOR ===" && kubectl get pods -n nim-operator 2>/dev/null | grep -i running || echo "NO_NIM_OPERATOR"; echo "=== NAMESPACE ===" && kubectl get namespace rag 2>/dev/null && echo "NAMESPACE_EXISTS" || echo "NO_NAMESPACE"; echo "=== HELM_RELEASE ===" && helm list -n rag 2>/dev/null | grep rag || echo "NO_EXISTING_RELEASE"; echo "=== PODS ===" && kubectl get pods -n rag 2>/dev/null | head -10 || echo "NO_PODS"; echo "=== NGC_KEY ===" && [ -n "$NGC_API_KEY" ] && echo "NGC_API_KEY SET" || echo "NGC_API_KEY NOT_SET"; echo "=== GPU_RESOURCES ===" && kubectl get nodes -o json 2>/dev/null | grep -o '"nvidia.com/gpu": "[0-9]*"' || echo "NO_GPU_RESOURCES" +``` + +Read `docs/support-matrix.md` for current Kubernetes, Helm, and OS version requirements. + +| Requirement | Check | +|-------------|-------| +| Kubernetes | Per `docs/support-matrix.md` | +| Helm | Per `docs/support-matrix.md` | +| NVIDIA GPU Operator | Installed and running | +| NVIDIA NIM Operator | Installed and running | +| Default StorageClass | Configured (e.g. local-path-provisioner) | +| Disk space | ≥200 GB per node | +| NGC_API_KEY | Set in environment | + +Report all missing prerequisites together so the user can fix everything in one pass. + +If NGC_API_KEY is NOT_SET: this is the one thing we must ask the user for. + +If an existing Helm release is detected: warn "Existing RAG Helm release found. Proceeding will upgrade it." Continue unless user objects. + +## Phase 2: Route to Reference + +Auto-detect the GPU variant from cluster nodes (not the local machine): + +```bash +echo "=== GPU_LABELS ===" && kubectl get nodes -o json 2>/dev/null | grep -oE '"nvidia.com/gpu.product":\s*"[^"]*"' | sort -u || echo "NO_GPU_LABELS"; echo "=== MIG ===" && kubectl get nodes -o json 2>/dev/null | grep -oE '"nvidia.com/mig.strategy":\s*"[^"]*"' || echo "NO_MIG" +``` + +Determine variant from node GPU labels: + +Route based on detection: + +- **MIG enabled** → read and follow `helm-mig.md` +- **RTX PRO 6000** → read and follow `helm-standard.md` (use the RTX values.yaml variant described there) +- **Standard (everything else)** → read and follow `helm-standard.md` + +Ask the user only if the variant is genuinely ambiguous. Default to standard deployment. + +## Phase 3: Expected Timelines + +Set expectations with the user: + +| Scenario | Duration | +|----------|----------| +| First deployment | 60–70 min (NIM cache download ~40–50 min, NIMService init ~10–15 min, pod startup ~5–10 min) | +| Subsequent deployments | 10–15 min (model caches already populated) | + +Pods in `ContainerCreating` or `Init` state for extended periods is normal — models download in the background without progress indicators. + +## Phase 4: Verification + +After deployment completes, verify: + +```bash +echo "=== PODS ===" && kubectl get pods -n rag; echo "=== NIMCACHE ===" && kubectl get nimcache -n rag; echo "=== NIMSERVICE ===" && kubectl get nimservice -n rag +``` + +Wait for all pods to reach `Running` status. Poll every 60 seconds for up to 70 minutes (first deployment involves model downloads). Show progress. + +Once pods are running, port-forward and verify health: + +```bash +kubectl port-forward -n rag service/rag-server 8081:8081 --address 0.0.0.0 & kubectl port-forward -n rag service/rag-frontend 3000:3000 --address 0.0.0.0 & sleep 3 && curl -s http://localhost:8081/v1/health?check_dependencies=true 2>/dev/null || echo "RAG_NOT_READY" +``` + +## Phase 5: Uninstall + +If the user wants to tear down: + +```bash +helm uninstall rag -n rag +kubectl delete nimcache --all -n rag +kubectl delete pvc --all -n rag +``` + +## On Success + +Tell the user: +- "RAG Blueprint is running on Kubernetes. Access the UI at http://localhost:3000 (via port-forward)." +- "Ask me to configure features (VLM, query rewriting, guardrails, etc.)" +- "Ask me to shutdown when you're done." + +## On Error + +1. Check pod status and events: `kubectl describe pod -n rag` and `kubectl get events -n rag --sort-by='.lastTimestamp' | tail -20`. +2. Read pod logs: `kubectl logs -n rag --tail 50`. +3. Read `references/troubleshoot.md` to match against common issues (PVC pending, OOM, image pull failure, port conflict). +4. Apply the fix and retry. If the fix requires data deletion (PVCs, namespace), confirm with user first. + +## Source Documentation +- `docs/support-matrix.md` — Kubernetes/Helm version requirements, GPU compatibility +- `docs/deploy-helm.md` — standard Helm deployment from NGC +- `docs/deploy-helm-from-repo.md` — Helm deployment from local repo diff --git a/skill-source/.agents/skills/rag-blueprint/references/deploy/library-full.md b/skill-source/.agents/skills/rag-blueprint/references/deploy/library-full.md new file mode 100644 index 000000000..1a386cb19 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/deploy/library-full.md @@ -0,0 +1,43 @@ +# Library Mode (Full) + +## When to Use +- User wants programmatic Python access to RAG via `nvidia_rag` package +- User prefers code-level configuration over Docker-based servers +- User asks about library mode, Python client, or `NvidiaRAG`/`NvidiaRAGIngestor` + +## Restrictions +- Python 3.11+ (< 3.14) +- Docker still required for backend services (Milvus, NV-Ingest, Redis, optionally NIMs) +- Self-hosted NIMs require supported GPUs (see `docs/support-matrix.md`) + +## Process +1. Read `docs/python-client.md` for full API reference, configuration, and backend setup +2. Create virtual environment and install `nvidia-rag[all]` +3. Start backend services via Docker (Milvus, NV-Ingest + Redis, optionally NIMs) +4. Load config from `notebooks/config.yaml` using `NvidiaRAGConfig.from_yaml()` +5. Create `NvidiaRAGIngestor` and `NvidiaRAG` instances +6. Use `ingestor.create_collection()`, `ingestor.upload_documents()`, `rag.generate()`, `rag.search()` + +## Decision Table + +| Goal | Source Doc | Key Action | +|------|-----------|------------| +| Self-hosted (local GPUs) | `docs/python-client.md` | Start nims.yaml + set on-prem config | +| Cloud (NVIDIA-hosted) | `docs/python-client.md` | Skip nims.yaml, override server URLs in config | +| Custom prompts | `docs/python-client.md` | Pass `prompts=` to NvidiaRAG constructor | +| Summarization | `docs/python-client.md` | `generate_summary=True` in upload_documents | + +## Agent-Specific Notes +- Config file: `notebooks/config.yaml`; env file: `notebooks/.env_library` +- Docker login is interactive — tell user to run `docker login nvcr.io` themselves +- For cloud deployment: override `config.embeddings.server_url`, `config.llm.server_url`, etc. in code +- Config changes take effect immediately (no container restart needed, unlike Docker mode) +- Prompt customization via constructor: `NvidiaRAG(config=config, prompts="custom_prompts.yaml")` +- `upload_documents()` is async — returns `task_id` for status polling +- NV-Ingest cloud endpoints must be exported before starting NV-Ingest container + +## Notebooks +- `notebooks/rag_library_usage.ipynb` — complete walkthrough: setup, ingestion, querying, search, summaries + +## Source Documentation +- `docs/python-client.md` — full API reference, backend setup, configuration, cloud/self-hosted options diff --git a/skill-source/.agents/skills/rag-blueprint/references/deploy/library-lite.md b/skill-source/.agents/skills/rag-blueprint/references/deploy/library-lite.md new file mode 100644 index 000000000..bf7da9041 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/deploy/library-lite.md @@ -0,0 +1,37 @@ +# Library Mode (Lite / Containerless) + +## When to Use +- Quick prototyping with zero infrastructure (no Docker, no GPU) +- User wants the fastest path to try RAG +- CI/CD pipelines needing lightweight RAG testing + +## Restrictions +- No image/table/chart citations +- No document summarization +- Subject to NVIDIA API rate limits (cloud-hosted inference) +- Requires Python 3.11+ (< 3.14), internet access, and `NGC_API_KEY` + +## Process +1. Read `docs/python-client.md` for full library mode documentation +2. Create virtualenv and install: `pip install nvidia-rag[all]` +3. Ensure `NGC_API_KEY` is exported — maps to `NVIDIA_API_KEY` internally +4. Run the lite notebook: `jupyter lab notebooks/rag_library_lite_usage.ipynb` + +## Agent-Specific Notes +- `NVIDIA_API_KEY` (used by `nvidia_rag` package) must be set from `NGC_API_KEY`: `os.environ["NVIDIA_API_KEY"] = os.environ.get("NGC_API_KEY", "")` +- Lite config lives in `notebooks/config.yaml`; override `server_url` for embeddings to the NVIDIA API Catalog endpoint (see `docs/python-client.md` for current URL), and set LLM/ranking URLs to empty string for cloud defaults +- Milvus Lite runs embedded (no container), NV-Ingest runs as subprocess (no container) +- Also install `python-dotenv jupyterlab` for notebook support + +## When Not to Use +- Production workloads — use Docker or Kubernetes +- Large-scale ingestion — rate limits apply +- Need citations from images/tables/charts or document summarization + +## Notebooks +| Notebook | Description | +|----------|-------------| +| `notebooks/rag_library_lite_usage.ipynb` | End-to-end lite mode: collection creation, ingestion, querying, search | + +## Source Documentation +- `docs/python-client.md` -- full library mode documentation (lite and full) diff --git a/skill-source/.agents/skills/rag-blueprint/references/deploy/library.md b/skill-source/.agents/skills/rag-blueprint/references/deploy/library.md new file mode 100644 index 000000000..a5162be7a --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/deploy/library.md @@ -0,0 +1,54 @@ +# RAG Library Mode Setup + +## Determine Mode + +If routed here from the deploy workflow, the mode (full or lite) may already be decided. Use it. + +If invoked directly, auto-detect: + +```bash +echo "=== DOCKER ===" && docker --version 2>/dev/null || echo "NO_DOCKER"; echo "=== GPU ===" && nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null || echo "NO_GPU"; echo "=== PYTHON ===" && python3 --version 2>/dev/null || echo "NO_PYTHON"; echo "=== PKG_MANAGER ===" && which uv 2>/dev/null && echo "UV_AVAILABLE" || (which pip3 2>/dev/null && echo "PIP_AVAILABLE" || echo "NO_PKG_MANAGER"); echo "=== VENV ===" && ls -d .venv/ venv/ nvidia-rag-env/ 2>/dev/null || echo "NO_EXISTING_VENV"; echo "=== INSTALLED ===" && pip3 show nvidia_rag 2>/dev/null | head -3 || echo "NOT_INSTALLED" +``` + +- Docker available → **full** (Python API + Docker backend services) +- No Docker or user explicitly says "lite" / "no docker" / "containerless" → **lite** + +Auto-route based on Docker availability. Only ask if both modes are equally valid. + +## Verify NGC_API_KEY + +Auto-check all locations: + +```bash +if [ -n "$NGC_API_KEY" ]; then echo "NGC_KEY_SET"; elif [ -n "$NVIDIA_API_KEY" ]; then echo "NVIDIA_KEY_SET"; else echo "NOT_SET"; fi +``` + +If NOT_SET: ask the user. Otherwise proceed silently. + +## Deploy + +Based on the mode: + +- **Full**: read and follow `library-full.md` +- **Lite**: read and follow `library-lite.md` + +## On Success + +Tell the user: +- Which mode was set up and how to start using it (notebook or Python script) +- "Ask me to configure features, change models, etc." +- "Ask me to shutdown backend services when done (if full mode)." + +## On Error + +1. Read the error output (pip install failure, import error, service connection error). +2. Read `references/troubleshoot.md` to match against common issues. +3. Common fixes to try: + - `pip install` failure → try `uv pip install` or check Python version ≥3.11. + - Import error → check if virtual environment is activated. + - Connection error to backend services → check Docker containers are running. +4. Retry the failed step after fixing. +5. If still failing, report the specific error to the user. + +## Source Documentation +- `docs/python-client.md` — Python library API, installation, full and lite mode setup diff --git a/skill-source/.agents/skills/rag-blueprint/references/shutdown.md b/skill-source/.agents/skills/rag-blueprint/references/shutdown.md new file mode 100644 index 000000000..7407b63d8 --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/shutdown.md @@ -0,0 +1,128 @@ +# RAG Shutdown + +Stopping containers and processes does not require confirmation. Deleting data (volumes, cache, images) does. + +## Step 1: Detect What Is Running + +Detect all deployment modes — Docker, K8s, and library: + +```bash +echo "=== DOCKER ===" && docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Image}}" 2>/dev/null || echo "NO_DOCKER"; echo "=== LIBRARY ===" && ps aux | grep -E "(nvidia_rag|uvicorn|jupyter)" | grep -v grep || echo "NO_LIBRARY_PROCESSES"; echo "=== K8S ===" && kubectl get pods -n rag 2>/dev/null | head -10 || echo "NO_K8S"; echo "=== HELM ===" && helm list -n rag 2>/dev/null | grep rag || echo "NO_HELM_RELEASE" +``` + +Based on what's detected, execute the appropriate shutdown path below. If multiple modes are active (e.g., Docker + library), stop all of them. + +## Step 2: Stop Services (Reverse Startup Order) + +Stop in this order — reverse of deployment. Only stop what is actually running (detected in Step 1). + +### 2a: Optional Services + +Stop these first if they are running: + +```bash +docker compose -f deploy/compose/docker-compose-nemo-guardrails.yaml down 2>/dev/null; docker compose -f deploy/compose/observability.yaml down 2>/dev/null +``` + +### 2b: Application Services + +```bash +docker compose -f deploy/compose/docker-compose-rag-server.yaml down; docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down +``` + +### 2c: Vector DB + +```bash +docker compose -f deploy/compose/vectordb.yaml down +``` + +If using Elasticsearch instead of Milvus: +```bash +docker compose -f deploy/compose/vectordb.yaml --profile elasticsearch down +``` + +### 2d: NIMs (Self-Hosted Only) + +Only present if self-hosted deployment was used: + +```bash +docker compose -f deploy/compose/nims.yaml down +``` + +This stops ALL NIM containers (LLM, embedding, ranking, OCR, detection, and any profile-specific NIMs like VLM, audio, nemotron-parse). + +### 2e: Library Mode Processes + +If library mode is active (detected Python processes): + +```bash +pkill -f "nvidia_rag" 2>/dev/null; pkill -f "uvicorn.*rag" 2>/dev/null; docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down 2>/dev/null; docker compose -f deploy/compose/vectordb.yaml down 2>/dev/null +``` + +### 2f: Kubernetes (Helm) Deployment + +If K8s deployment was detected, use the release name and namespace from `helm list` output in step 1: + +```bash +helm uninstall -n 2>/dev/null +``` + +To also clean up persistent data (only if user requests full cleanup): +```bash +kubectl delete nimcache --all -n 2>/dev/null; kubectl delete pvc --all -n 2>/dev/null +``` + +## Step 3: Verify Everything Stopped + +```bash +echo "=== REMAINING ===" && docker ps --format "table {{.Names}}\t{{.Status}}" 2>/dev/null; echo "=== K8S ===" && kubectl get pods -n rag 2>/dev/null | head -10 || echo "NOT_K8S"; helm list -n rag 2>/dev/null || true +``` + +If any RAG-related containers remain, force remove: +```bash +docker ps -a --format "{{.Names}}" | grep -E "(rag|milvus|nim|ingest|redis|nemo|grafana|prometheus|embedding|ranking|vlm|ocr|page-elements|graphic-elements|table-structure)" | xargs -r docker rm -f +``` + +If pods remain after `helm uninstall`, force delete: +```bash +kubectl delete pods --all -n rag --force --grace-period=0 2>/dev/null +``` + +## Step 4: Optional Cleanup + +Ask the user if they want to clean up data/volumes: + +- **Remove Docker volumes** (deletes ingested data, Milvus indices): + ```bash + docker volume prune -f + ``` + +- **Remove model cache** (frees 100-200 GB for self-hosted): + ```bash + rm -rf ~/.cache/model-cache/ + ``` + +- **Remove Docker images** (frees disk space): + ```bash + docker images | grep -E "nvcr.io/nvidia|milvusdb" | awk '{print $3}' | xargs -r docker rmi + ``` + +Only perform cleanup if the user explicitly requests it. + +## Quick One-Liner (All Docker Services) + +If the user wants a fast full teardown: + +```bash +cd "$(git rev-parse --show-toplevel)" && \ +docker compose -f deploy/compose/docker-compose-nemo-guardrails.yaml down 2>/dev/null; \ +docker compose -f deploy/compose/observability.yaml down 2>/dev/null; \ +docker compose -f deploy/compose/docker-compose-rag-server.yaml down 2>/dev/null; \ +docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down 2>/dev/null; \ +docker compose -f deploy/compose/vectordb.yaml down 2>/dev/null; \ +docker compose -f deploy/compose/nims.yaml down 2>/dev/null; \ +echo "All RAG services stopped." +``` + +## Source Documentation +- `docs/troubleshooting.md` — if services won't stop or containers hang diff --git a/skill-source/.agents/skills/rag-blueprint/references/troubleshoot.md b/skill-source/.agents/skills/rag-blueprint/references/troubleshoot.md new file mode 100644 index 000000000..bf0308f8b --- /dev/null +++ b/skill-source/.agents/skills/rag-blueprint/references/troubleshoot.md @@ -0,0 +1,146 @@ +# RAG Troubleshooting + +## Auto-Triage: Run First + +Start with this diagnostic sweep: + +```bash +echo "=== CONTAINERS ===" && docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" 2>/dev/null | grep -E "(rag|milvus|nim|ingest|redis|etcd|minio)" | head -20; echo "=== HEALTH ===" && curl -s http://localhost:8081/v1/health?check_dependencies=true 2>/dev/null || echo "RAG_UNREACHABLE"; curl -s http://localhost:8082/v1/health?check_dependencies=true 2>/dev/null || echo "INGESTOR_UNREACHABLE"; echo "=== LOGS ===" && for svc in rag-server ingestor-server nim-llm-ms nemoretriever-embedding-ms nemoretriever-ranking-ms; do echo "--- $svc ---"; docker logs --tail 20 "$svc" 2>/dev/null | grep -iE "(error|fail|exception|timeout|oom)" || echo "OK"; done; echo "=== GPU ===" && nvidia-smi 2>/dev/null | head -20 || echo "NO_GPU"; echo "=== DISK ===" && df -h / | tail -1; echo "=== DOCKER_DISK ===" && docker system df 2>/dev/null; echo "=== K8S ===" && kubectl get pods -n rag 2>/dev/null | head -20 || echo "NOT_K8S" +``` + +Analyze all output, then diagnose and fix. If Auto-Triage doesn't reveal the cause, dig deeper into the specific failing service's logs (`docker logs --tail 100` or `kubectl logs -n rag --tail 100`). + +Confirm with the user before deleting data (volumes, collections, model cache), changing deployment mode, or modifying API keys. + +## Source Documentation for Detailed Diagnosis + +Read these docs to find specific issue descriptions, causes, and fixes: + +- `docs/troubleshooting.md` — primary reference: all common issues with detailed symptoms/fixes +- `docs/debugging.md` — Pipeline debugging: monitoring deployment, verifying endpoints, tracing requests +- `docs/service-port-gpu-reference.md` — Complete port/GPU mapping table for all services + +## Expected Deployment Times + +If user reports "deployment is taking too long," compare against these baselines: + +| Mode | First Run | Subsequent | +|------|-----------|------------| +| Docker (self-hosted) | 15--30 min (model downloads) | 2--5 min | +| Docker (NVIDIA-hosted) | 5--10 min (no model downloads) | 1--2 min | +| K8s/Helm | 60--70 min (NIM cache 40--50 min + init 10--15 min + pod startup 5--10 min) | 10--15 min | + +If deployment exceeds these times, check NIM container logs: `docker logs nim-llm-ms --tail 50` and model cache disk usage: `watch -n 10 'du -sh ~/.cache/model-cache/'`. + +## Service Health Endpoints + +Read `docs/service-port-gpu-reference.md` for the complete port/GPU mapping. Quick check: + +| Service | URL | Expected | +|---------|-----|----------| +| RAG Server | `http://localhost:8081/v1/health?check_dependencies=true` | `{"status":"healthy"}` | +| Ingestor | `http://localhost:8082/v1/health?check_dependencies=true` | `{"status":"healthy"}` | +| NV-Ingest | `http://localhost:7670/v1/health/ready` | 200 OK | +| Embedding NIM | `http://localhost:9080/v1/health/ready` | 200 OK | +| LLM NIM | `http://localhost:8999/v1/health/ready` | 200 OK | +| Ranking NIM | `http://localhost:1976/v1/health/ready` | 200 OK | +| Milvus | `http://localhost:9091/healthz` | 200 OK | + +## Kubernetes Monitoring Commands + +```bash +kubectl get nimcache -n rag +kubectl get pods -n rag +kubectl logs -f -n rag +kubectl get pvc -n rag +kubectl get events -n rag --sort-by='.lastTimestamp' +``` + +Pods in `ContainerCreating` or `Init` state during model download is expected. Use `kubectl get nimcache -n rag -w` to watch download progress. + +## Enable Debug Logging + +```bash +export LOGLEVEL=DEBUG +docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d --no-deps ingestor-server +docker compose -f deploy/compose/docker-compose-rag-server.yaml up -d --no-deps rag-server +``` + +--- + +## Symptom-to-Fix Quick Index + +Match the symptom from Auto-Triage output, then read `docs/troubleshooting.md` for the detailed fix. For pipeline debugging steps, read `docs/debugging.md`. + +| Symptom | Category | Quick Fix | +|---------|----------|-----------| +| NIM container stuck at `(health: starting)` >30min | NIM Startup | Check GPU memory, NGC auth, disk space. First-run model downloads are slow — wait and monitor cache size. | +| Milvus unhealthy / search returns nothing | Milvus | Restart vectordb compose. Check etcd/MinIO. Port 19530 conflict. Corrupt data → `down -v` (destroys data). | +| Document upload fails / ingestor health check fails | NV-Ingest | Check Redis, OCR NIMs. Rate limit (429) → reduce batch vars. Large PDFs → reduce batch size. | +| Chat returns errors / /generate fails | RAG Server | Check LLM NIM health, embedding NIM, cloud API key. Verify `APP_LLM_MODELNAME` matches deployed NIM. | +| DNS resolution failed for `:` | Networking | Service container not running. Check `docker ps`, restart missing service. | +| Port already in use | Networking | `lsof -i :` to find conflicting process. See port table above. | +| GPU out of memory / `torch.OutOfMemoryError` | GPU | Kill other GPU processes, use `--profile rag` for fewer NIMs, or set correct `NIM_MODEL_PROFILE`. | +| `nvidia-container-cli: unknown device` | GPU | GPU ID exceeds available GPUs. Run `nvidia-smi -L`, adjust `*_GPU_ID` vars to valid IDs. | +| Disk full / insufficient space | Disk | `docker system prune -f`, remove unused images, check model cache size. | +| `no configuration file provided: not found` | Docker Compose | Run from the repo root directory. | +| `too many open files` | Docker Compose | Set `LimitNOFILE=65536` in containerd override, restart containerd. | +| PVC stuck in Pending | Helm | Create missing StorageClass or update PVC. | +| `ProvisioningFailed` access mode mismatch | Helm | Patch NIMCache to `ReadWriteOnce`. | +| Ingestor OOMKilled | Helm | Increase memory limits in values.yaml. Set `SUMMARY_MAX_PARALLELIZATION=1`. | +| Elasticsearch timeout during ingestion | Elasticsearch | Increase `ES_REQUEST_TIMEOUT` (default 600s). | +| Hallucination / out-of-context responses | Quality | Add missing-info handling to prompt in `prompt.yaml`. | +| Embedding dimensions mismatch | Models | Set `APP_EMBEDDINGS_DIMENSIONS` to match model output. Re-ingest. | +| Hybrid/dense search type mismatch | Search | Align `APP_VECTORSTORE_SEARCHTYPE` on ingestor and rag-server. Re-ingest. | +| Confidence threshold filtering all results | Search | Lower `RERANKER_SCORE_THRESHOLD` (range 0.0–1.0, default 0.0). | +| OCR not starting / connection errors | OCR | Check GPU memory, NGC auth. Verify `OCR_GRPC_ENDPOINT`/`OCR_HTTP_ENDPOINT` match running service. | +| NVIDIA API credits exhausted | Cloud | Contact NVIDIA representative for additional credits. | +| Image-only PDFs not ingesting | Ingestion | Enable `APP_NVINGEST_EXTRACTINFOGRAPHICS`. Consider image captioning. | + +--- + +## Troubleshooting Checklists + +### Ingestion Checklist +- [ ] All required containers running (ingestor-server, nv-ingest-ms-runtime, milvus, redis) +- [ ] Vector database accessible (`curl http://localhost:9091/healthz`) +- [ ] Embedding service healthy (`curl http://localhost:9080/v1/health/ready`) +- [ ] File format supported and size <= 400 MB +- [ ] Sufficient disk space (`df -h /`) +- [ ] GPU resources available (`nvidia-smi`) + +### Retrieval Checklist +- [ ] RAG server running and healthy +- [ ] LLM service accessible (`curl http://localhost:8999/v1/health/ready`) +- [ ] Vector database contains data (collection exists with documents) +- [ ] Collection name is correct +- [ ] Query format is valid + +### Quality Checklist +- [ ] Reranker is enabled and healthy +- [ ] Top-K values are appropriate +- [ ] Collection has sufficient relevant data +- [ ] Query rewriting configured correctly +- [ ] Prompt template appropriate for use case + +--- + +## Full Reset + +Destroys all data (volumes, images, caches). Confirm with the user before running. + +If nothing else works and the user confirms: + +```bash +cd "$(git rev-parse --show-toplevel)" +docker compose -f deploy/compose/docker-compose-nemo-guardrails.yaml down 2>/dev/null +docker compose -f deploy/compose/observability.yaml down 2>/dev/null +docker compose -f deploy/compose/docker-compose-rag-server.yaml down 2>/dev/null +docker compose -f deploy/compose/docker-compose-ingestor-server.yaml down 2>/dev/null +docker compose -f deploy/compose/vectordb.yaml down -v 2>/dev/null +docker compose -f deploy/compose/nims.yaml down 2>/dev/null + +docker system prune -af --volumes +``` + +Then deploy fresh using the deploy workflow. diff --git a/skill-source/README.md b/skill-source/README.md new file mode 100644 index 000000000..f5a787677 --- /dev/null +++ b/skill-source/README.md @@ -0,0 +1,111 @@ +# RAG Blueprint Agent Skill + +A single agent skill that enables AI coding assistants (Claude Code, Cursor, Codex, etc.) to deploy, configure, troubleshoot, and manage the NVIDIA RAG Blueprint autonomously. + +## Installation + +```bash +npx skills add . +``` + +Select **rag-blueprint** — it includes all capabilities (deploy, configure, shutdown, troubleshoot) in one skill. + +## Architecture: Skills = Process, Docs = Truth + +``` +SKILL.md = ROUTER (intent detection, autonomy rules, configure routing table) +Reference files = WHAT/HOW (deployment workflows, feature playbooks, diagnostics) +docs/*.md = SOURCE OF TRUTH (never copied into skills) +notebooks/*.ipynb = RUNNABLE EXAMPLES (referenced from relevant skills) +``` + +The SKILL.md detects user intent and routes to the correct reference file. Reference files are concise playbooks that point to `docs/*.md` for detailed configuration — this prevents staleness from duplicated content. + +## Skill Structure + +``` +skill-source/.agents/skills/rag-blueprint/ + SKILL.md ← Single entry point (intent router) + references/ + deploy.md ← Deployment: env analysis, NGC key, routing + deploy/ + docker.md ← Docker Compose deployment workflow + docker-self-hosted.md ← Self-hosted NIMs (local GPU inference) + docker-nvidia-hosted.md ← Cloud NIMs (NVIDIA API endpoints) + docker-retrieval-only.md ← Search/retrieve only (no LLM) + helm.md ← Kubernetes / Helm deployment workflow + helm-standard.md ← Standard Helm chart deployment + helm-mig.md ← Multi-Instance GPU deployment + library.md ← Python library mode workflow + library-full.md ← Python API + Docker backend + library-lite.md ← Containerless (Milvus Lite + cloud APIs) + configure/ + vlm.md ← VLM, VLM embeddings, image captioning + guardrails.md ← NeMo Guardrails + query-and-conversation.md ← Query rewriting, decomposition, multi-turn + ingestion.md ← Text-only, audio, Nemotron Parse, OCR, batch CLI + search-and-retrieval.md ← Hybrid search, multi-collection, metadata, filters + models-and-infrastructure.md ← Model changes, vector DB, auth, API keys, profiles + reasoning-and-generation.md ← Reasoning, self-reflection, prompts, generation params + summarization.md ← Document summarization during ingestion + observability.md ← Tracing, Zipkin, Grafana, Prometheus + multimodal-query.md ← Image + text querying with VLM embeddings + data-catalog.md ← Collection/document metadata management + user-interface.md ← RAG UI settings and usage + api-reference.md ← REST API endpoints and schemas + evaluation.md ← RAGAS quality metrics + mcp.md ← MCP server & client tools + migration.md ← Version upgrade guide + notebooks.md ← Notebook environment and catalog + shutdown.md ← Stop and tear down services + troubleshoot.md ← Diagnose and fix common issues +``` + +## How It Works + +1. User says "deploy RAG" → SKILL.md routes to `references/deploy.md` → env analysis → routes to `deploy/docker.md`, `deploy/helm.md`, or `deploy/library.md` +2. User says "enable VLM" → SKILL.md routes to `references/configure/vlm.md` → reads `docs/vlm.md` for detailed steps +3. User says "RAG is broken" → SKILL.md routes to `references/troubleshoot.md` → auto-triage diagnostic sweep +4. User says "stop RAG" → SKILL.md routes to `references/shutdown.md` → detects and stops all services + +## Supported Deployment Modes + +Read `docs/support-matrix.md` for current hardware requirements per mode. + +| Mode | Docker Required | Description | +|------|-----------------|-------------| +| Docker (self-hosted) | Yes | Full on-prem with local NIM inference | +| Docker (NVIDIA-hosted) | Yes | Cloud APIs for model inference | +| Docker (retrieval-only) | Yes | No LLM, search/retrieve only | +| Helm / Kubernetes | No (K8s) | Production K8s with NIM Operator | +| Library (full) | Yes (backend) | Python API with Docker backend services | +| Library (lite) | No | Milvus Lite + cloud APIs, zero infrastructure | + +## NGC_API_KEY Handling + +Skills never expose the API key value to the LLM. The approach: + +1. Check if `NGC_API_KEY` is set: `[ -n "$NGC_API_KEY" ] && echo "SET" || echo "NOT_SET"` +2. If not set, ask the user to run `export NGC_API_KEY="nvapi-your-key"` in the terminal +3. For `docker login`, the user runs it themselves (the command expands the key) +4. As a fallback, offer to write a placeholder to `deploy/compose/.env` for the user to replace + +## Notebook Integration + +All 13 notebooks are referenced from relevant reference files: + +| Notebook | Referenced In | +|----------|--------------| +| `ingestion_api_usage.ipynb` | `references/configure/ingestion.md` | +| `retriever_api_usage.ipynb` | `references/configure/search-and-retrieval.md` | +| `image_input.ipynb` | `references/configure/vlm.md`, `references/configure/multimodal-query.md` | +| `summarization.ipynb` | `references/configure/summarization.md` | +| `evaluation_01_ragas.ipynb` | `references/configure/evaluation.md` | +| `evaluation_02_recall.ipynb` | `references/configure/evaluation.md` | +| `nb_metadata.ipynb` | `references/configure/search-and-retrieval.md` | +| `rag_library_usage.ipynb` | `references/deploy/library-full.md` | +| `rag_library_lite_usage.ipynb` | `references/deploy/library-lite.md` | +| `building_rag_vdb_operator.ipynb` | `references/configure/models-and-infrastructure.md` | +| `mcp_server_usage.ipynb` | `references/configure/mcp.md` | +| `nat_mcp_integration.ipynb` | `references/configure/mcp.md` | +| `launchable.ipynb` | `SKILL.md` | From 0d40efa4e4e64c5572459c4e2534a68d9aeb3fad Mon Sep 17 00:00:00 2001 From: Shubhadeep Das <149712532+shubhadeepd@users.noreply.github.com> Date: Wed, 11 Mar 2026 23:55:39 +0530 Subject: [PATCH 54/81] doc: Fix broken links in notebooks and docs (#426) --- docs/nemoretriever-ocr.md | 4 +- docs/support-matrix.md | 3 +- notebooks/evaluation_01_ragas.ipynb | 1103 +++++++-------- notebooks/image_input.ipynb | 1980 +++++++++++++-------------- notebooks/launchable.ipynb | 6 +- notebooks/rag_event_ingest.ipynb | 1582 ++++++++++----------- 6 files changed, 2342 insertions(+), 2336 deletions(-) diff --git a/docs/nemoretriever-ocr.md b/docs/nemoretriever-ocr.md index d8ef4f3c5..1893da0aa 100644 --- a/docs/nemoretriever-ocr.md +++ b/docs/nemoretriever-ocr.md @@ -49,7 +49,7 @@ By default, the NVIDIA RAG Blueprint is configured to use NeMo Retriever OCR wit ### Hardware Requirements -For detailed hardware requirements and GPU support, refer to the [NeMo Retriever OCR Support Matrix](https://docs.nvidia.com/nim/ingestion/image-ocr/1.2.0/support-matrix.html). +For detailed hardware requirements and GPU support, refer to the [NeMo Retriever OCR Support Matrix](https://docs.nvidia.com/nim/ingestion/image-ocr/1.2.1/support-matrix.html). ### Docker Configuration @@ -83,8 +83,6 @@ Consider using Paddle OCR if you: ### Hardware Requirements -For detailed hardware requirements, refer to the [Paddle OCR Support Matrix](https://docs.nvidia.com/nim/ingestion/table-extraction/latest/support-matrix.html#supported-hardware). - ### Docker Configuration The Paddle OCR service configuration: diff --git a/docs/support-matrix.md b/docs/support-matrix.md index 6344822e1..07843a774 100644 --- a/docs/support-matrix.md +++ b/docs/support-matrix.md @@ -78,8 +78,7 @@ The following are requirements and recommendations for the individual components - **LLM NIM (llama-3.3-nemotron-super-49b-v1.5)** – Refer to the [Support Matrix](https://docs.nvidia.com/nim/large-language-models/latest/supported-models.html#llama-3-3-nemotron-super-49b-v1-5). - **Embedding NIM (Llama-3.2-NV-EmbedQA-1B-v2 )** – Refer to the [Support Matrix](https://docs.nvidia.com/nim/nemo-retriever/text-embedding/latest/support-matrix.html#llama-3-2-nv-embedqa-1b-v2). - **Reranking NIM (llama-3_2-nv-rerankqa-1b-v2 )**: Refer to the [Support Matrix](https://docs.nvidia.com/nim/nemo-retriever/text-reranking/latest/support-matrix.html#llama-3-2-nv-rerankqa-1b-v2). -- **NeMo Retriever OCR (Default)**: Refer to the [Support Matrix](https://docs.nvidia.com/nim/ingestion/image-ocr/1.2.0/support-matrix.html). -- **NVIDIA NIM for Image OCR (baidu/paddleocr - Legacy)**: Refer to the [Support Matrix](https://docs.nvidia.com/nim/ingestion/table-extraction/latest/support-matrix.html#supported-hardware). +- **NeMo Retriever OCR (Default)**: Refer to the [Support Matrix](https://docs.nvidia.com/nim/ingestion/image-ocr/1.2.1/support-matrix.html). - **NVIDIA NIMs for Object Detection**: - NeMo Retriever Page Elements v3 [Support Matrix](https://docs.nvidia.com/nim/ingestion/object-detection/latest/support-matrix.html#nemo-retriever-page-elements-v3) - NeMo Retriever Graphic Elements v1 [Support Matrix](https://docs.nvidia.com/nim/ingestion/object-detection/latest/support-matrix.html#nemo-retriever-graphic-elements-v1) diff --git a/notebooks/evaluation_01_ragas.ipynb b/notebooks/evaluation_01_ragas.ipynb index 4b50aceb0..d3f12933d 100644 --- a/notebooks/evaluation_01_ragas.ipynb +++ b/notebooks/evaluation_01_ragas.ipynb @@ -1,550 +1,559 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "f1de2541", - "metadata": {}, - "source": [ - "# Evaluate Your RAG Pipeline with Ragas: Answer Accuracy, Context Relevancy, and Groundedness\n", - "\n", - "In this notebook, we will evaluate our RAG system using three key metrics with the [Ragas](https://docs.ragas.io/en/stable/) library. \n", - "\n", - "Ragas provides a set of metrics that you can use to evaluate the performance of your LLM application. These metrics are designed to help you objectively measure the performance of your application. \n", - "\n", - "## Evaluation Metrics\n", - "\n", - "In this notebook, we will use the following three metrics, introduced to Ragas by NVIDIA:\n", - "- **Answer Accuracy** – Measures the agreement between a model’s response and a reference ground truth for a given question.\n", - "- **Context Relevancy** – Evaluates whether the retrieved contexts (chunks or passages) are pertinent to the user input. \n", - "- **Response Groundedness** – Measures how well a response is supported or \"grounded\" by the retrieved contexts. It assesses whether each claim in the response can be found, either wholly or partially, in the provided contexts.\n", - "\n", - "## Prerequisites\n", - "\n", - "This notebook assumes you are familiar with the RAG system and you have both `rag-server` and `ingestor-server` up and running. If you have not done that, you can refer to [Get Started](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/docs/deploy-docker-self-hosted.md) to start the RAG server.\n", - "\n", - "## 1. Download Evaluation Documents\n", - "\n", - "First, let's download the FinanceBench dataset to evaluate our RAG system. This dataset includes PDF files with information and reports about publicly traded companies, as well as ground truth question and answer pairs.\n", - "\n", - "We'll clone the repository into our data directory in a subdirectory called `financebench`. The PDFs can be found in the `pdfs` subdirectory.\n" - ] + "cells": [ + { + "cell_type": "markdown", + "id": "f1de2541", + "metadata": {}, + "source": [ + "# Evaluate Your RAG Pipeline with Ragas: Answer Accuracy, Context Relevancy, and Groundedness\n", + "\n", + "In this notebook, we will evaluate our RAG system using three key metrics with the [Ragas](https://docs.ragas.io/en/stable/) library. \n", + "\n", + "Ragas provides a set of metrics that you can use to evaluate the performance of your LLM application. These metrics are designed to help you objectively measure the performance of your application. \n", + "\n", + "## Evaluation Metrics\n", + "\n", + "In this notebook, we will use the following three metrics, introduced to Ragas by NVIDIA:\n", + "- **Answer Accuracy** – Measures the agreement between a model’s response and a reference ground truth for a given question.\n", + "- **Context Relevancy** – Evaluates whether the retrieved contexts (chunks or passages) are pertinent to the user input. \n", + "- **Response Groundedness** – Measures how well a response is supported or \"grounded\" by the retrieved contexts. It assesses whether each claim in the response can be found, either wholly or partially, in the provided contexts.\n", + "\n", + "## Prerequisites\n", + "\n", + "This notebook assumes you are familiar with the RAG system and you have both `rag-server` and `ingestor-server` up and running. If you have not done that, you can refer to [Get Started](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/docs/deploy-docker-self-hosted.md) to start the RAG server.\n", + "\n", + "## 1. Download Evaluation Documents\n", + "\n", + "First, let's download the FinanceBench dataset to evaluate our RAG system. This dataset includes PDF files with information and reports about publicly traded companies, as well as ground truth question and answer pairs.\n", + "\n", + "We'll clone the repository into our data directory in a subdirectory called `financebench`. The PDFs can be found in the `pdfs` subdirectory.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d87b89d8", + "metadata": {}, + "outputs": [], + "source": [ + "! git clone https://github.com/patronus-ai/financebench.git ../data/financebench" + ] + }, + { + "cell_type": "markdown", + "id": "702a5f07", + "metadata": {}, + "source": [ + "## 2. Ingest Evaluation Documents\n", + "\n", + "For evaluation, we will use the FinanceBench dataset. In the data directory, we have the PDF files for the FinanceBench dataset, as well as the `financebench_open_source.jsonl` file, which includes ground truth question and answer pairs. \n", + "\n", + "Let's start by creating a collection called `financebench` and upload the relevant documents.\n", + "\n", + "This process is similar to the `ingestion_api_usage` notebook. First, we'll install the required packages and set up our API connections." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b88ef79", + "metadata": {}, + "outputs": [], + "source": [ + "# Installing required Python packages\n", + "! pip install aiohttp langchain-nvidia-ai-endpoints ragas httpx" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11bcb3fe", + "metadata": {}, + "outputs": [], + "source": [ + "import aiohttp\n", + "import os\n", + "import json\n", + "import glob\n", + "import httpx" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa7a4226", + "metadata": {}, + "outputs": [], + "source": [ + "IPADDRESS = \"ingestor-server\" if os.environ.get(\"AI_WORKBENCH\", \"false\") == \"true\" else \"localhost\" # Replace this with the correct IP address\n", + "INGESTOR_SERVER_PORT = \"8082\"\n", + "INGESTOR_BASE_URL = f\"http://{IPADDRESS}:{INGESTOR_SERVER_PORT}\" # Replace with your server URL\n", + "\n", + "async def print_response(response):\n", + " \"\"\"Helper to print API response.\"\"\"\n", + " try:\n", + " response_json = await response.json()\n", + " print(json.dumps(response_json, indent=2))\n", + " except aiohttp.ClientResponseError:\n", + " print(await response.text())\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47cc6774", + "metadata": {}, + "outputs": [], + "source": [ + "async def create_collection(\n", + " collection_name: str = None,\n", + " metadata_schema: list = []\n", + "):\n", + " \"\"\"Create a new collection in the vector database.\"\"\"\n", + " data = {\n", + " \"collection_name\": collection_name,\n", + " \"metadata_schema\": metadata_schema\n", + " }\n", + "\n", + " HEADERS = {\"Content-Type\": \"application/json\"}\n", + "\n", + " async with aiohttp.ClientSession() as session:\n", + " try:\n", + " async with session.post(f\"{INGESTOR_BASE_URL}/v1/collection\", json=data, headers=HEADERS) as response:\n", + " await print_response(response)\n", + " except aiohttp.ClientError as e:\n", + " return 500, {\"error\": str(e)}\n", + "\n", + "# Create the financebench collection\n", + "await create_collection(\n", + " collection_name=\"financebench\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "92418e23", + "metadata": {}, + "outputs": [], + "source": [ + "# Get all PDF files from the financebench directory\n", + "FILEPATHS = glob.glob(os.path.join(\"../data/financebench/pdfs\", \"*.pdf\"))\n", + "\n", + "async def upload_documents(collection_name: str = \"\"):\n", + " \"\"\"Upload documents to the specified collection.\"\"\"\n", + " data = {\n", + " \"collection_name\": collection_name,\n", + " \"blocking\": False, # If True, upload is blocking; else async. Status API not needed when blocking\n", + " \"split_options\": {\n", + " \"chunk_size\": 512,\n", + " \"chunk_overlap\": 150\n", + " },\n", + " \"generate_summary\": False # Set to True to optionally generate summaries for all documents after ingestion\n", + " }\n", + "\n", + " form_data = aiohttp.FormData()\n", + " \n", + " # Add all PDF files to the form data\n", + " for file_path in FILEPATHS:\n", + " form_data.add_field(\"documents\", open(file_path, \"rb\"), filename=os.path.basename(file_path), content_type=\"application/pdf\")\n", + "\n", + " form_data.add_field(\"data\", json.dumps(data), content_type=\"application/json\")\n", + "\n", + " async with aiohttp.ClientSession() as session:\n", + " try:\n", + " async with session.post(f\"{INGESTOR_BASE_URL}/v1/documents\", data=form_data) as response: # Replace with session.patch for reingesting\n", + " await print_response(response)\n", + " # Return the response JSON for task_id extraction\n", + " response_json = await response.json()\n", + " return response_json\n", + " except aiohttp.ClientError as e:\n", + " print(f\"Error uploading documents: {e}\")\n", + " return None\n", + "\n", + "# Store the response and extract task_id\n", + "upload_response = await upload_documents(collection_name=\"financebench\")\n", + "task_id = upload_response.get(\"task_id\") if upload_response else None\n", + "print(f\"Extracted task_id: {task_id}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "798b7771", + "metadata": {}, + "source": [ + "**⚠️ Note**: During the document ingestion process, two files (`INTEL_2023_8K_dated-2023-08-16.pdf` and `INTEL_2023_8K_dated-2023-02-10.pdf`) may fail to process due to formatting issues. This is expected and can be safely ignored, as it will not affect the evaluation methodology or results. The remaining documents in the dataset are sufficient for comprehensive evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82b3e199", + "metadata": {}, + "outputs": [], + "source": [ + "# This might take a few minutes to complete depending on the number of documents uploaded\n", + "async def get_task_status(\n", + " task_id: str\n", + "):\n", + "\n", + " params = {\n", + " \"task_id\": task_id,\n", + " }\n", + "\n", + " HEADERS = {\"Content-Type\": \"application/json\"}\n", + "\n", + " async with aiohttp.ClientSession() as session:\n", + " try:\n", + " async with session.get(f\"{INGESTOR_BASE_URL}/v1/status\", params=params, headers=HEADERS) as response:\n", + " await print_response(response)\n", + " except aiohttp.ClientError as e:\n", + " return 500, {\"error\": str(e)}\n", + "\n", + "# Use the extracted task_id from the upload_documents response\n", + "if task_id:\n", + " await get_task_status(task_id=task_id)\n", + "else:\n", + " print(\"No task_id available. Please run the upload_documents cell first.\")" + ] + }, + { + "cell_type": "markdown", + "id": "8bb5edff", + "metadata": {}, + "source": [ + "## 3. Create Dataset for Ragas Evaluation\n", + "\n", + "In `data/financebench/data`, there is a file called `financebench_open_source.jsonl`. This file contains questions about the PDFs, as well as corresponding ground truth answers.\n", + "\n", + "For each ground-truth question and answer pair, we will:\n", + "1. Generate an answer from our RAG system\n", + "2. Retrieve the relevant document contexts\n", + "3. Create a dataset suitable for Ragas evaluation\n", + "\n", + "The answer and context retrieval from the RAG system is similar to the `retriever_api_usage` notebook.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b96c09f1", + "metadata": {}, + "outputs": [], + "source": [ + "IPADDRESS = \"rag-server\" if os.environ.get(\"AI_WORKBENCH\", \"false\") == \"true\" else \"localhost\" #Replace this with the correct IP address\n", + "RAG_SERVER_PORT = \"8081\"\n", + "RAG_BASE_URL = f\"http://{IPADDRESS}:{RAG_SERVER_PORT}\" # Replace with your server URL\n", + "\n", + "generate_url = f\"{RAG_BASE_URL}/v1/generate\"\n", + "\n", + "async def generate_answer(payload):\n", + " \"\"\"Generate an answer using the RAG server.\"\"\"\n", + " rag_response = \"\"\n", + " citations = []\n", + " is_first_token = True\n", + "\n", + " async with httpx.AsyncClient(timeout=300.0) as client:\n", + " try:\n", + " async with client.stream(\"POST\", url=generate_url, json=payload) as response:\n", + " # Raise an exception for bad status codes like 4xx or 5xx\n", + " response.raise_for_status()\n", + "\n", + " # iterate over the response lines\n", + " async for line in response.aiter_lines():\n", + " if line.startswith(\"data: \"):\n", + " json_str = line[6:].strip()\n", + " if not json_str:\n", + " continue\n", + "\n", + " try:\n", + " data = json.loads(json_str)\n", + "\n", + " # --- Extract the response from the RAG server ---\n", + " message = data.get(\"choices\", [{}])[0].get(\"message\", {}).get(\"content\", \"\")\n", + " if message:\n", + " rag_response += message\n", + "\n", + " # --- Extract the citations from the RAG server ---\n", + " if is_first_token and data.get(\"citations\"):\n", + " for result in data.get(\"citations\", {}).get(\"results\", []):\n", + " description = result.get(\"metadata\", {}).get(\"description\")\n", + " if description:\n", + " citations.append(description)\n", + " is_first_token = False\n", + "\n", + " finish_reason = data.get(\"choices\", [{}])[0].get(\"finish_reason\")\n", + " if finish_reason == \"stop\":\n", + " return rag_response, citations\n", + "\n", + " except json.JSONDecodeError:\n", + " print(f\"Skipping malformed JSON line: {json_str}\")\n", + " continue\n", + " \n", + " except httpx.HTTPStatusError as e:\n", + " print(f\"HTTP error occurred: {e.response.status_code} - {e.response.text}\")\n", + " except httpx.RequestError as e:\n", + " print(f\"An error occurred while requesting {e.request.url!r}: {e}\")\n", + " except Exception as e:\n", + " print(f\"An error occurred: {e}\")\n", + "\n", + " return rag_response, citations\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "805c5744", + "metadata": {}, + "outputs": [], + "source": [ + "# Load the question and ground-truth answer pairs from the FinanceBench dataset\n", + "with open('../data/financebench/data/financebench_open_source.jsonl', 'r') as file:\n", + " gt_qa_pairs = [json.loads(line) for line in file]\n", + "\n", + "print(f\"Loaded {len(gt_qa_pairs)} question-answer pairs from FinanceBench dataset\")\n", + "\n", + "dataset = []\n", + "\n", + "# For the purposes of keeping this demo brief, we will only evaluate on 50 questions. \n", + "# You can increase this to the full dataset for more comprehensive results.\n", + "n = 50 \n", + "print(f\"Evaluating on {n} questions...\")\n", + "\n", + "for idx, qa_pair in enumerate(gt_qa_pairs[:n]):\n", + " question = qa_pair['question']\n", + " \n", + " print(f\"Processing question {idx + 1}/{n}: {question[:100]}...\")\n", + "\n", + " generate_payload = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": question\n", + " }\n", + " ],\n", + " \"use_knowledge_base\": True,\n", + " \"reranker_top_k\": 2,\n", + " \"vdb_top_k\": 10,\n", + " \"vdb_endpoint\": \"http://milvus:19530\",\n", + " \"collection_names\": [\"financebench\"],\n", + " \"enable_reranker\": True,\n", + " \"enable_citations\": True,\n", + " \"stop\": [],\n", + " \"filter_expr\": ''\n", + " }\n", + " \n", + " rag_answer, citations = await generate_answer(generate_payload)\n", + "\n", + " dataset.append({\n", + " \"user_input\": question,\n", + " \"retrieved_contexts\": citations,\n", + " \"response\": rag_answer,\n", + " \"reference\": qa_pair['answer'],\n", + " })\n", + "\n", + "print(f\"Created dataset with {len(dataset)} entries for evaluation\")" + ] + }, + { + "cell_type": "markdown", + "id": "43e68742", + "metadata": {}, + "source": [ + "\n", + "## 4. Evaluate with Ragas\n", + "\n", + "In this example, we will use the NVIDIA hosted endpoint for our judge model. To use this endpoint, please provide your NVIDIA API Key below. \n", + "\n", + "### Rate Limiting Considerations\n", + "\n", + "When using the public endpoint for the Judge LLM, you will likely encounter rate limit errors. We can try to reduce the number of errors by adjusting the configuration, which we do below. \n", + "\n", + "Alternatively, you can use self-hosted NIM Microservices endpoints to avoid these errors altogether. If you're using a self-hosted NIM, you do not need to provide your API Key.\n", + "\n", + "### Getting Your NVIDIA API Key\n", + "\n", + "To generate an API Key:\n", + "1. Go to [build.nvidia.com](https://build.nvidia.com/)\n", + "2. Click the green \"Get API Key\" button in the top right corner\n", + "3. Paste your key below to save it as an environment variable\n", + "\n", + "### Self-Hosted Option\n", + "\n", + "To deploy the Judge LLM as a NIM on your own infrastructure, follow the instructions [here](https://build.nvidia.com/openai/gpt-oss-120b/deploy).\n" + ] + }, + { + "cell_type": "markdown", + "id": "32df51d0", + "metadata": {}, + "source": [ + "Note: Mixtral 8x22b is the preferred model if you have required compute available. You can deploy it following steps [here](https://build.nvidia.com/mistralai/mixtral-8x22b-instruct/deploy)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31df3819", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from getpass import getpass\n", + "# del os.environ['NVIDIA_API_KEY'] ## delete key and reset if needed\n", + "if os.environ.get(\"NVIDIA_API_KEY\", \"\").startswith(\"nvapi-\"):\n", + " print(\"Valid NVIDIA_API_KEY already in environment. Delete to reset\")\n", + "else:\n", + " candidate_api_key = getpass(\"NVAPI Key (starts with nvapi-): \")\n", + " assert candidate_api_key.startswith(\"nvapi-\"), (\n", + " f\"{candidate_api_key[:5]}... is not a valid key\"\n", + " )\n", + " os.environ[\"NVIDIA_API_KEY\"] = candidate_api_key" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "78fb75fe", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Note: Models on build.nvidia.com are rate limited.\n", + "# To avoid rate-limit issues, either deploy the judge model locally (self-hosted NIM)\n", + "# or use any OpenAI-compatible LLM as the judge for evaluation.\n", + "from langchain_nvidia_ai_endpoints.chat_models import ChatNVIDIA\n", + "\n", + "# Initialize the judge LLM for evaluation\n", + "# You can use any other model by creating a ChatNVIDIA object with a different model id\n", + "llm = ChatNVIDIA(model=\"openai/gpt-oss-120b\") # For using NVIDIA hosted endpoint\n", + "# llm = ChatNVIDIA(model=\"openai/gpt-oss-120b\", base_url=\"http://0.0.0.0:8000/v1\") # If using self-hosted NIM" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "928a3c8a", + "metadata": {}, + "outputs": [], + "source": [ + "# Create the evaluation dataset from our collected data\n", + "from ragas import EvaluationDataset\n", + "\n", + "evaluation_dataset = EvaluationDataset.from_list(dataset)\n", + "print(f\"Created evaluation dataset with {len(evaluation_dataset)} samples\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3ec24f4", + "metadata": {}, + "outputs": [], + "source": [ + "# Import the required metrics and evaluation components\n", + "from ragas.metrics import AnswerAccuracy, ContextRelevance, ResponseGroundedness\n", + "from ragas import evaluate\n", + "from ragas.llms import LangchainLLMWrapper\n", + "\n", + "# Wrap the LLM for use with Ragas\n", + "evaluator_llm = LangchainLLMWrapper(llm)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f2f4245", + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.run_config import RunConfig\n", + "\n", + "custom_config = RunConfig(max_workers=1, max_wait=120)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3a3571af", + "metadata": {}, + "outputs": [], + "source": [ + "# Run the evaluation with our three metrics\n", + "print(\"Starting Ragas evaluation...\")\n", + "print(\"This may take several minutes depending on the dataset size.\")\n", + "\n", + "results = evaluate(\n", + " dataset=evaluation_dataset,\n", + " metrics=[AnswerAccuracy(), ContextRelevance(), ResponseGroundedness()],\n", + " llm=evaluator_llm, \n", + " run_config=custom_config\n", + ")\n", + "\n", + "print(\"Evaluation completed!\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "bac9dde6", + "metadata": {}, + "source": [ + "## 5. Analyze Results\n", + "\n", + "Finally, let's examine our evaluation results. We'll look at both the overall metrics and individual sample performance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4c90647f", + "metadata": {}, + "outputs": [], + "source": [ + "results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2da683a1", + "metadata": {}, + "outputs": [], + "source": [ + "# Convert results to pandas DataFrame for detailed analysis of individual queries\n", + "results_df = results.to_pandas()\n", + "\n", + "import pandas as pd\n", + "\n", + "# 1. Set the option to display ALL columns, preventing the '...'\n", + "pd.set_option('display.max_columns', None)\n", + "\n", + "# 2. To prevent long text in cells from being cut off, you can set the column width\n", + "pd.set_option('display.max_colwidth', 80)\n", + "\n", + "results_df.head()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "evaluate", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.9" + } }, - { - "cell_type": "code", - "execution_count": null, - "id": "d87b89d8", - "metadata": {}, - "outputs": [], - "source": [ - "! git clone https://github.com/patronus-ai/financebench.git ../data/financebench" - ] - }, - { - "cell_type": "markdown", - "id": "702a5f07", - "metadata": {}, - "source": [ - "## 2. Ingest Evaluation Documents\n", - "\n", - "For evaluation, we will use the FinanceBench dataset. In the data directory, we have the PDF files for the FinanceBench dataset, as well as the `financebench_open_source.jsonl` file, which includes ground truth question and answer pairs. \n", - "\n", - "Let's start by creating a collection called `financebench` and upload the relevant documents.\n", - "\n", - "This process is similar to the `ingestion_api_usage` notebook. First, we'll install the required packages and set up our API connections." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0b88ef79", - "metadata": {}, - "outputs": [], - "source": [ - "# Installing required Python packages\n", - "! pip install aiohttp langchain-nvidia-ai-endpoints ragas httpx" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11bcb3fe", - "metadata": {}, - "outputs": [], - "source": [ - "import aiohttp\n", - "import os\n", - "import json\n", - "import glob\n", - "import httpx" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fa7a4226", - "metadata": {}, - "outputs": [], - "source": [ - "IPADDRESS = \"ingestor-server\" if os.environ.get(\"AI_WORKBENCH\", \"false\") == \"true\" else \"localhost\" # Replace this with the correct IP address\n", - "INGESTOR_SERVER_PORT = \"8082\"\n", - "INGESTOR_BASE_URL = f\"http://{IPADDRESS}:{INGESTOR_SERVER_PORT}\" # Replace with your server URL\n", - "\n", - "async def print_response(response):\n", - " \"\"\"Helper to print API response.\"\"\"\n", - " try:\n", - " response_json = await response.json()\n", - " print(json.dumps(response_json, indent=2))\n", - " except aiohttp.ClientResponseError:\n", - " print(await response.text())\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "47cc6774", - "metadata": {}, - "outputs": [], - "source": [ - "async def create_collection(\n", - " collection_name: str = None,\n", - " metadata_schema: list = []\n", - "):\n", - " \"\"\"Create a new collection in the vector database.\"\"\"\n", - " data = {\n", - " \"collection_name\": collection_name,\n", - " \"metadata_schema\": metadata_schema\n", - " }\n", - "\n", - " HEADERS = {\"Content-Type\": \"application/json\"}\n", - "\n", - " async with aiohttp.ClientSession() as session:\n", - " try:\n", - " async with session.post(f\"{INGESTOR_BASE_URL}/v1/collection\", json=data, headers=HEADERS) as response:\n", - " await print_response(response)\n", - " except aiohttp.ClientError as e:\n", - " return 500, {\"error\": str(e)}\n", - "\n", - "# Create the financebench collection\n", - "await create_collection(\n", - " collection_name=\"financebench\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "92418e23", - "metadata": {}, - "outputs": [], - "source": [ - "# Get all PDF files from the financebench directory\n", - "FILEPATHS = glob.glob(os.path.join(\"../data/financebench/pdfs\", \"*.pdf\"))\n", - "\n", - "async def upload_documents(collection_name: str = \"\"):\n", - " \"\"\"Upload documents to the specified collection.\"\"\"\n", - " data = {\n", - " \"collection_name\": collection_name,\n", - " \"blocking\": False, # If True, upload is blocking; else async. Status API not needed when blocking\n", - " \"split_options\": {\n", - " \"chunk_size\": 512,\n", - " \"chunk_overlap\": 150\n", - " },\n", - " \"generate_summary\": False # Set to True to optionally generate summaries for all documents after ingestion\n", - " }\n", - "\n", - " form_data = aiohttp.FormData()\n", - " \n", - " # Add all PDF files to the form data\n", - " for file_path in FILEPATHS:\n", - " form_data.add_field(\"documents\", open(file_path, \"rb\"), filename=os.path.basename(file_path), content_type=\"application/pdf\")\n", - "\n", - " form_data.add_field(\"data\", json.dumps(data), content_type=\"application/json\")\n", - "\n", - " async with aiohttp.ClientSession() as session:\n", - " try:\n", - " async with session.post(f\"{INGESTOR_BASE_URL}/v1/documents\", data=form_data) as response: # Replace with session.patch for reingesting\n", - " await print_response(response)\n", - " # Return the response JSON for task_id extraction\n", - " response_json = await response.json()\n", - " return response_json\n", - " except aiohttp.ClientError as e:\n", - " print(f\"Error uploading documents: {e}\")\n", - " return None\n", - "\n", - "# Store the response and extract task_id\n", - "upload_response = await upload_documents(collection_name=\"financebench\")\n", - "task_id = upload_response.get(\"task_id\") if upload_response else None\n", - "print(f\"Extracted task_id: {task_id}\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "798b7771", - "metadata": {}, - "source": [ - "**⚠️ Note**: During the document ingestion process, two files (`INTEL_2023_8K_dated-2023-08-16.pdf` and `INTEL_2023_8K_dated-2023-02-10.pdf`) may fail to process due to formatting issues. This is expected and can be safely ignored, as it will not affect the evaluation methodology or results. The remaining documents in the dataset are sufficient for comprehensive evaluation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "82b3e199", - "metadata": {}, - "outputs": [], - "source": [ - "# This might take a few minutes to complete depending on the number of documents uploaded\n", - "async def get_task_status(\n", - " task_id: str\n", - "):\n", - "\n", - " params = {\n", - " \"task_id\": task_id,\n", - " }\n", - "\n", - " HEADERS = {\"Content-Type\": \"application/json\"}\n", - "\n", - " async with aiohttp.ClientSession() as session:\n", - " try:\n", - " async with session.get(f\"{INGESTOR_BASE_URL}/v1/status\", params=params, headers=HEADERS) as response:\n", - " await print_response(response)\n", - " except aiohttp.ClientError as e:\n", - " return 500, {\"error\": str(e)}\n", - "\n", - "# Use the extracted task_id from the upload_documents response\n", - "if task_id:\n", - " await get_task_status(task_id=task_id)\n", - "else:\n", - " print(\"No task_id available. Please run the upload_documents cell first.\")" - ] - }, - { - "cell_type": "markdown", - "id": "8bb5edff", - "metadata": {}, - "source": [ - "## 3. Create Dataset for Ragas Evaluation\n", - "\n", - "In `data/financebench/data`, there is a file called `financebench_open_source.jsonl`. This file contains questions about the PDFs, as well as corresponding ground truth answers.\n", - "\n", - "For each ground-truth question and answer pair, we will:\n", - "1. Generate an answer from our RAG system\n", - "2. Retrieve the relevant document contexts\n", - "3. Create a dataset suitable for Ragas evaluation\n", - "\n", - "The answer and context retrieval from the RAG system is similar to the `retriever_api_usage` notebook.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b96c09f1", - "metadata": {}, - "outputs": [], - "source": [ - "IPADDRESS = \"rag-server\" if os.environ.get(\"AI_WORKBENCH\", \"false\") == \"true\" else \"localhost\" #Replace this with the correct IP address\n", - "RAG_SERVER_PORT = \"8081\"\n", - "RAG_BASE_URL = f\"http://{IPADDRESS}:{RAG_SERVER_PORT}\" # Replace with your server URL\n", - "\n", - "generate_url = f\"{RAG_BASE_URL}/v1/generate\"\n", - "\n", - "async def generate_answer(payload):\n", - " \"\"\"Generate an answer using the RAG server.\"\"\"\n", - " rag_response = \"\"\n", - " citations = []\n", - " is_first_token = True\n", - "\n", - " async with httpx.AsyncClient(timeout=300.0) as client:\n", - " try:\n", - " async with client.stream(\"POST\", url=generate_url, json=payload) as response:\n", - " # Raise an exception for bad status codes like 4xx or 5xx\n", - " response.raise_for_status()\n", - "\n", - " # iterate over the response lines\n", - " async for line in response.aiter_lines():\n", - " if line.startswith(\"data: \"):\n", - " json_str = line[6:].strip()\n", - " if not json_str:\n", - " continue\n", - "\n", - " try:\n", - " data = json.loads(json_str)\n", - "\n", - " # --- Extract the response from the RAG server ---\n", - " message = data.get(\"choices\", [{}])[0].get(\"message\", {}).get(\"content\", \"\")\n", - " if message:\n", - " rag_response += message\n", - "\n", - " # --- Extract the citations from the RAG server ---\n", - " if is_first_token and data.get(\"citations\"):\n", - " for result in data.get(\"citations\", {}).get(\"results\", []):\n", - " description = result.get(\"metadata\", {}).get(\"description\")\n", - " if description:\n", - " citations.append(description)\n", - " is_first_token = False\n", - "\n", - " finish_reason = data.get(\"choices\", [{}])[0].get(\"finish_reason\")\n", - " if finish_reason == \"stop\":\n", - " return rag_response, citations\n", - "\n", - " except json.JSONDecodeError:\n", - " print(f\"Skipping malformed JSON line: {json_str}\")\n", - " continue\n", - " \n", - " except httpx.HTTPStatusError as e:\n", - " print(f\"HTTP error occurred: {e.response.status_code} - {e.response.text}\")\n", - " except httpx.RequestError as e:\n", - " print(f\"An error occurred while requesting {e.request.url!r}: {e}\")\n", - " except Exception as e:\n", - " print(f\"An error occurred: {e}\")\n", - "\n", - " return rag_response, citations\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "805c5744", - "metadata": {}, - "outputs": [], - "source": [ - "# Load the question and ground-truth answer pairs from the FinanceBench dataset\n", - "with open('../data/financebench/data/financebench_open_source.jsonl', 'r') as file:\n", - " gt_qa_pairs = [json.loads(line) for line in file]\n", - "\n", - "print(f\"Loaded {len(gt_qa_pairs)} question-answer pairs from FinanceBench dataset\")\n", - "\n", - "dataset = []\n", - "\n", - "# For the purposes of keeping this demo brief, we will only evaluate on 50 questions. \n", - "# You can increase this to the full dataset for more comprehensive results.\n", - "n = 50 \n", - "print(f\"Evaluating on {n} questions...\")\n", - "\n", - "for idx, qa_pair in enumerate(gt_qa_pairs[:n]):\n", - " question = qa_pair['question']\n", - " \n", - " print(f\"Processing question {idx + 1}/{n}: {question[:100]}...\")\n", - "\n", - " generate_payload = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": question\n", - " }\n", - " ],\n", - " \"use_knowledge_base\": True,\n", - " \"reranker_top_k\": 2,\n", - " \"vdb_top_k\": 10,\n", - " \"vdb_endpoint\": \"http://milvus:19530\",\n", - " \"collection_names\": [\"financebench\"],\n", - " \"enable_reranker\": True,\n", - " \"enable_citations\": True,\n", - " \"stop\": [],\n", - " \"filter_expr\": ''\n", - " }\n", - " \n", - " rag_answer, citations = await generate_answer(generate_payload)\n", - "\n", - " dataset.append({\n", - " \"user_input\": question,\n", - " \"retrieved_contexts\": citations,\n", - " \"response\": rag_answer,\n", - " \"reference\": qa_pair['answer'],\n", - " })\n", - "\n", - "print(f\"Created dataset with {len(dataset)} entries for evaluation\")" - ] - }, - { - "cell_type": "markdown", - "id": "43e68742", - "metadata": {}, - "source": [ - "\n", - "## 4. Evaluate with Ragas\n", - "\n", - "In this example, we will use the NVIDIA hosted endpoint for our judge model. To use this endpoint, please provide your NVIDIA API Key below. \n", - "\n", - "### Rate Limiting Considerations\n", - "\n", - "When using the public endpoint for the Judge LLM, you will likely encounter rate limit errors. We can try to reduce the number of errors by adjusting the configuration, which we do below. \n", - "\n", - "Alternatively, you can use self-hosted NIM Microservices endpoints to avoid these errors altogether. If you're using a self-hosted NIM, you do not need to provide your API Key.\n", - "\n", - "### Getting Your NVIDIA API Key\n", - "\n", - "To generate an API Key:\n", - "1. Go to [build.nvidia.com](https://build.nvidia.com/)\n", - "2. Click the green \"Get API Key\" button in the top right corner\n", - "3. Paste your key below to save it as an environment variable\n", - "\n", - "### Self-Hosted Option\n", - "\n", - "To deploy the Judge LLM as a NIM on your own infrastructure, follow the instructions [here](https://build.nvidia.com/mistralai/mixtral-8x22b-instruct/deploy).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "31df3819", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from getpass import getpass\n", - "# del os.environ['NVIDIA_API_KEY'] ## delete key and reset if needed\n", - "if os.environ.get(\"NVIDIA_API_KEY\", \"\").startswith(\"nvapi-\"):\n", - " print(\"Valid NVIDIA_API_KEY already in environment. Delete to reset\")\n", - "else:\n", - " candidate_api_key = getpass(\"NVAPI Key (starts with nvapi-): \")\n", - " assert candidate_api_key.startswith(\"nvapi-\"), (\n", - " f\"{candidate_api_key[:5]}... is not a valid key\"\n", - " )\n", - " os.environ[\"NVIDIA_API_KEY\"] = candidate_api_key" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "78fb75fe", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "# Note: Models on build.nvidia.com are rate limited.\n", - "# To avoid rate-limit issues, either deploy the judge model locally (self-hosted NIM)\n", - "# or use any OpenAI-compatible LLM as the judge for evaluation.\n", - "from langchain_nvidia_ai_endpoints.chat_models import ChatNVIDIA\n", - "\n", - "# Initialize the judge LLM for evaluation\n", - "# You can use any other model by creating Chat Model object\n", - "llm = ChatNVIDIA(model=\"openai/gpt-oss-120b\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "928a3c8a", - "metadata": {}, - "outputs": [], - "source": [ - "# Create the evaluation dataset from our collected data\n", - "from ragas import EvaluationDataset\n", - "\n", - "evaluation_dataset = EvaluationDataset.from_list(dataset)\n", - "print(f\"Created evaluation dataset with {len(evaluation_dataset)} samples\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b3ec24f4", - "metadata": {}, - "outputs": [], - "source": [ - "# Import the required metrics and evaluation components\n", - "from ragas.metrics import AnswerAccuracy, ContextRelevance, ResponseGroundedness\n", - "from ragas import evaluate\n", - "from ragas.llms import LangchainLLMWrapper\n", - "\n", - "# Wrap the LLM for use with Ragas\n", - "evaluator_llm = LangchainLLMWrapper(llm)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9f2f4245", - "metadata": {}, - "outputs": [], - "source": [ - "from ragas.run_config import RunConfig\n", - "\n", - "custom_config = RunConfig(max_workers=1, max_wait=120)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3a3571af", - "metadata": {}, - "outputs": [], - "source": [ - "# Run the evaluation with our three metrics\n", - "print(\"Starting Ragas evaluation...\")\n", - "print(\"This may take several minutes depending on the dataset size.\")\n", - "\n", - "results = evaluate(\n", - " dataset=evaluation_dataset,\n", - " metrics=[AnswerAccuracy(), ContextRelevance(), ResponseGroundedness()],\n", - " llm=evaluator_llm, \n", - " run_config=custom_config\n", - ")\n", - "\n", - "print(\"Evaluation completed!\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "bac9dde6", - "metadata": {}, - "source": [ - "## 5. Analyze Results\n", - "\n", - "Finally, let's examine our evaluation results. We'll look at both the overall metrics and individual sample performance." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4c90647f", - "metadata": {}, - "outputs": [], - "source": [ - "results" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2da683a1", - "metadata": {}, - "outputs": [], - "source": [ - "# Convert results to pandas DataFrame for detailed analysis of individual queries\n", - "results_df = results.to_pandas()\n", - "\n", - "import pandas as pd\n", - "\n", - "# 1. Set the option to display ALL columns, preventing the '...'\n", - "pd.set_option('display.max_columns', None)\n", - "\n", - "# 2. To prevent long text in cells from being cut off, you can set the column width\n", - "pd.set_option('display.max_colwidth', 80)\n", - "\n", - "results_df.head()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "evaluate", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.9" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/notebooks/image_input.ipynb b/notebooks/image_input.ipynb index 4f778a2fa..5698b7985 100644 --- a/notebooks/image_input.ipynb +++ b/notebooks/image_input.ipynb @@ -1,992 +1,992 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "e20e694c", - "metadata": {}, - "source": [ - "# Retriever API Usage with Multimodal Query Support\n", - "\n", - "This notebook demonstrates how to use the NVIDIA RAG retriever APIs with **multimodal queries** (text + images). You'll learn how to:\n", - "\n", - "- 🔍 Search for relevant documents using queries that contain images\n", - "- 🤖 Generate AI responses using the end-to-end RAG API with vision-language models (VLMs)\n", - "- 📊 Work with multimodal embeddings and vector databases\n", - "\n", - "**Use Case**: Query documents with images (e.g., \"What is the price of this item?\" + product image)" - ] - }, - { - "cell_type": "markdown", - "id": "0152f1eb", - "metadata": {}, - "source": [ - "## 📦 Setting up the Dependencies\n", - "\n", - "This section will guide you through:\n", - "1. Configuring your NGC API key for accessing NVIDIA services\n", - "2. Deploying the Milvus vector database\n", - "3. Setting up NVIDIA NIMs (NVIDIA Inference Microservices) for embeddings and VLM\n", - "4. Starting the NVIDIA Ingest runtime for document processing\n", - "5. Launching the RAG server\n", - "\n", - "**Note**: This setup uses Docker Compose to orchestrate all services." - ] - }, - { - "cell_type": "markdown", - "id": "d77a630e", - "metadata": {}, - "source": [ - "### 0. Create a Virtual Environment (Recommended)\n", - "\n", - "Before running this notebook, create a virtual environment using `uv` to isolate dependencies:\n", - "\n", - "```bash\n", - "# Create a virtual environment\n", - "uv venv .venv\n", - "\n", - "# Activate the virtual environment\n", - "source .venv/bin/activate # Linux/macOS\n", - "# .venv\\Scripts\\activate # Windows\n", - "\n", - "# Install Jupyter Lab and ipykernel (if not already installed)\n", - "uv pip install jupyterlab ipykernel\n", - "\n", - "# Register the venv as a Jupyter kernel\n", - "python -m ipykernel install --user --name=.venv --display-name=\"Python (.venv)\"\n", - "```\n", - "\n", - "After setup, select the venv as the kernel for this notebook:\n", - "1. In Jupyter/VS Code/Cursor, click on the kernel selector (top right)\n", - "2. Choose **\".venv\"** or **\"Python (.venv)\"** as the kernel\n", - "\n", - "This ensures all packages installed via `uv pip install` in the notebook cells are installed into the isolated environment.\n" - ] - }, - { - "cell_type": "markdown", - "id": "c39e628e", - "metadata": {}, - "source": [ - "### 1. Setup the Default Configurations\n", - "\n", - "Import necessary libraries for environment management." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c03780a7", - "metadata": {}, - "outputs": [], - "source": [ - "# Install python-dotenv for environment variable management\n", - "! uv pip install python-dotenv\n", - "\n", - "import os\n", - "from getpass import getpass" - ] - }, - { - "cell_type": "markdown", - "id": "8a19cef7", - "metadata": {}, - "source": [ - "Provide your NGC_API_KEY after executing the cell below. You can obtain a key by following steps [here](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/docs/quickstart.md##obtain-an-api-key)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c1f7ffa3", - "metadata": {}, - "outputs": [], - "source": [ - "# Check if NGC_API_KEY is already set, otherwise prompt for it\n", - "# Uncomment the line below to reset your API key\n", - "# del os.environ['NGC_API_KEY']\n", - "\n", - "if os.environ.get(\"NGC_API_KEY\", \"\").startswith(\"nvapi-\"):\n", - " print(\"Valid NGC_API_KEY already in environment. Delete to reset\")\n", - "else:\n", - " candidate_api_key = getpass(\"NVAPI Key (starts with nvapi-): \")\n", - " assert candidate_api_key.startswith(\"nvapi-\"), (\n", - " f\"{candidate_api_key[:5]}... is not a valid key\"\n", - " )\n", - " os.environ[\"NGC_API_KEY\"] = candidate_api_key" - ] - }, - { - "cell_type": "markdown", - "id": "20ec8b61", - "metadata": {}, - "source": [ - "Login to nvcr.io which is needed for pulling the containers of dependencies" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "03972882", - "metadata": {}, - "outputs": [], - "source": [ - "# Login to NVIDIA Container Registry (nvcr.io) to pull required containers\n", - "!echo \"${NGC_API_KEY}\" | docker login nvcr.io -u '$oauthtoken' --password-stdin" - ] - }, - { - "cell_type": "markdown", - "id": "84642fbb", - "metadata": {}, - "source": [ - "### 2. Setup the Milvus Vector Database\n", - "\n", - "Milvus is a high-performance vector database used to store and search multimodal embeddings.\n", - "\n", - "**Configuration Notes**:\n", - "- By default, Milvus uses GPU indexing for faster performance\n", - "- Ensure you have provided the correct GPU ID below\n", - "- If you don't have a GPU available, you can switch to CPU-only Milvus by following the instructions in [milvus-configuration.md](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/docs/milvus-configuration.md)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8125f717", - "metadata": {}, - "outputs": [], - "source": [ - "# Specify which GPU to use for Milvus (change if using a different GPU)\n", - "os.environ[\"VECTORSTORE_GPU_DEVICE_ID\"] = \"0\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3e2d3457", - "metadata": {}, - "outputs": [], - "source": [ - "# Start Milvus vector database service\n", - "# This will run in the background (-d flag)\n", - "!docker compose -f ../deploy/compose/vectordb.yaml up -d" - ] - }, - { - "cell_type": "markdown", - "id": "afe17557", - "metadata": {}, - "source": [ - "### 3. Setup NVIDIA Inference Microservices (NIMs)\n", - "\n", - "NIMs provide optimized inference for AI models. For multimodal RAG, we need:\n", - "- **VLM (Vision-Language Model)**: `nvidia/nemotron-nano-12b-v2-vl` for understanding images and generating responses\n", - "- **Embedding Model**: `llama-3.2-nemoretriever-1b-vlm-embed-v1` for creating multimodal embeddings" - ] - }, - { - "cell_type": "markdown", - "id": "89a135eb", - "metadata": {}, - "source": [ - "#### Deploy On-Premise Models\n", - "\n", - "This section deploys NIMs locally using Docker. Models will be cached to avoid re-downloading." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1b3d2e5c", - "metadata": {}, - "outputs": [], - "source": [ - "# Create the model cache directory\n", - "!mkdir -p ~/.cache/model-cache" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "390df52d", - "metadata": {}, - "outputs": [], - "source": [ - "# Set the MODEL_DIRECTORY environment variable to specify where models are cached\n", - "import os\n", - "\n", - "os.environ[\"MODEL_DIRECTORY\"] = os.path.expanduser(\"~/.cache/model-cache\")\n", - "print(\"MODEL_DIRECTORY set to:\", os.environ[\"MODEL_DIRECTORY\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "62a9946a", - "metadata": {}, - "outputs": [], - "source": [ - "# Deploy NIMs with VLM and embedding profiles\n", - "# ⚠️ WARNING: This may take 10-20 minutes as models download (~10GB+)\n", - "# If the kernel times out, just rerun this cell - it will resume where it left off\n", - "# Select a free GPU for VLM Microservice\n", - "os.environ[\"VLM_MS_GPU_ID\"] = \"1\"\n", - "! USERID=$(id -u) docker compose --profile vlm-ingest --profile vlm-only -f ../deploy/compose/nims.yaml up -d" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e91f511a", - "metadata": {}, - "outputs": [], - "source": [ - "# Monitor the status of running containers\n", - "# Run this cell repeatedly to check if all services are healthy\n", - "# Look for STATUS showing \"healthy\" or \"Up\" for all containers\n", - "!docker ps" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cfb34a6a", - "metadata": {}, - "outputs": [], - "source": [ - "# Configure the model names and service URLs for the RAG pipeline\n", - "# These settings tell the RAG server which models and endpoints to use\n", - "\n", - "# VLM (Vision-Language Model) configuration\n", - "os.environ[\"APP_VLM_MODELNAME\"] = \"nvidia/nemotron-nano-12b-v2-vl\"\n", - "os.environ[\"APP_VLM_SERVERURL\"] = \"http://vlm-ms:8000/v1\"\n", - "\n", - "# Multimodal embedding model configuration\n", - "os.environ[\"APP_EMBEDDINGS_MODELNAME\"] = \"nvidia/llama-nemotron-embed-vl-1b-v2\"\n", - "os.environ[\"APP_EMBEDDINGS_SERVERURL\"] = \"nemotron-vlm-embedding-ms:8000/v1\"\n", - "os.environ[\"ENABLE_VLM_INFERENCE\"] = \"true\"\n", - "os.environ[\"VLM_TO_LLM_FALLBACK\"] = \"false\"\n", - "os.environ[\"ENABLE_RERANKER\"] = \"false\"" - ] - }, - { - "cell_type": "markdown", - "id": "e62c7037", - "metadata": {}, - "source": [ - "#### Cloud based deployment\n", - "Using NVIDIA hosted cloud model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "82084d4d", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "# OCR and document processing endpoints - cloud hosted\n", - "os.environ[\"OCR_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr\"\n", - "os.environ[\"OCR_INFER_PROTOCOL\"] = \"http\"\n", - "os.environ[\"OCR_MODEL_NAME\"] = \"scene_text_ensemble\"\n", - "os.environ[\"YOLOX_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3\"\n", - "os.environ[\"YOLOX_INFER_PROTOCOL\"] = \"http\"\n", - "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1\"\n", - "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL\"] = \"http\"\n", - "os.environ[\"YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1\"\n", - "os.environ[\"YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL\"] = \"http\"\n", - "os.environ[\"APP_NVINGEST_CAPTIONENDPOINTURL\"] = \"https://integrate.api.nvidia.com/v1/chat/completions\"\n", - "\n", - "# VLM Model configuration - cloud hosted\n", - "os.environ[\"APP_VLM_MODELNAME\"] = \"nvidia/nemotron-nano-12b-v2-vl\"\n", - "os.environ[\"APP_VLM_SERVERURL\"] = \"https://integrate.api.nvidia.com/v1\"\n", - "os.environ[\"APP_LLM_SERVERURL\"] = \"\"\n", - "\n", - "# Multimodal embedding model configuration - cloud hosted\n", - "os.environ[\"APP_EMBEDDINGS_MODELNAME\"] = \"nvidia/llama-nemotron-embed-vl-1b-v2\"\n", - "os.environ[\"APP_EMBEDDINGS_SERVERURL\"] = \"https://integrate.api.nvidia.com/v1\"\n", - "os.environ[\"ENABLE_VLM_INFERENCE\"] = \"true\"\n", - "os.environ[\"VLM_TO_LLM_FALLBACK\"] = \"false\"\n", - "os.environ[\"ENABLE_RERANKER\"] = \"false\"" - ] - }, - { - "cell_type": "markdown", - "id": "7cbcfa50", - "metadata": {}, - "source": [ - "### 4. Setup NVIDIA Ingest Runtime\n", - "\n", - "NVIDIA Ingest processes documents to extract text, images, and other elements. We'll configure it to:\n", - "- Extract images from documents\n", - "- Handle multimodal content" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a5e0d73f", - "metadata": {}, - "outputs": [], - "source": [ - "# Configure NVIDIA Ingest to extract and process images from documents\n", - "os.environ[\"APP_NVINGEST_STRUCTURED_ELEMENTS_MODALITY\"] = \"\" # No special handling for structured elements\n", - "os.environ[\"APP_NVINGEST_IMAGE_ELEMENTS_MODALITY\"] = \"image\" # Process image elements as images\n", - "os.environ[\"APP_NVINGEST_EXTRACTIMAGES\"] = \"True\" # Extract images from documents\n", - "\n", - "# Start the ingestor server with Redis\n", - "! docker compose -f ../deploy/compose/docker-compose-ingestor-server.yaml up -d --build" - ] - }, - { - "cell_type": "markdown", - "id": "da1bd9a3", - "metadata": {}, - "source": [ - "### 5. Setup the NVIDIA RAG Server\n", - "\n", - "The RAG server provides the main API endpoints for search and generation. It orchestrates all the components (embeddings, vector DB, VLM) to deliver intelligent responses." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "38ba7752", - "metadata": {}, - "outputs": [], - "source": [ - "# Start the RAG server (accessible at localhost:8081)\n", - "os.environ[\"APP_RANKING_SERVERURL\"] = \"\"\n", - "! docker compose -f ../deploy/compose/docker-compose-rag-server.yaml up -d --build" - ] - }, - { - "cell_type": "markdown", - "id": "ce492ce3", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## 📚 Document Ingestion Workflow\n", - "\n", - "Now that all services are running, let's ingest documents into a collection.\n", - "\n", - "### 6. Create a Collection\n", - "\n", - "A collection is a logical grouping of documents in the vector database. Think of it as a database table optimized for similarity search." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a8611aa1", - "metadata": {}, - "outputs": [], - "source": [ - "# Install aiohttp for async HTTP requests\n", - "! uv pip install aiohttp\n", - "\n", - "# Configure the ingestor server URL\n", - "# Use \"ingestor-server\" when running in AI Workbench, otherwise \"localhost\"\n", - "IPADDRESS = (\n", - " \"ingestor-server\"\n", - " if os.environ.get(\"AI_WORKBENCH\", \"false\") == \"true\"\n", - " else \"localhost\"\n", - ")\n", - "INGESTOR_SERVER_PORT = \"8082\"\n", - "BASE_URL = f\"http://{IPADDRESS}:{INGESTOR_SERVER_PORT}\"\n", - "\n", - "async def print_response(response):\n", - " \"\"\"Helper function to pretty-print API responses.\"\"\"\n", - " try:\n", - " response_json = await response.json()\n", - " print(json.dumps(response_json, indent=2))\n", - " except aiohttp.ClientResponseError:\n", - " print(await response.text())\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "688bc70f", - "metadata": {}, - "outputs": [], - "source": [ - "# Define a unique name for your collection\n", - "# Change this if you want to create a different collection\n", - "collection_name = \"multimodal_query\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24378f6f", - "metadata": {}, - "outputs": [], - "source": [ - "import aiohttp\n", - "import json\n", - "\n", - "\n", - "async def create_collection(\n", - " collection_name: str | None = None,\n", - " metadata_schema: list = [],\n", - "):\n", - " \"\"\"\n", - " Create a new collection in the vector database.\n", - " \n", - " Args:\n", - " collection_name: Unique identifier for the collection\n", - " metadata_schema: Optional schema for metadata fields\n", - " \"\"\"\n", - " data = {\n", - " \"collection_name\": collection_name,\n", - " \"metadata_schema\": metadata_schema,\n", - " }\n", - "\n", - " HEADERS = {\"Content-Type\": \"application/json\"}\n", - "\n", - " async with aiohttp.ClientSession() as session:\n", - " try:\n", - " async with session.post(\n", - " f\"{BASE_URL}/v1/collection\", json=data, headers=HEADERS\n", - " ) as response:\n", - " await print_response(response)\n", - " except aiohttp.ClientError as e:\n", - " return 500, {\"error\": str(e)}\n", - "\n", - "\n", - "# Create the collection\n", - "# The embedding dimension is 2048 for the multimodal embedding model we're using\n", - "await create_collection(\n", - " collection_name=collection_name,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a29f4633", - "metadata": {}, - "outputs": [], - "source": [ - "# Specify the documents to upload\n", - "# This PDF contains product images with pricing information\n", - "FILEPATHS = [\n", - " \"../data/multimodal/product_catalog.pdf\",\n", - "]\n", - "\n", - "async def upload_documents(collection_name: str = \"\"):\n", - " \"\"\"\n", - " Upload and process documents into the collection.\n", - " \n", - " This will:\n", - " 1. Extract text and images from the PDFs\n", - " 2. Chunk the content for optimal retrieval\n", - " 3. Generate multimodal embeddings\n", - " 4. Store everything in the vector database\n", - " \"\"\"\n", - " data = {\n", - " \"collection_name\": collection_name,\n", - " \"blocking\": False, # Async upload - use status API to check progress\n", - " \"split_options\": {\n", - " \"chunk_size\": 512, # Characters per chunk\n", - " \"chunk_overlap\": 150 # Overlap between chunks for context\n", - " },\n", - " \"generate_summary\": False # Set to True to generate document summaries\n", - " }\n", - "\n", - " form_data = aiohttp.FormData()\n", - " \n", - " # Add all PDF files to the form data\n", - " for file_path in FILEPATHS:\n", - " form_data.add_field(\"documents\", open(file_path, \"rb\"), \n", - " filename=os.path.basename(file_path), \n", - " content_type=\"application/pdf\")\n", - "\n", - " form_data.add_field(\"data\", json.dumps(data), content_type=\"application/json\")\n", - "\n", - " async with aiohttp.ClientSession() as session:\n", - " try:\n", - " # Use POST for new uploads, PATCH for re-ingesting existing documents\n", - " async with session.post(f\"{BASE_URL}/v1/documents\", data=form_data) as response:\n", - " await print_response(response)\n", - " response_json = await response.json()\n", - " return response_json\n", - " except aiohttp.ClientError as e:\n", - " print(f\"Error uploading documents: {e}\")\n", - " return None\n", - "\n", - "# Upload the documents and get the task ID for tracking progress\n", - "upload_response = await upload_documents(collection_name=collection_name)\n", - "task_id = upload_response.get(\"task_id\") if upload_response else None\n", - "print(f\"\\nTask ID for tracking: {task_id}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2234e059", - "metadata": {}, - "outputs": [], - "source": [ - "async def get_task_status(task_id: str):\n", - " \"\"\"\n", - " Check the status of an asynchronous ingestion task.\n", - " \n", - " Possible statuses:\n", - " - \"pending\": Task is queued\n", - " - \"processing\": Currently processing documents\n", - " - \"completed\": Successfully finished\n", - " - \"failed\": Error occurred\n", - " \"\"\"\n", - " params = {\n", - " \"task_id\": task_id,\n", - " }\n", - "\n", - " HEADERS = {\"Content-Type\": \"application/json\"}\n", - "\n", - " async with aiohttp.ClientSession() as session:\n", - " try:\n", - " async with session.get(\n", - " f\"{BASE_URL}/v1/status\", params=params, headers=HEADERS\n", - " ) as response:\n", - " await print_response(response)\n", - " except aiohttp.ClientError as e:\n", - " return 500, {\"error\": str(e)}\n", - "\n", - "\n", - "# Check the ingestion status\n", - "# Run this cell multiple times until status shows \"completed\"\n", - "await get_task_status(\n", - " task_id=[task_id]\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "e2c1f1a8", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## 🔍 Querying with Multimodal Inputs\n", - "\n", - "Now that documents are ingested, let's query them using both text and images!\n", - "\n", - "### 7. Using the Search and Generate APIs\n", - "\n", - "We'll demonstrate two approaches:\n", - "1. **Search API**: Find relevant documents without generating a response\n", - "2. **Generate API**: Get an AI-generated answer with citations" - ] - }, - { - "cell_type": "markdown", - "id": "3990ca33", - "metadata": {}, - "source": [ - "#### Prepare a Multimodal Query\n", - "\n", - "To query with an image, we need to:\n", - "1. Convert the image to base64 encoding\n", - "2. Format it according to the OpenAI vision API format\n", - "3. Combine it with a text prompt" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "02dde830", - "metadata": {}, - "outputs": [], - "source": [ - "! uv pip install requests httpx\n", - "import base64\n", - "import requests\n", - "from IPython.display import Image, Markdown, display\n", - "\n", - "def get_base64_image(image_source: str) -> str:\n", - " \"\"\"\n", - " Convert an image to base64 encoding.\n", - " \n", - " Args:\n", - " image_source: Local file path or URL to the image\n", - " \n", - " Returns:\n", - " Base64 encoded string of the image\n", - " \"\"\"\n", - " if image_source.startswith(('http://', 'https://')):\n", - " # Download image from URL\n", - " response = requests.get(image_source)\n", - " return base64.b64encode(response.content).decode()\n", - " else:\n", - " # Read local file\n", - " with open(image_source, \"rb\") as image_file:\n", - " return base64.b64encode(image_file.read()).decode()\n", - "\n", - "# Convert the query image to base64\n", - "# Try different images to test different queries:\n", - "image_b64 = get_base64_image(\"../data/multimodal/Creme_clutch_purse1-small.jpg\")\n", - "\n", - "# Display the query image for reference\n", - "query_image_path = \"../data/multimodal/Creme_clutch_purse1-small.jpg\"\n", - "print(\"📷 Query Image:\")\n", - "display(Image(filename=query_image_path, width=300))\n", - "\n", - "# Format as a data URL\n", - "image_input = f\"data:image/png;base64,{image_b64}\"\n", - "\n", - "# Create the multimodal query with text + image\n", - "# This follows the OpenAI vision API format\n", - "query_1 = \"What material is this made of?\"\n", - "image_query = [\n", - " {\"type\": \"text\", \"text\": query_1},\n", - " {\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {\n", - " \"url\": image_input,\n", - " \"detail\": \"auto\" # Let the model decide the appropriate detail level\n", - " }\n", - " }\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c311f9d0", - "metadata": {}, - "outputs": [], - "source": [ - "import httpx\n", - "import json\n", - "from IPython.display import Image, Markdown, display\n", - "\n", - "RAG_BASE_URL = \"http://localhost:8081\"\n", - "\n", - "async def search_documents(payload):\n", - " \"\"\"\n", - " Search for relevant documents using a multimodal query.\n", - " \n", - " This performs similarity search in the vector database and optionally\n", - " reranks results for better relevance.\n", - " \"\"\"\n", - " search_url = f\"{RAG_BASE_URL}/v1/search\"\n", - " \n", - " async with httpx.AsyncClient(timeout=300.0) as client:\n", - " try:\n", - " response = await client.post(url=search_url, json=payload)\n", - " response.raise_for_status()\n", - " \n", - " search_results = response.json()\n", - " print(\"Search Results:\")\n", - " \n", - " # Display search results with nice formatting\n", - " if \"results\" in search_results:\n", - " for idx, result in enumerate(search_results[\"results\"]):\n", - " doc_type = result.get(\"document_type\", \"text\")\n", - " content = result.get(\"content\", \"\")\n", - " doc_name = result.get(\"document_name\", f\"Result {idx + 1}\")\n", - " score = result.get(\"score\", \"N/A\")\n", - " \n", - " display(Markdown(f\"**Result {idx + 1}: {doc_name} (Score: {score})**\"))\n", - " try:\n", - " if doc_type == \"image\":\n", - " # Display image results\n", - " image_bytes = base64.b64decode(content)\n", - " display(Image(data=image_bytes))\n", - " else:\n", - " # Display text results\n", - " display(Markdown(f\"```\\n{content}\\n```\"))\n", - " except Exception as e:\n", - " print(f\"Error displaying content: {e}\")\n", - " display(Markdown(f\"```\\n{content}\\n```\"))\n", - " \n", - " return search_results\n", - " \n", - " except httpx.HTTPStatusError as e:\n", - " print(f\"HTTP error occurred: {e.response.status_code} - {e.response.text}\")\n", - " except httpx.RequestError as e:\n", - " print(f\"An error occurred while requesting {e.request.url!r}: {e}\")\n", - " except Exception as e:\n", - " print(f\"An error occurred: {e}\")\n", - "\n", - "# Configure the search parameters\n", - "search_payload = {\n", - " \"query\": image_query, # Our multimodal query (text + image)\n", - " \"messages\": [], # No conversation history\n", - " \"use_knowledge_base\": True, # Search the vector database\n", - " \"collection_names\": [collection_name], # Which collection to search\n", - " \"vdb_top_k\": 5, # Retrieve top 5 results from vector DB\n", - " \"vdb_endpoint\": \"http://milvus:19530\", # Milvus connection string\n", - " \"enable_reranker\": False, # Set to True for better relevance (slower)\n", - " \"reranker_top_k\": 3, # If reranker enabled, return top 3\n", - " \"filter_expr\": \"\", # Optional metadata filter\n", - "}\n", - "\n", - "# Execute the search\n", - "print(\"🔍 Searching for documents matching the query...\\n\")\n", - "search_result = await search_documents(search_payload)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bc5b1545", - "metadata": {}, - "outputs": [], - "source": [ - "import base64\n", - "import json\n", - "from IPython.display import Image, Markdown, display\n", - "\n", - "\n", - "async def print_streaming_response_and_citations(response_generator):\n", - " \"\"\"\n", - " Helper function to display streaming responses with citations.\n", - " \n", - " This function:\n", - " 1. Streams the AI-generated response token by token\n", - " 2. Extracts citations from the first chunk\n", - " 3. Displays citations (text or images) after the response completes\n", - " \"\"\"\n", - " first_chunk_data = None\n", - " \n", - " async for chunk in response_generator:\n", - " # Parse Server-Sent Events (SSE) format\n", - " if chunk.startswith(\"data: \"):\n", - " chunk = chunk[len(\"data: \") :].strip()\n", - " if not chunk:\n", - " continue\n", - " \n", - " try:\n", - " data = json.loads(chunk)\n", - " except Exception as e:\n", - " print(f\"JSON decode error: {e}\")\n", - " continue\n", - " \n", - " choices = data.get(\"choices\", [])\n", - " if not choices:\n", - " continue\n", - " \n", - " # Save the first chunk with citations\n", - " if first_chunk_data is None and data.get(\"citations\"):\n", - " first_chunk_data = data\n", - " \n", - " # Print streaming text\n", - " delta = choices[0].get(\"delta\", {})\n", - " text = delta.get(\"content\")\n", - " if not text:\n", - " message = choices[0].get(\"message\", {})\n", - " text = message.get(\"content\", \"\")\n", - " print(text, end=\"\", flush=True)\n", - " \n", - " print() # Newline after streaming\n", - "\n", - " # Display citations after streaming is done\n", - " if first_chunk_data and first_chunk_data.get(\"citations\"):\n", - " print(\"\\n📚 Citations:\")\n", - " citations = first_chunk_data[\"citations\"]\n", - " for idx, citation in enumerate(citations.get(\"results\", [])):\n", - " doc_type = citation.get(\"document_type\", \"text\")\n", - " content = citation.get(\"content\", \"\")\n", - " doc_name = citation.get(\"document_name\", f\"Citation {idx + 1}\")\n", - " display(Markdown(f\"**Citation {idx + 1}: {doc_name}**\"))\n", - " try:\n", - " # Try to display as image\n", - " image_bytes = base64.b64decode(content)\n", - " display(Image(data=image_bytes))\n", - " except Exception:\n", - " # Fall back to text display\n", - " display(Markdown(f\"```\\n{content}\\n```\"))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "31071359", - "metadata": {}, - "outputs": [], - "source": [ - "import httpx\n", - "\n", - "# Configure RAG server URL\n", - "IPADDRESS = \"rag-server\" if os.environ.get(\"AI_WORKBENCH\", \"false\") == \"true\" else \"localhost\"\n", - "RAG_SERVER_PORT = \"8081\"\n", - "RAG_BASE_URL = f\"http://{IPADDRESS}:{RAG_SERVER_PORT}\"\n", - "generate_url = f\"{RAG_BASE_URL}/v1/generate\"\n", - "\n", - "async def generate_answer(payload):\n", - " \"\"\"\n", - " Generate an AI answer using the RAG pipeline.\n", - " \n", - " This function:\n", - " 1. Sends the query to the RAG server\n", - " 2. Retrieves relevant context from the vector database\n", - " 3. Streams the AI-generated response\n", - " 4. Displays citations (sources) used to generate the answer\n", - " \"\"\"\n", - " rag_response = \"\"\n", - " citations = []\n", - " is_first_token = True\n", - "\n", - " async with httpx.AsyncClient(timeout=300.0) as client:\n", - " try:\n", - " async with client.stream(\"POST\", url=generate_url, json=payload) as response:\n", - " # Raise an exception for bad status codes like 4xx or 5xx\n", - " response.raise_for_status()\n", - "\n", - " # Iterate over the streaming response\n", - " async for line in response.aiter_lines():\n", - " if line.startswith(\"data: \"):\n", - " json_str = line[6:].strip()\n", - " if not json_str:\n", - " continue\n", - "\n", - " try:\n", - " data = json.loads(json_str)\n", - "\n", - " # Extract and display the streaming response\n", - " message = data.get(\"choices\", [{}])[0].get(\"message\", {}).get(\"content\", \"\")\n", - " if message:\n", - " rag_response += message\n", - "\n", - " # Extract and display citations from the first chunk\n", - " if is_first_token and data.get(\"citations\"):\n", - " print(\"\\n📚 Citations:\")\n", - " citations = data[\"citations\"]\n", - " for idx, citation in enumerate(citations.get(\"results\", [])):\n", - " doc_type = citation.get(\"document_type\", \"text\")\n", - " content = citation.get(\"content\", \"\")\n", - " doc_name = citation.get(\"document_name\", f\"Citation {idx + 1}\")\n", - " display(Markdown(f\"**Citation {idx + 1}: {doc_name}**\"))\n", - " try:\n", - " # Display image citations\n", - " image_bytes = base64.b64decode(content)\n", - " display(Image(data=image_bytes))\n", - " except Exception:\n", - " # Display text citations\n", - " display(Markdown(f\"```\\n{content}\\n```\"))\n", - " is_first_token = False\n", - "\n", - " # Check if streaming is complete\n", - " finish_reason = data.get(\"choices\", [{}])[0].get(\"finish_reason\")\n", - " if finish_reason == \"stop\":\n", - " return rag_response\n", - "\n", - " except json.JSONDecodeError:\n", - " print(f\"Skipping malformed JSON line: {json_str}\")\n", - " continue\n", - " \n", - " except httpx.HTTPStatusError as e:\n", - " print(f\"HTTP error occurred: {e.response.status_code} - {e.response.text}\")\n", - " except httpx.RequestError as e:\n", - " print(f\"An error occurred while requesting {e.request.url!r}: {e}\")\n", - " except Exception as e:\n", - " print(f\"An error occurred: {e}\")\n", - "\n", - " print(\"\\n✅ Response complete!\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c3049696", - "metadata": {}, - "outputs": [], - "source": [ - "# Format the query as a chat message\n", - "messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": image_query # Our multimodal query (text + image)\n", - " }\n", - "]\n", - "\n", - "# Configure the generate API parameters\n", - "payload = {\n", - " \"messages\": messages, # Chat conversation\n", - " \"use_knowledge_base\": True, # Enable RAG - use vector DB for context\n", - " \"temperature\": 0.2, # Lower = more deterministic, higher = more creative\n", - " \"top_p\": 0.7, # Nucleus sampling parameter\n", - " \"max_tokens\": 1024, # Maximum response length\n", - " \"reranker_top_k\": 2, # Keep top 2 results after reranking\n", - " \"vdb_top_k\": 10, # Retrieve top 10 from vector DB initially\n", - " \"vdb_endpoint\": \"http://milvus:19530\", # Milvus connection\n", - " \"collection_names\": [collection_name], # Which collection to search\n", - " \"enable_query_rewriting\": True, # Improve query before searching\n", - " \"enable_citations\": True, # Include source citations in response\n", - " \"stop\": [], # Optional stop sequences\n", - " \"filter_expr\": \"\", # Optional metadata filter \n", - "}\n", - "\n", - "# Generate the answer with RAG\n", - "print(\"🤖 Generating answer with RAG...\\n\")\n", - "await generate_answer(payload)" - ] - }, - { - "cell_type": "markdown", - "id": "dcddc78d", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## 🎉 Summary\n", - "\n", - "Congratulations! You've successfully:\n", - "\n", - "✅ **Set up the infrastructure**: Deployed Milvus vector DB, NVIDIA NIMs, and RAG services \n", - "✅ **Ingested multimodal documents**: Uploaded PDFs with images and extracted their content \n", - "✅ **Created multimodal queries**: Combined text and images in your search queries \n", - "✅ **Retrieved relevant context**: Used semantic search to find matching documents \n", - "✅ **Generated AI responses**: Got intelligent answers with source citations \n", - "\n", - "### Next Steps\n", - "\n", - "- **Try different queries**: Change the query text or use different query images\n", - "- **Upload more documents**: Add more PDFs to enrich your knowledge base\n", - "- **Experiment with parameters**: Adjust `temperature`, `top_k`, reranker settings\n", - "- **Build applications**: Integrate these APIs into your own applications\n", - "\n", - "### Cleanup\n", - "\n", - "To stop all services and free up resources:\n", - "\n", - "```bash\n", - "cd ../deploy/compose\n", - "docker compose -f docker-compose-rag-server.yaml down\n", - "docker compose -f docker-compose-ingestor-server.yaml down\n", - "docker compose -f nims.yaml down\n", - "docker compose -f vectordb.yaml down\n", - "```\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "id": "e20e694c", + "metadata": {}, + "source": [ + "# Retriever API Usage with Multimodal Query Support\n", + "\n", + "This notebook demonstrates how to use the NVIDIA RAG retriever APIs with **multimodal queries** (text + images). You'll learn how to:\n", + "\n", + "- 🔍 Search for relevant documents using queries that contain images\n", + "- 🤖 Generate AI responses using the end-to-end RAG API with vision-language models (VLMs)\n", + "- 📊 Work with multimodal embeddings and vector databases\n", + "\n", + "**Use Case**: Query documents with images (e.g., \"What is the price of this item?\" + product image)" + ] + }, + { + "cell_type": "markdown", + "id": "0152f1eb", + "metadata": {}, + "source": [ + "## 📦 Setting up the Dependencies\n", + "\n", + "This section will guide you through:\n", + "1. Configuring your NGC API key for accessing NVIDIA services\n", + "2. Deploying the Milvus vector database\n", + "3. Setting up NVIDIA NIMs (NVIDIA Inference Microservices) for embeddings and VLM\n", + "4. Starting the NVIDIA Ingest runtime for document processing\n", + "5. Launching the RAG server\n", + "\n", + "**Note**: This setup uses Docker Compose to orchestrate all services." + ] + }, + { + "cell_type": "markdown", + "id": "d77a630e", + "metadata": {}, + "source": [ + "### 0. Create a Virtual Environment (Recommended)\n", + "\n", + "Before running this notebook, create a virtual environment using `uv` to isolate dependencies:\n", + "\n", + "```bash\n", + "# Create a virtual environment\n", + "uv venv .venv\n", + "\n", + "# Activate the virtual environment\n", + "source .venv/bin/activate # Linux/macOS\n", + "# .venv\\Scripts\\activate # Windows\n", + "\n", + "# Install Jupyter Lab and ipykernel (if not already installed)\n", + "uv pip install jupyterlab ipykernel\n", + "\n", + "# Register the venv as a Jupyter kernel\n", + "python -m ipykernel install --user --name=.venv --display-name=\"Python (.venv)\"\n", + "```\n", + "\n", + "After setup, select the venv as the kernel for this notebook:\n", + "1. In Jupyter/VS Code/Cursor, click on the kernel selector (top right)\n", + "2. Choose **\".venv\"** or **\"Python (.venv)\"** as the kernel\n", + "\n", + "This ensures all packages installed via `uv pip install` in the notebook cells are installed into the isolated environment.\n" + ] + }, + { + "cell_type": "markdown", + "id": "c39e628e", + "metadata": {}, + "source": [ + "### 1. Setup the Default Configurations\n", + "\n", + "Import necessary libraries for environment management." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c03780a7", + "metadata": {}, + "outputs": [], + "source": [ + "# Install python-dotenv for environment variable management\n", + "! uv pip install python-dotenv\n", + "\n", + "import os\n", + "from getpass import getpass" + ] + }, + { + "cell_type": "markdown", + "id": "8a19cef7", + "metadata": {}, + "source": [ + "Provide your NGC_API_KEY after executing the cell below. You can obtain a key by following steps [here](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/docs/api-key.md)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1f7ffa3", + "metadata": {}, + "outputs": [], + "source": [ + "# Check if NGC_API_KEY is already set, otherwise prompt for it\n", + "# Uncomment the line below to reset your API key\n", + "# del os.environ['NGC_API_KEY']\n", + "\n", + "if os.environ.get(\"NGC_API_KEY\", \"\").startswith(\"nvapi-\"):\n", + " print(\"Valid NGC_API_KEY already in environment. Delete to reset\")\n", + "else:\n", + " candidate_api_key = getpass(\"NVAPI Key (starts with nvapi-): \")\n", + " assert candidate_api_key.startswith(\"nvapi-\"), (\n", + " f\"{candidate_api_key[:5]}... is not a valid key\"\n", + " )\n", + " os.environ[\"NGC_API_KEY\"] = candidate_api_key" + ] + }, + { + "cell_type": "markdown", + "id": "20ec8b61", + "metadata": {}, + "source": [ + "Login to nvcr.io which is needed for pulling the containers of dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03972882", + "metadata": {}, + "outputs": [], + "source": [ + "# Login to NVIDIA Container Registry (nvcr.io) to pull required containers\n", + "!echo \"${NGC_API_KEY}\" | docker login nvcr.io -u '$oauthtoken' --password-stdin" + ] + }, + { + "cell_type": "markdown", + "id": "84642fbb", + "metadata": {}, + "source": [ + "### 2. Setup the Milvus Vector Database\n", + "\n", + "Milvus is a high-performance vector database used to store and search multimodal embeddings.\n", + "\n", + "**Configuration Notes**:\n", + "- By default, Milvus uses GPU indexing for faster performance\n", + "- Ensure you have provided the correct GPU ID below\n", + "- If you don't have a GPU available, you can switch to CPU-only Milvus by following the instructions in [milvus-configuration.md](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/docs/milvus-configuration.md)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8125f717", + "metadata": {}, + "outputs": [], + "source": [ + "# Specify which GPU to use for Milvus (change if using a different GPU)\n", + "os.environ[\"VECTORSTORE_GPU_DEVICE_ID\"] = \"0\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3e2d3457", + "metadata": {}, + "outputs": [], + "source": [ + "# Start Milvus vector database service\n", + "# This will run in the background (-d flag)\n", + "!docker compose -f ../deploy/compose/vectordb.yaml up -d" + ] + }, + { + "cell_type": "markdown", + "id": "afe17557", + "metadata": {}, + "source": [ + "### 3. Setup NVIDIA Inference Microservices (NIMs)\n", + "\n", + "NIMs provide optimized inference for AI models. For multimodal RAG, we need:\n", + "- **VLM (Vision-Language Model)**: `nvidia/nemotron-nano-12b-v2-vl` for understanding images and generating responses\n", + "- **Embedding Model**: `llama-3.2-nemoretriever-1b-vlm-embed-v1` for creating multimodal embeddings" + ] + }, + { + "cell_type": "markdown", + "id": "89a135eb", + "metadata": {}, + "source": [ + "#### Deploy On-Premise Models\n", + "\n", + "This section deploys NIMs locally using Docker. Models will be cached to avoid re-downloading." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1b3d2e5c", + "metadata": {}, + "outputs": [], + "source": [ + "# Create the model cache directory\n", + "!mkdir -p ~/.cache/model-cache" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "390df52d", + "metadata": {}, + "outputs": [], + "source": [ + "# Set the MODEL_DIRECTORY environment variable to specify where models are cached\n", + "import os\n", + "\n", + "os.environ[\"MODEL_DIRECTORY\"] = os.path.expanduser(\"~/.cache/model-cache\")\n", + "print(\"MODEL_DIRECTORY set to:\", os.environ[\"MODEL_DIRECTORY\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "62a9946a", + "metadata": {}, + "outputs": [], + "source": [ + "# Deploy NIMs with VLM and embedding profiles\n", + "# ⚠️ WARNING: This may take 10-20 minutes as models download (~10GB+)\n", + "# If the kernel times out, just rerun this cell - it will resume where it left off\n", + "# Select a free GPU for VLM Microservice\n", + "os.environ[\"VLM_MS_GPU_ID\"] = \"1\"\n", + "! USERID=$(id -u) docker compose --profile vlm-ingest --profile vlm-only -f ../deploy/compose/nims.yaml up -d" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e91f511a", + "metadata": {}, + "outputs": [], + "source": [ + "# Monitor the status of running containers\n", + "# Run this cell repeatedly to check if all services are healthy\n", + "# Look for STATUS showing \"healthy\" or \"Up\" for all containers\n", + "!docker ps" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cfb34a6a", + "metadata": {}, + "outputs": [], + "source": [ + "# Configure the model names and service URLs for the RAG pipeline\n", + "# These settings tell the RAG server which models and endpoints to use\n", + "\n", + "# VLM (Vision-Language Model) configuration\n", + "os.environ[\"APP_VLM_MODELNAME\"] = \"nvidia/nemotron-nano-12b-v2-vl\"\n", + "os.environ[\"APP_VLM_SERVERURL\"] = \"http://vlm-ms:8000/v1\"\n", + "\n", + "# Multimodal embedding model configuration\n", + "os.environ[\"APP_EMBEDDINGS_MODELNAME\"] = \"nvidia/llama-nemotron-embed-vl-1b-v2\"\n", + "os.environ[\"APP_EMBEDDINGS_SERVERURL\"] = \"nemotron-vlm-embedding-ms:8000/v1\"\n", + "os.environ[\"ENABLE_VLM_INFERENCE\"] = \"true\"\n", + "os.environ[\"VLM_TO_LLM_FALLBACK\"] = \"false\"\n", + "os.environ[\"ENABLE_RERANKER\"] = \"false\"" + ] + }, + { + "cell_type": "markdown", + "id": "e62c7037", + "metadata": {}, + "source": [ + "#### Cloud based deployment\n", + "Using NVIDIA hosted cloud model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82084d4d", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# OCR and document processing endpoints - cloud hosted\n", + "os.environ[\"OCR_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr\"\n", + "os.environ[\"OCR_INFER_PROTOCOL\"] = \"http\"\n", + "os.environ[\"OCR_MODEL_NAME\"] = \"scene_text_ensemble\"\n", + "os.environ[\"YOLOX_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3\"\n", + "os.environ[\"YOLOX_INFER_PROTOCOL\"] = \"http\"\n", + "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1\"\n", + "os.environ[\"YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL\"] = \"http\"\n", + "os.environ[\"YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT\"] = \"https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1\"\n", + "os.environ[\"YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL\"] = \"http\"\n", + "os.environ[\"APP_NVINGEST_CAPTIONENDPOINTURL\"] = \"https://integrate.api.nvidia.com/v1/chat/completions\"\n", + "\n", + "# VLM Model configuration - cloud hosted\n", + "os.environ[\"APP_VLM_MODELNAME\"] = \"nvidia/nemotron-nano-12b-v2-vl\"\n", + "os.environ[\"APP_VLM_SERVERURL\"] = \"https://integrate.api.nvidia.com/v1\"\n", + "os.environ[\"APP_LLM_SERVERURL\"] = \"\"\n", + "\n", + "# Multimodal embedding model configuration - cloud hosted\n", + "os.environ[\"APP_EMBEDDINGS_MODELNAME\"] = \"nvidia/llama-nemotron-embed-vl-1b-v2\"\n", + "os.environ[\"APP_EMBEDDINGS_SERVERURL\"] = \"https://integrate.api.nvidia.com/v1\"\n", + "os.environ[\"ENABLE_VLM_INFERENCE\"] = \"true\"\n", + "os.environ[\"VLM_TO_LLM_FALLBACK\"] = \"false\"\n", + "os.environ[\"ENABLE_RERANKER\"] = \"false\"" + ] + }, + { + "cell_type": "markdown", + "id": "7cbcfa50", + "metadata": {}, + "source": [ + "### 4. Setup NVIDIA Ingest Runtime\n", + "\n", + "NVIDIA Ingest processes documents to extract text, images, and other elements. We'll configure it to:\n", + "- Extract images from documents\n", + "- Handle multimodal content" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5e0d73f", + "metadata": {}, + "outputs": [], + "source": [ + "# Configure NVIDIA Ingest to extract and process images from documents\n", + "os.environ[\"APP_NVINGEST_STRUCTURED_ELEMENTS_MODALITY\"] = \"\" # No special handling for structured elements\n", + "os.environ[\"APP_NVINGEST_IMAGE_ELEMENTS_MODALITY\"] = \"image\" # Process image elements as images\n", + "os.environ[\"APP_NVINGEST_EXTRACTIMAGES\"] = \"True\" # Extract images from documents\n", + "\n", + "# Start the ingestor server with Redis\n", + "! docker compose -f ../deploy/compose/docker-compose-ingestor-server.yaml up -d --build" + ] + }, + { + "cell_type": "markdown", + "id": "da1bd9a3", + "metadata": {}, + "source": [ + "### 5. Setup the NVIDIA RAG Server\n", + "\n", + "The RAG server provides the main API endpoints for search and generation. It orchestrates all the components (embeddings, vector DB, VLM) to deliver intelligent responses." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38ba7752", + "metadata": {}, + "outputs": [], + "source": [ + "# Start the RAG server (accessible at localhost:8081)\n", + "os.environ[\"APP_RANKING_SERVERURL\"] = \"\"\n", + "! docker compose -f ../deploy/compose/docker-compose-rag-server.yaml up -d --build" + ] + }, + { + "cell_type": "markdown", + "id": "ce492ce3", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 📚 Document Ingestion Workflow\n", + "\n", + "Now that all services are running, let's ingest documents into a collection.\n", + "\n", + "### 6. Create a Collection\n", + "\n", + "A collection is a logical grouping of documents in the vector database. Think of it as a database table optimized for similarity search." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8611aa1", + "metadata": {}, + "outputs": [], + "source": [ + "# Install aiohttp for async HTTP requests\n", + "! uv pip install aiohttp\n", + "\n", + "# Configure the ingestor server URL\n", + "# Use \"ingestor-server\" when running in AI Workbench, otherwise \"localhost\"\n", + "IPADDRESS = (\n", + " \"ingestor-server\"\n", + " if os.environ.get(\"AI_WORKBENCH\", \"false\") == \"true\"\n", + " else \"localhost\"\n", + ")\n", + "INGESTOR_SERVER_PORT = \"8082\"\n", + "BASE_URL = f\"http://{IPADDRESS}:{INGESTOR_SERVER_PORT}\"\n", + "\n", + "async def print_response(response):\n", + " \"\"\"Helper function to pretty-print API responses.\"\"\"\n", + " try:\n", + " response_json = await response.json()\n", + " print(json.dumps(response_json, indent=2))\n", + " except aiohttp.ClientResponseError:\n", + " print(await response.text())\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "688bc70f", + "metadata": {}, + "outputs": [], + "source": [ + "# Define a unique name for your collection\n", + "# Change this if you want to create a different collection\n", + "collection_name = \"multimodal_query\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24378f6f", + "metadata": {}, + "outputs": [], + "source": [ + "import aiohttp\n", + "import json\n", + "\n", + "\n", + "async def create_collection(\n", + " collection_name: str | None = None,\n", + " metadata_schema: list = [],\n", + "):\n", + " \"\"\"\n", + " Create a new collection in the vector database.\n", + " \n", + " Args:\n", + " collection_name: Unique identifier for the collection\n", + " metadata_schema: Optional schema for metadata fields\n", + " \"\"\"\n", + " data = {\n", + " \"collection_name\": collection_name,\n", + " \"metadata_schema\": metadata_schema,\n", + " }\n", + "\n", + " HEADERS = {\"Content-Type\": \"application/json\"}\n", + "\n", + " async with aiohttp.ClientSession() as session:\n", + " try:\n", + " async with session.post(\n", + " f\"{BASE_URL}/v1/collection\", json=data, headers=HEADERS\n", + " ) as response:\n", + " await print_response(response)\n", + " except aiohttp.ClientError as e:\n", + " return 500, {\"error\": str(e)}\n", + "\n", + "\n", + "# Create the collection\n", + "# The embedding dimension is 2048 for the multimodal embedding model we're using\n", + "await create_collection(\n", + " collection_name=collection_name,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a29f4633", + "metadata": {}, + "outputs": [], + "source": [ + "# Specify the documents to upload\n", + "# This PDF contains product images with pricing information\n", + "FILEPATHS = [\n", + " \"../data/multimodal/product_catalog.pdf\",\n", + "]\n", + "\n", + "async def upload_documents(collection_name: str = \"\"):\n", + " \"\"\"\n", + " Upload and process documents into the collection.\n", + " \n", + " This will:\n", + " 1. Extract text and images from the PDFs\n", + " 2. Chunk the content for optimal retrieval\n", + " 3. Generate multimodal embeddings\n", + " 4. Store everything in the vector database\n", + " \"\"\"\n", + " data = {\n", + " \"collection_name\": collection_name,\n", + " \"blocking\": False, # Async upload - use status API to check progress\n", + " \"split_options\": {\n", + " \"chunk_size\": 512, # Characters per chunk\n", + " \"chunk_overlap\": 150 # Overlap between chunks for context\n", + " },\n", + " \"generate_summary\": False # Set to True to generate document summaries\n", + " }\n", + "\n", + " form_data = aiohttp.FormData()\n", + " \n", + " # Add all PDF files to the form data\n", + " for file_path in FILEPATHS:\n", + " form_data.add_field(\"documents\", open(file_path, \"rb\"), \n", + " filename=os.path.basename(file_path), \n", + " content_type=\"application/pdf\")\n", + "\n", + " form_data.add_field(\"data\", json.dumps(data), content_type=\"application/json\")\n", + "\n", + " async with aiohttp.ClientSession() as session:\n", + " try:\n", + " # Use POST for new uploads, PATCH for re-ingesting existing documents\n", + " async with session.post(f\"{BASE_URL}/v1/documents\", data=form_data) as response:\n", + " await print_response(response)\n", + " response_json = await response.json()\n", + " return response_json\n", + " except aiohttp.ClientError as e:\n", + " print(f\"Error uploading documents: {e}\")\n", + " return None\n", + "\n", + "# Upload the documents and get the task ID for tracking progress\n", + "upload_response = await upload_documents(collection_name=collection_name)\n", + "task_id = upload_response.get(\"task_id\") if upload_response else None\n", + "print(f\"\\nTask ID for tracking: {task_id}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2234e059", + "metadata": {}, + "outputs": [], + "source": [ + "async def get_task_status(task_id: str):\n", + " \"\"\"\n", + " Check the status of an asynchronous ingestion task.\n", + " \n", + " Possible statuses:\n", + " - \"pending\": Task is queued\n", + " - \"processing\": Currently processing documents\n", + " - \"completed\": Successfully finished\n", + " - \"failed\": Error occurred\n", + " \"\"\"\n", + " params = {\n", + " \"task_id\": task_id,\n", + " }\n", + "\n", + " HEADERS = {\"Content-Type\": \"application/json\"}\n", + "\n", + " async with aiohttp.ClientSession() as session:\n", + " try:\n", + " async with session.get(\n", + " f\"{BASE_URL}/v1/status\", params=params, headers=HEADERS\n", + " ) as response:\n", + " await print_response(response)\n", + " except aiohttp.ClientError as e:\n", + " return 500, {\"error\": str(e)}\n", + "\n", + "\n", + "# Check the ingestion status\n", + "# Run this cell multiple times until status shows \"completed\"\n", + "await get_task_status(\n", + " task_id=[task_id]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "e2c1f1a8", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 🔍 Querying with Multimodal Inputs\n", + "\n", + "Now that documents are ingested, let's query them using both text and images!\n", + "\n", + "### 7. Using the Search and Generate APIs\n", + "\n", + "We'll demonstrate two approaches:\n", + "1. **Search API**: Find relevant documents without generating a response\n", + "2. **Generate API**: Get an AI-generated answer with citations" + ] + }, + { + "cell_type": "markdown", + "id": "3990ca33", + "metadata": {}, + "source": [ + "#### Prepare a Multimodal Query\n", + "\n", + "To query with an image, we need to:\n", + "1. Convert the image to base64 encoding\n", + "2. Format it according to the OpenAI vision API format\n", + "3. Combine it with a text prompt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02dde830", + "metadata": {}, + "outputs": [], + "source": [ + "! uv pip install requests httpx\n", + "import base64\n", + "import requests\n", + "from IPython.display import Image, Markdown, display\n", + "\n", + "def get_base64_image(image_source: str) -> str:\n", + " \"\"\"\n", + " Convert an image to base64 encoding.\n", + " \n", + " Args:\n", + " image_source: Local file path or URL to the image\n", + " \n", + " Returns:\n", + " Base64 encoded string of the image\n", + " \"\"\"\n", + " if image_source.startswith(('http://', 'https://')):\n", + " # Download image from URL\n", + " response = requests.get(image_source)\n", + " return base64.b64encode(response.content).decode()\n", + " else:\n", + " # Read local file\n", + " with open(image_source, \"rb\") as image_file:\n", + " return base64.b64encode(image_file.read()).decode()\n", + "\n", + "# Convert the query image to base64\n", + "# Try different images to test different queries:\n", + "image_b64 = get_base64_image(\"../data/multimodal/Creme_clutch_purse1-small.jpg\")\n", + "\n", + "# Display the query image for reference\n", + "query_image_path = \"../data/multimodal/Creme_clutch_purse1-small.jpg\"\n", + "print(\"📷 Query Image:\")\n", + "display(Image(filename=query_image_path, width=300))\n", + "\n", + "# Format as a data URL\n", + "image_input = f\"data:image/png;base64,{image_b64}\"\n", + "\n", + "# Create the multimodal query with text + image\n", + "# This follows the OpenAI vision API format\n", + "query_1 = \"What material is this made of?\"\n", + "image_query = [\n", + " {\"type\": \"text\", \"text\": query_1},\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": image_input,\n", + " \"detail\": \"auto\" # Let the model decide the appropriate detail level\n", + " }\n", + " }\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c311f9d0", + "metadata": {}, + "outputs": [], + "source": [ + "import httpx\n", + "import json\n", + "from IPython.display import Image, Markdown, display\n", + "\n", + "RAG_BASE_URL = \"http://localhost:8081\"\n", + "\n", + "async def search_documents(payload):\n", + " \"\"\"\n", + " Search for relevant documents using a multimodal query.\n", + " \n", + " This performs similarity search in the vector database and optionally\n", + " reranks results for better relevance.\n", + " \"\"\"\n", + " search_url = f\"{RAG_BASE_URL}/v1/search\"\n", + " \n", + " async with httpx.AsyncClient(timeout=300.0) as client:\n", + " try:\n", + " response = await client.post(url=search_url, json=payload)\n", + " response.raise_for_status()\n", + " \n", + " search_results = response.json()\n", + " print(\"Search Results:\")\n", + " \n", + " # Display search results with nice formatting\n", + " if \"results\" in search_results:\n", + " for idx, result in enumerate(search_results[\"results\"]):\n", + " doc_type = result.get(\"document_type\", \"text\")\n", + " content = result.get(\"content\", \"\")\n", + " doc_name = result.get(\"document_name\", f\"Result {idx + 1}\")\n", + " score = result.get(\"score\", \"N/A\")\n", + " \n", + " display(Markdown(f\"**Result {idx + 1}: {doc_name} (Score: {score})**\"))\n", + " try:\n", + " if doc_type == \"image\":\n", + " # Display image results\n", + " image_bytes = base64.b64decode(content)\n", + " display(Image(data=image_bytes))\n", + " else:\n", + " # Display text results\n", + " display(Markdown(f\"```\\n{content}\\n```\"))\n", + " except Exception as e:\n", + " print(f\"Error displaying content: {e}\")\n", + " display(Markdown(f\"```\\n{content}\\n```\"))\n", + " \n", + " return search_results\n", + " \n", + " except httpx.HTTPStatusError as e:\n", + " print(f\"HTTP error occurred: {e.response.status_code} - {e.response.text}\")\n", + " except httpx.RequestError as e:\n", + " print(f\"An error occurred while requesting {e.request.url!r}: {e}\")\n", + " except Exception as e:\n", + " print(f\"An error occurred: {e}\")\n", + "\n", + "# Configure the search parameters\n", + "search_payload = {\n", + " \"query\": image_query, # Our multimodal query (text + image)\n", + " \"messages\": [], # No conversation history\n", + " \"use_knowledge_base\": True, # Search the vector database\n", + " \"collection_names\": [collection_name], # Which collection to search\n", + " \"vdb_top_k\": 5, # Retrieve top 5 results from vector DB\n", + " \"vdb_endpoint\": \"http://milvus:19530\", # Milvus connection string\n", + " \"enable_reranker\": False, # Set to True for better relevance (slower)\n", + " \"reranker_top_k\": 3, # If reranker enabled, return top 3\n", + " \"filter_expr\": \"\", # Optional metadata filter\n", + "}\n", + "\n", + "# Execute the search\n", + "print(\"🔍 Searching for documents matching the query...\\n\")\n", + "search_result = await search_documents(search_payload)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc5b1545", + "metadata": {}, + "outputs": [], + "source": [ + "import base64\n", + "import json\n", + "from IPython.display import Image, Markdown, display\n", + "\n", + "\n", + "async def print_streaming_response_and_citations(response_generator):\n", + " \"\"\"\n", + " Helper function to display streaming responses with citations.\n", + " \n", + " This function:\n", + " 1. Streams the AI-generated response token by token\n", + " 2. Extracts citations from the first chunk\n", + " 3. Displays citations (text or images) after the response completes\n", + " \"\"\"\n", + " first_chunk_data = None\n", + " \n", + " async for chunk in response_generator:\n", + " # Parse Server-Sent Events (SSE) format\n", + " if chunk.startswith(\"data: \"):\n", + " chunk = chunk[len(\"data: \") :].strip()\n", + " if not chunk:\n", + " continue\n", + " \n", + " try:\n", + " data = json.loads(chunk)\n", + " except Exception as e:\n", + " print(f\"JSON decode error: {e}\")\n", + " continue\n", + " \n", + " choices = data.get(\"choices\", [])\n", + " if not choices:\n", + " continue\n", + " \n", + " # Save the first chunk with citations\n", + " if first_chunk_data is None and data.get(\"citations\"):\n", + " first_chunk_data = data\n", + " \n", + " # Print streaming text\n", + " delta = choices[0].get(\"delta\", {})\n", + " text = delta.get(\"content\")\n", + " if not text:\n", + " message = choices[0].get(\"message\", {})\n", + " text = message.get(\"content\", \"\")\n", + " print(text, end=\"\", flush=True)\n", + " \n", + " print() # Newline after streaming\n", + "\n", + " # Display citations after streaming is done\n", + " if first_chunk_data and first_chunk_data.get(\"citations\"):\n", + " print(\"\\n📚 Citations:\")\n", + " citations = first_chunk_data[\"citations\"]\n", + " for idx, citation in enumerate(citations.get(\"results\", [])):\n", + " doc_type = citation.get(\"document_type\", \"text\")\n", + " content = citation.get(\"content\", \"\")\n", + " doc_name = citation.get(\"document_name\", f\"Citation {idx + 1}\")\n", + " display(Markdown(f\"**Citation {idx + 1}: {doc_name}**\"))\n", + " try:\n", + " # Try to display as image\n", + " image_bytes = base64.b64decode(content)\n", + " display(Image(data=image_bytes))\n", + " except Exception:\n", + " # Fall back to text display\n", + " display(Markdown(f\"```\\n{content}\\n```\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31071359", + "metadata": {}, + "outputs": [], + "source": [ + "import httpx\n", + "\n", + "# Configure RAG server URL\n", + "IPADDRESS = \"rag-server\" if os.environ.get(\"AI_WORKBENCH\", \"false\") == \"true\" else \"localhost\"\n", + "RAG_SERVER_PORT = \"8081\"\n", + "RAG_BASE_URL = f\"http://{IPADDRESS}:{RAG_SERVER_PORT}\"\n", + "generate_url = f\"{RAG_BASE_URL}/v1/generate\"\n", + "\n", + "async def generate_answer(payload):\n", + " \"\"\"\n", + " Generate an AI answer using the RAG pipeline.\n", + " \n", + " This function:\n", + " 1. Sends the query to the RAG server\n", + " 2. Retrieves relevant context from the vector database\n", + " 3. Streams the AI-generated response\n", + " 4. Displays citations (sources) used to generate the answer\n", + " \"\"\"\n", + " rag_response = \"\"\n", + " citations = []\n", + " is_first_token = True\n", + "\n", + " async with httpx.AsyncClient(timeout=300.0) as client:\n", + " try:\n", + " async with client.stream(\"POST\", url=generate_url, json=payload) as response:\n", + " # Raise an exception for bad status codes like 4xx or 5xx\n", + " response.raise_for_status()\n", + "\n", + " # Iterate over the streaming response\n", + " async for line in response.aiter_lines():\n", + " if line.startswith(\"data: \"):\n", + " json_str = line[6:].strip()\n", + " if not json_str:\n", + " continue\n", + "\n", + " try:\n", + " data = json.loads(json_str)\n", + "\n", + " # Extract and display the streaming response\n", + " message = data.get(\"choices\", [{}])[0].get(\"message\", {}).get(\"content\", \"\")\n", + " if message:\n", + " rag_response += message\n", + "\n", + " # Extract and display citations from the first chunk\n", + " if is_first_token and data.get(\"citations\"):\n", + " print(\"\\n📚 Citations:\")\n", + " citations = data[\"citations\"]\n", + " for idx, citation in enumerate(citations.get(\"results\", [])):\n", + " doc_type = citation.get(\"document_type\", \"text\")\n", + " content = citation.get(\"content\", \"\")\n", + " doc_name = citation.get(\"document_name\", f\"Citation {idx + 1}\")\n", + " display(Markdown(f\"**Citation {idx + 1}: {doc_name}**\"))\n", + " try:\n", + " # Display image citations\n", + " image_bytes = base64.b64decode(content)\n", + " display(Image(data=image_bytes))\n", + " except Exception:\n", + " # Display text citations\n", + " display(Markdown(f\"```\\n{content}\\n```\"))\n", + " is_first_token = False\n", + "\n", + " # Check if streaming is complete\n", + " finish_reason = data.get(\"choices\", [{}])[0].get(\"finish_reason\")\n", + " if finish_reason == \"stop\":\n", + " return rag_response\n", + "\n", + " except json.JSONDecodeError:\n", + " print(f\"Skipping malformed JSON line: {json_str}\")\n", + " continue\n", + " \n", + " except httpx.HTTPStatusError as e:\n", + " print(f\"HTTP error occurred: {e.response.status_code} - {e.response.text}\")\n", + " except httpx.RequestError as e:\n", + " print(f\"An error occurred while requesting {e.request.url!r}: {e}\")\n", + " except Exception as e:\n", + " print(f\"An error occurred: {e}\")\n", + "\n", + " print(\"\\n✅ Response complete!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3049696", + "metadata": {}, + "outputs": [], + "source": [ + "# Format the query as a chat message\n", + "messages = [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": image_query # Our multimodal query (text + image)\n", + " }\n", + "]\n", + "\n", + "# Configure the generate API parameters\n", + "payload = {\n", + " \"messages\": messages, # Chat conversation\n", + " \"use_knowledge_base\": True, # Enable RAG - use vector DB for context\n", + " \"temperature\": 0.2, # Lower = more deterministic, higher = more creative\n", + " \"top_p\": 0.7, # Nucleus sampling parameter\n", + " \"max_tokens\": 1024, # Maximum response length\n", + " \"reranker_top_k\": 2, # Keep top 2 results after reranking\n", + " \"vdb_top_k\": 10, # Retrieve top 10 from vector DB initially\n", + " \"vdb_endpoint\": \"http://milvus:19530\", # Milvus connection\n", + " \"collection_names\": [collection_name], # Which collection to search\n", + " \"enable_query_rewriting\": True, # Improve query before searching\n", + " \"enable_citations\": True, # Include source citations in response\n", + " \"stop\": [], # Optional stop sequences\n", + " \"filter_expr\": \"\", # Optional metadata filter \n", + "}\n", + "\n", + "# Generate the answer with RAG\n", + "print(\"🤖 Generating answer with RAG...\\n\")\n", + "await generate_answer(payload)" + ] + }, + { + "cell_type": "markdown", + "id": "dcddc78d", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 🎉 Summary\n", + "\n", + "Congratulations! You've successfully:\n", + "\n", + "✅ **Set up the infrastructure**: Deployed Milvus vector DB, NVIDIA NIMs, and RAG services \n", + "✅ **Ingested multimodal documents**: Uploaded PDFs with images and extracted their content \n", + "✅ **Created multimodal queries**: Combined text and images in your search queries \n", + "✅ **Retrieved relevant context**: Used semantic search to find matching documents \n", + "✅ **Generated AI responses**: Got intelligent answers with source citations \n", + "\n", + "### Next Steps\n", + "\n", + "- **Try different queries**: Change the query text or use different query images\n", + "- **Upload more documents**: Add more PDFs to enrich your knowledge base\n", + "- **Experiment with parameters**: Adjust `temperature`, `top_k`, reranker settings\n", + "- **Build applications**: Integrate these APIs into your own applications\n", + "\n", + "### Cleanup\n", + "\n", + "To stop all services and free up resources:\n", + "\n", + "```bash\n", + "cd ../deploy/compose\n", + "docker compose -f docker-compose-rag-server.yaml down\n", + "docker compose -f docker-compose-ingestor-server.yaml down\n", + "docker compose -f nims.yaml down\n", + "docker compose -f vectordb.yaml down\n", + "```\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/notebooks/launchable.ipynb b/notebooks/launchable.ipynb index 1bc8e793b..22e293f90 100644 --- a/notebooks/launchable.ipynb +++ b/notebooks/launchable.ipynb @@ -792,7 +792,7 @@ " print(f\" • {warn}\")\n", " print(\"\\n\" + \"=\" * 70)\n", " print(\"Please fix the above errors before continuing.\")\n", - " print(\"See: https://docs.nvidia.com/ai-blueprints/rag/latest/support-matrix.html\")\n", + " print(\"See: https://docs.nvidia.com/rag/latest/support-matrix.html\")\n", " print(\"=\" * 70)\n", "elif warnings:\n", " print(\"\\n✅ REQUIREMENTS MET (with warnings)\")\n", @@ -2030,9 +2030,9 @@ "\n", "## 📚 Additional Resources\n", "\n", - "- **Documentation**: https://docs.nvidia.com/ai-blueprints/rag/latest/\n", + "- **Documentation**: https://docs.nvidia.com/rag/latest/\n", "- **GitHub**: https://github.com/NVIDIA-AI-Blueprints/rag\n", - "- **Support Matrix**: https://docs.nvidia.com/ai-blueprints/rag/latest/support-matrix.html\n", + "- **Support Matrix**: https://docs.nvidia.com/rag/latest/support-matrix.html\n", "\n", "---\n", "\n", diff --git a/notebooks/rag_event_ingest.ipynb b/notebooks/rag_event_ingest.ipynb index b2bde4727..a38f976af 100644 --- a/notebooks/rag_event_ingest.ipynb +++ b/notebooks/rag_event_ingest.ipynb @@ -1,793 +1,793 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Document Continuous Ingestion from Object Storage\n", - "\n", - "## Purpose\n", - "\n", - "This notebook demonstrates an **automated document ingestion pipeline** that:\n", - "\n", - "1. Monitors emulated object storage for new uploads via Kafka events\n", - "2. Routes documents to appropriate AI services for indexing\n", - "5. Enables RAG Agent for semantic search and contextual Q&A over all ingested content\n", - "\n", - "## What Gets Deployed\n", - "\n", - "1. **NVIDIA RAG** - Document indexing, vector search, and AI-powered Q&A (NIMs, Milvus, Ingestor)\n", - "2. **Continuous Ingestion** - Event-driven ingestion pipeline (Kafka, MinIO, Consumer)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prerequisites\n", - "\n", - "### Hardware\n", - "- **GPU**: 2x RTX PRO 6000 Blackwell or 2x H100\n", - "\n", - "#### Default GPU Assignment\n", - "\n", - "| GPU | Service |\n", - "|-----|---------|\n", - "| 0 | RAG NIMs (Embedding, Reranker) |\n", - "| 1 | RAG LLM NIM (Llama-3.3-Nemotron-Super-49B) |\n", - "\n", - "\n", - "### Software (pre-installed required)\n", - "- Ubuntu 22.04 or later\n", - "- Docker 24.0+ with Docker Compose v2\n", - "- NVIDIA Driver 570+\n", - "- NVIDIA Container Toolkit\n", - "\n", - "### API Keys\n", - "\n", - "\n", - "\n", - "\n", - "
KeyPurposeHow to Get
NGC_API_KEYDocker login, NIM deploymentsNGC Portal → Generate API Key
\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Table of Contents\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SectionDescription
SetupClone repo, install deps, set API keys, load helpers
Deploy RAGNIMs, Vector DB, Ingestor, RAG Server
Deploy Continuous IngestionKafka, MinIO, Consumer
TestingUpload documents, query RAG
Clean UpStop services, clean data
\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## References\n", - "\n", - "- **RAG Blueprint**: [NVIDIA RAG Documentation](https://github.com/NVIDIA-AI-Blueprints/rag/blob/develop/docs/deploy-docker-self-hosted.md)\n", - "- **NIM**: [NVIDIA NIM Documentation](https://docs.nvidia.com/nim/index.html)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Setup\n", - "\n", - "Clone the repository, configure API keys, and load helper functions.\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Clone Repository\n", - "\n", - "Clone the RAG Blueprint repo to `~/rag`. This includes the consumer source code, deploy configs, and sample test data.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess, sys, os, shutil\n", - "\n", - "RAG_REPO_DIR = os.path.expanduser(\"~/rag\")\n", - "RAG_REPO_URL = \"https://github.com/NVIDIA-AI-Blueprints/rag.git\"\n", - "\n", - "# Ensure git-lfs is installed before any LFS operations\n", - "if not shutil.which(\"git-lfs\"):\n", - " print(\"[INSTALLING] git-lfs...\")\n", - " subprocess.run(\"sudo apt-get update && sudo apt-get install -y git-lfs && git lfs install\", shell=True, check=True)\n", - "else:\n", - " print(\"[OK] git-lfs found\")\n", - "\n", - "# Clone from correct branch (skip if already exists)\n", - "if not os.path.exists(RAG_REPO_DIR):\n", - " subprocess.run(f\"git clone {RAG_REPO_URL} {RAG_REPO_DIR}\", shell=True, check=True)\n", - "else:\n", - " print(f\"[OK] RAG repo already exists: {RAG_REPO_DIR}\")\n", - "subprocess.run(\"git lfs pull\", shell=True, cwd=RAG_REPO_DIR, check=True)\n", - "\n", - "# Verify\n", - "for path in [\"deploy/compose\", \"examples/rag_event_ingest/kafka_consumer\", \"examples/rag_event_ingest/data\"]:\n", - " status = \"[OK]\" if os.path.exists(os.path.join(RAG_REPO_DIR, path)) else \"[MISSING]\"\n", - " print(f\" {status} {path}\")\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Install Dependencies\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "! python3 -m ensurepip --upgrade" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Ensure pip is available (some minimal Python installs lack it)\n", - "subprocess.run([sys.executable, \"-m\", \"ensurepip\", \"--upgrade\"], capture_output=True)\n", - "\n", - "def check_install_system_pkg(cmd: str, install_cmd: str):\n", - " if shutil.which(cmd):\n", - " print(f\" [OK] {cmd} found\")\n", - " return True\n", - " print(f\" [INSTALLING] {cmd}...\")\n", - " result = subprocess.run(install_cmd, shell=True, capture_output=True, text=True)\n", - " if result.returncode == 0:\n", - " print(f\" [OK] {cmd} installed\")\n", - " return True\n", - " print(f\" [ERROR] Failed to install {cmd}. Please install manually: {install_cmd}\")\n", - " return False\n", - "\n", - "check_install_system_pkg(\"git\", \"sudo apt-get update && sudo apt-get install -y git\")\n", - "\n", - "# Install Python packages\n", - "subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"minio\", \"aiohttp\", \"requests\", \"python-dotenv\", \"pyyaml\"])\n", - "print(\"[OK] Ready\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Set API Keys\n", - "\n", - "Configure NGC API key for NIM deployments.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import getpass\n", - "\n", - "def set_api_key(env_var: str, prompt: str, required: bool = True):\n", - " if os.environ.get(env_var):\n", - " print(f\" [OK] {env_var} already set ({os.environ[env_var][:10]}...)\")\n", - " return True\n", - " key = getpass.getpass(prompt)\n", - " if key:\n", - " os.environ[env_var] = key\n", - " print(f\" [OK] {env_var} set\")\n", - " return True\n", - " if required:\n", - " print(f\" [ERROR] {env_var} is required\")\n", - " return False\n", - " print(f\" [SKIP] {env_var} (optional)\")\n", - " return True\n", - "\n", - "set_api_key(\"NGC_API_KEY\", \"Enter NGC_API_KEY (starts with 'nvapi-'): \", required=True)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Helper Functions\n", - "\n", - "Shared utilities for deployment, file upload, status checks, and RAG queries.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Install dependencies\n", - "import sys\n", - "!{sys.executable} -m pip install -q minio aiohttp requests python-dotenv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os, sys, json, re, subprocess, time, socket, asyncio\n", - "import aiohttp, requests\n", - "from typing import List, Optional, Dict\n", - "\n", - "try:\n", - " from minio import Minio\n", - " from minio.error import S3Error\n", - "except ImportError:\n", - " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"minio\"])\n", - " from minio import Minio\n", - " from minio.error import S3Error\n", - "\n", - "# =============================================================================\n", - "# CONFIGURATION\n", - "# =============================================================================\n", - "\n", - "# Paths relative to RAG repo root\n", - "RAG_REPO_DIR = os.path.expanduser(\"~/rag\")\n", - "EXAMPLE_DIR = os.path.join(RAG_REPO_DIR, \"examples/rag_event_ingest\")\n", - "AIDP_COMPOSE_FILE = os.path.join(EXAMPLE_DIR, \"deploy/docker-compose.yaml\")\n", - "DATA_DIR = os.path.join(EXAMPLE_DIR, \"data\")\n", - "RAG_SERVER_URL = \"http://localhost:8081\"\n", - "INGESTOR_URL = \"http://localhost:8082\"\n", - "\n", - "LOCAL_NIM_CACHE = os.path.expanduser(\"~/.cache/nim\")\n", - "\n", - "MINIO_ENDPOINT = \"localhost:9201\"\n", - "MINIO_ACCESS_KEY = \"minioadmin\"\n", - "MINIO_SECRET_KEY = \"minioadmin\"\n", - "MINIO_BUCKET = \"aidp-bucket\"\n", - "MINIO_COLLECTION = \"aidp_bucket\"\n", - "MINIO_CONSOLE_PORT = 9211\n", - "\n", - "# =============================================================================\n", - "# SHARED UTILITIES\n", - "# =============================================================================\n", - "\n", - "def run_command(cmd: str, capture: bool = False) -> Optional[str]:\n", - " \"\"\"Execute a shell command and print it.\"\"\"\n", - " print(f\"$ {cmd}\")\n", - " result = subprocess.run(cmd, shell=True, capture_output=capture, text=True)\n", - " return result.stdout if capture else None\n", - "\n", - "def get_host_ip() -> str:\n", - " \"\"\"Get host IP address for external access URLs.\"\"\"\n", - " try:\n", - " s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n", - " s.connect((\"8.8.8.8\", 80))\n", - " ip = s.getsockname()[0]\n", - " s.close()\n", - " return ip\n", - " except OSError:\n", - " return \"localhost\"\n", - "\n", - "def get_minio_client() -> Minio:\n", - " \"\"\"Create MinIO client for AIDP bucket operations.\"\"\"\n", - " return Minio(MINIO_ENDPOINT, access_key=MINIO_ACCESS_KEY, secret_key=MINIO_SECRET_KEY, secure=False)\n", - "\n", - "def upload_file(local_path: str, object_name: Optional[str] = None) -> bool:\n", - " \"\"\"Upload a local file to MinIO AIDP bucket.\"\"\"\n", - " if not os.path.exists(local_path):\n", - " print(f\"[ERROR] File not found: {local_path}\")\n", - " return False\n", - " obj = object_name or os.path.basename(local_path)\n", - " try:\n", - " client = get_minio_client()\n", - " if not client.bucket_exists(MINIO_BUCKET):\n", - " client.make_bucket(MINIO_BUCKET)\n", - " client.fput_object(MINIO_BUCKET, obj, local_path)\n", - " print(f\"[OK] Uploaded: {obj}\")\n", - " return True\n", - " except S3Error as e:\n", - " print(f\"[ERROR] {e}\")\n", - " return False\n", - "\n", - "def verify_file_in_storage(object_name: str, bucket: str = MINIO_BUCKET) -> bool:\n", - " \"\"\"Check if a file exists in MinIO bucket and print verification status.\"\"\"\n", - " try:\n", - " client = get_minio_client()\n", - " stat = client.stat_object(bucket, object_name)\n", - " print(f\"[OK] File verified in storage:\")\n", - " print(f\" Bucket: {bucket}\")\n", - " print(f\" Object: {object_name}\")\n", - " print(f\" Size: {stat.size:,} bytes\")\n", - " print(f\" Modified: {stat.last_modified}\")\n", - " return True\n", - " except S3Error as e:\n", - " print(f\"[ERROR] File not found in storage: {object_name}\")\n", - " print(f\" Error: {e}\")\n", - " return False\n", - "\n", - "def get_consumer_logs(lines: int = 30) -> None:\n", - " \"\"\"Show recent Kafka consumer logs.\"\"\"\n", - " run_command(f\"docker logs kafka-consumer --tail {lines}\")\n", - "\n", - "async def query_rag(question: str, collection: str = None) -> Optional[str]:\n", - " \"\"\"Query RAG system and print the answer.\"\"\"\n", - " coll = collection or MINIO_COLLECTION\n", - " print(f\"Q: {question}\\nCollection: {coll}\\n\" + \"-\" * 40)\n", - "\n", - " payload = {\n", - " \"messages\": [{\"role\": \"user\", \"content\": question}],\n", - " \"use_knowledge_base\": True,\n", - " \"collection_names\": [coll],\n", - " }\n", - " try:\n", - " async with aiohttp.ClientSession() as session:\n", - " async with session.post(\n", - " f\"{RAG_SERVER_URL}/generate\", json=payload,\n", - " timeout=aiohttp.ClientTimeout(total=120),\n", - " ) as resp:\n", - " text = await resp.text()\n", - " # Parse SSE response: extract content from each \"data: {...}\" line\n", - " chunks = []\n", - " for line in text.split(\"\\n\"):\n", - " if not line.startswith(\"data: \") or line[6:] == \"[DONE]\":\n", - " continue\n", - " try:\n", - " msg = json.loads(line[6:]).get(\"choices\", [{}])[0].get(\"message\", {})\n", - " if msg.get(\"content\"):\n", - " chunks.append(msg[\"content\"])\n", - " except json.JSONDecodeError:\n", - " pass\n", - " answer = \"\".join(chunks)\n", - " print(f\"Answer: {answer}\")\n", - " return answer\n", - " except aiohttp.ClientError as e:\n", - " print(f\"[ERROR] {e}\")\n", - " return None\n", - "\n", - "print(f\"[OK] Helpers loaded | Host IP: {get_host_ip()}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Deploy NVIDIA RAG\n", - "\n", - "Deploy the NVIDIA RAG: NIMs (LLM, Embedding, Reranker), Milvus vector database, Ingestor server, and RAG server.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ngc_key = os.environ.get(\"NGC_API_KEY\")\n", - "if not ngc_key:\n", - " raise RuntimeError(\"NGC_API_KEY not set! Run the API keys cell first.\")\n", - "\n", - "os.chdir(RAG_REPO_DIR)\n", - "\n", - "# Set env vars needed by docker compose\n", - "os.environ[\"NGC_API_KEY\"] = ngc_key\n", - "os.environ[\"USERID\"] = f\"{os.getuid()}:{os.getgid()}\"\n", - "os.environ[\"COLLECTION_NAME\"] = MINIO_COLLECTION\n", - "\n", - "# Load RAG .env defaults (MODEL_DIRECTORY, etc.)\n", - "from dotenv import load_dotenv\n", - "env_file = os.path.join(RAG_REPO_DIR, \"deploy/compose/.env\")\n", - "if os.path.exists(env_file):\n", - " load_dotenv(env_file, override=False)\n", - "\n", - "# Login to nvcr.io\n", - "subprocess.run(f\"echo {ngc_key} | docker login nvcr.io -u '$oauthtoken' --password-stdin\",\n", - " shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", - "\n", - "# Deploy components\n", - "for label, compose_file in [\n", - " (\"NIMs\", \"deploy/compose/nims.yaml\"),\n", - " (\"Vector DB\", \"deploy/compose/vectordb.yaml\"),\n", - "]:\n", - " print(f\"Deploying {label}...\")\n", - " run_command(f\"USERID=$(id -u) docker compose -f {compose_file} up -d\")\n", - "\n", - "print(\"Waiting 30s for Milvus...\")\n", - "time.sleep(30)\n", - "\n", - "for label, compose_file in [\n", - " (\"Ingestor\", \"deploy/compose/docker-compose-ingestor-server.yaml\"),\n", - " (\"RAG Server\", \"deploy/compose/docker-compose-rag-server.yaml\"),\n", - "]:\n", - " print(f\"Deploying {label}...\")\n", - " run_command(f\"docker compose -f {compose_file} up -d\")\n", - "\n", - "ip = get_host_ip()\n", - "print(f\"\\nRAG deployed: http://{ip}:8081 (server) | http://{ip}:8082 (ingestor) | http://{ip}:8090 (UI)\")\n", - "print(f\"COLLECTION_NAME: {MINIO_COLLECTION}\")\n", - "print(\"Wait ~10 minutes for NIMs to load models, then run the status check cell.\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Verify RAG services are healthy. Wait ~10 minutes for NIMs to load models.\n", - "\n", - "The deployment status should be:\n", - "```\n", - "NAMES STATUS\n", - "rag-frontend Up About a minute\n", - "rag-server Up About a minute\n", - "ingestor-server Up About a minute\n", - "milvus-standalone Up 2 minutes (healthy)\n", - "milvus-etcd Up 2 minutes (healthy)\n", - "milvus-minio Up 2 minutes (healthy)\n", - "nim-llm-ms Up 2 minutes (healthy)\n", - "nemotron-embedding-ms Up 2 minutes (healthy)\n", - "nemotron-ranking-ms Up 2 minutes (healthy)\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Check service status and print access URLs\n", - "print(\"Wait ~10 minutes for services to become healthy.\")\n", - "print(\"Run this cell again after waiting.\\n\")\n", - "\n", - "ip = get_host_ip()\n", - "for name, port, path in [\n", - " (\"RAG Server\", 8081, \"/health\"), (\"Ingestor\", 8082, \"/health\"),\n", - " (\"Frontend\", 8090, \"/\"), (\"Milvus\", 19530, \"/v1/vector/collections\"),\n", - "]:\n", - " try:\n", - " s = \"[OK]\" if requests.get(f\"http://localhost:{port}{path}\", timeout=10).status_code == 200 else \"[WARN]\"\n", - " except requests.ConnectionError:\n", - " s = \"[DOWN]\"\n", - " except requests.Timeout:\n", - " s = \"[TIMEOUT]\"\n", - " print(f\" {s} {name}: http://{ip}:{port}\")\n", - "run_command(\"docker ps --format 'table {{.Names}}\\t{{.Status}}' | grep -E '(rag|milvus|ingestor|nim|nemotron|NAMES)'\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Deploy Continuous Ingestion from emulated object storage\n", - "\n", - "Deploy the Continuous Ingestion: Kafka message broker, MinIO object storage, and Kafka consumer for automated ingestion.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Deploy Services\n", - "\n", - "Deploy Kafka, MinIO, and the Kafka consumer." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Verify prerequisites\n", - "net_check = subprocess.run(\"docker network inspect nvidia-rag\", shell=True, capture_output=True)\n", - "if net_check.returncode != 0:\n", - " raise RuntimeError(\"nvidia-rag network not found. Deploy RAG first.\")\n", - "\n", - "ngc_key = os.environ.get(\"NGC_API_KEY\", \"\")\n", - "if not ngc_key:\n", - " raise RuntimeError(\"NGC_API_KEY not set!\")\n", - "\n", - "host_ip = get_host_ip()\n", - "\n", - "# Set environment variables for docker compose\n", - "os.environ[\"HOST_IP\"] = host_ip\n", - "\n", - "# Login + pull + build\n", - "subprocess.run(f\"echo {ngc_key} | docker login nvcr.io -u '$oauthtoken' --password-stdin\",\n", - " shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", - "\n", - "compose = f\"docker compose -f {AIDP_COMPOSE_FILE}\"\n", - "subprocess.run(f\"{compose} pull --ignore-pull-failures\", shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", - "subprocess.run(f\"{compose} up -d --build\", shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", - "\n", - "print(f\"Continuous Ingestion deployed:\")\n", - "print(f\" Kafka UI: http://{host_ip}:8080\")\n", - "print(f\" MinIO Console: http://{host_ip}:{MINIO_CONSOLE_PORT}\")\n", - "print(f\" Credentials: minioadmin / minioadmin\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Verify continuous ingestion services are running.\n", - "\n", - "The deployment status should be:\n", - "```\n", - "NAMES STATUS\n", - "kafka-consumer Up About a minute\n", - "aidp-kafka-ui Up About a minute\n", - "aidp-minio-mc Up About a minute\n", - "aidp-minio Up About a minute (healthy)\n", - "kafka Up About a minute (healthy)\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Check service status and print access URLs\n", - "ip = get_host_ip()\n", - "for name, port, path in [\n", - " (\"Kafka UI\", 8080, \"/\"),\n", - " (\"MinIO Console\", MINIO_CONSOLE_PORT, \"/\"),\n", - "]:\n", - " try:\n", - " s = \"[OK]\" if requests.get(f\"http://localhost:{port}{path}\", timeout=10).status_code == 200 else \"[WARN]\"\n", - " except requests.ConnectionError:\n", - " s = \"[DOWN]\"\n", - " except requests.Timeout:\n", - " s = \"[TIMEOUT]\"\n", - " print(f\" {s} {name}: http://{ip}:{port}\")\n", - "\n", - "# Check kafka-consumer container status\n", - "result = subprocess.run(\"docker inspect -f '{{.State.Status}}' kafka-consumer 2>/dev/null\",\n", - " shell=True, capture_output=True, text=True)\n", - "status = result.stdout.strip()\n", - "s = \"[OK]\" if status == \"running\" else \"[DOWN]\"\n", - "print(f\" {s} Kafka Consumer: {status or 'not found'}\")\n", - "\n", - "run_command(\"docker ps --format 'table {{.Names}}\\t{{.Status}}' | grep -E '(kafka|minio|NAMES)'\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Testing\n", - "\n", - "Test the deployment by uploading documents, then querying via RAG.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Document Upload\n", - "\n", - "Upload a PDF document to MinIO, which triggers automatic ingestion via Kafka consumer.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.1 Upload to Storage\n", - "\n", - "Upload the document to MinIO object storage.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Sample documents are included in the repo under examples/rag_event_ingest/data/\n", - "pdf_path = os.path.join(DATA_DIR, \"documents\", \"Seahawks-Patriots in Super Bowl LX_ What We Learned from Seattle's 29-13 win.pdf\")\n", - "upload_file(pdf_path, \"Seahawks-Patriots_SuperBowl_LX_Analysis.pdf\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.2 Verify Document Ingestion" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check consumer logs to verify document processing status.\n", - "\n", - "The logs should show the document being picked up and successfully ingested:\n", - "```\n", - "services.document_indexer - INFO - Task ...: PENDING (0s)\n", - "services.document_indexer - INFO - Task ...: PENDING (5s)\n", - "handlers.base - INFO - [DocumentHandler] ✓ Seahawks-Patriots_SuperBowl_LX_Analysis.pdf → SUCCESS\n", - "consumer - INFO - ✓ SUMMARY: Seahawks-Patriots_SuperBowl_LX_Analysis.pdf | Collection: aidp_bucket | Duration: 12.76s | Status: SUCCESS\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Verify file landed in object storage\n", - "verify_file_in_storage(\"Seahawks-Patriots_SuperBowl_LX_Analysis.pdf\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.3 Verify Document Ingestion\n", - "\n", - "Check consumer logs to verify document processing status.\n", - "\n", - "The logs should show the document being picked up and successfully ingested:\n", - "```\n", - "services.document_indexer - INFO - Task ...: PENDING (0s)\n", - "services.document_indexer - INFO - Task ...: PENDING (5s)\n", - "handlers.base - INFO - [DocumentHandler] ✓ Seahawks-Patriots_SuperBowl_LX_Analysis.pdf → SUCCESS\n", - "consumer - INFO - ✓ SUMMARY: Seahawks-Patriots_SuperBowl_LX_Analysis.pdf | Collection: aidp_bucket | Duration: 12.76s | Status: SUCCESS\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Check consumer logs for ingestion status\n", - "print(\"Waiting for document processing...\")\n", - "get_consumer_logs(50)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.4 Query Document via RAG\n", - "\n", - "You can query the ingested document either **programmatically** below or via the **RAG Frontend UI**.\n", - "\n", - "> **💡 RAG Frontend**: Open `http://:8090` in your browser for an interactive Q&A interface.\n", - "> Make sure to select the collection **`aidp_bucket`** in the UI.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Query the document\n", - "await query_rag(\"What was the final score and who won Super Bowl LX?\", MINIO_COLLECTION)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Ask another question about the document.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Query about key takeaways\n", - "await query_rag(\"What were the key lessons learned from Seattle's victory in Super Bowl LX?\", MINIO_COLLECTION)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Clean Up\n", - "\n", - "Stop all services and clean up ingested data.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Stop RAG Deployment\n", - "\n", - "Stop all RAG services (NIMs, Milvus, Ingestor, RAG server).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "os.chdir(RAG_REPO_DIR)\n", - "for f in [\n", - " \"deploy/compose/docker-compose-rag-server.yaml\",\n", - " \"deploy/compose/docker-compose-ingestor-server.yaml\",\n", - " \"deploy/compose/vectordb.yaml\",\n", - " \"deploy/compose/nims.yaml\",\n", - "]:\n", - " run_command(f\"docker compose -f {f} down\")\n", - "print(\"[OK] RAG stopped\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Stop Continuous ingestion Deployment\n", - "\n", - "Stop Continuous ingestion services (Kafka, MinIO, Consumer).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "run_command(f\"docker compose -f {AIDP_COMPOSE_FILE} down\")\n", - "print(\"[OK] Continuous ingestion stopped\")\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Document Continuous Ingestion from Object Storage\n", + "\n", + "## Purpose\n", + "\n", + "This notebook demonstrates an **automated document ingestion pipeline** that:\n", + "\n", + "1. Monitors emulated object storage for new uploads via Kafka events\n", + "2. Routes documents to appropriate AI services for indexing\n", + "5. Enables RAG Agent for semantic search and contextual Q&A over all ingested content\n", + "\n", + "## What Gets Deployed\n", + "\n", + "1. **NVIDIA RAG** - Document indexing, vector search, and AI-powered Q&A (NIMs, Milvus, Ingestor)\n", + "2. **Continuous Ingestion** - Event-driven ingestion pipeline (Kafka, MinIO, Consumer)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "\n", + "### Hardware\n", + "- **GPU**: 2x RTX PRO 6000 Blackwell or 2x H100\n", + "\n", + "#### Default GPU Assignment\n", + "\n", + "| GPU | Service |\n", + "|-----|---------|\n", + "| 0 | RAG NIMs (Embedding, Reranker) |\n", + "| 1 | RAG LLM NIM (Llama-3.3-Nemotron-Super-49B) |\n", + "\n", + "\n", + "### Software (pre-installed required)\n", + "- Ubuntu 22.04 or later\n", + "- Docker 24.0+ with Docker Compose v2\n", + "- NVIDIA Driver 570+\n", + "- NVIDIA Container Toolkit\n", + "\n", + "### API Keys\n", + "\n", + "\n", + "\n", + "\n", + "
KeyPurposeHow to Get
NGC_API_KEYDocker login, NIM deploymentsNGC Portal → Generate API Key
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Table of Contents\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SectionDescription
SetupClone repo, install deps, set API keys, load helpers
Deploy RAGNIMs, Vector DB, Ingestor, RAG Server
Deploy Continuous IngestionKafka, MinIO, Consumer
TestingUpload documents, query RAG
Clean UpStop services, clean data
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## References\n", + "\n", + "- **RAG Blueprint**: [NVIDIA RAG Documentation](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/docs/deploy-docker-self-hosted.md)\n", + "- **NIM**: [NVIDIA NIM Documentation](https://docs.nvidia.com/nim/index.html)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup\n", + "\n", + "Clone the repository, configure API keys, and load helper functions.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Clone Repository\n", + "\n", + "Clone the RAG Blueprint repo to `~/rag`. This includes the consumer source code, deploy configs, and sample test data.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess, sys, os, shutil\n", + "\n", + "RAG_REPO_DIR = os.path.expanduser(\"~/rag\")\n", + "RAG_REPO_URL = \"https://github.com/NVIDIA-AI-Blueprints/rag.git\"\n", + "\n", + "# Ensure git-lfs is installed before any LFS operations\n", + "if not shutil.which(\"git-lfs\"):\n", + " print(\"[INSTALLING] git-lfs...\")\n", + " subprocess.run(\"sudo apt-get update && sudo apt-get install -y git-lfs && git lfs install\", shell=True, check=True)\n", + "else:\n", + " print(\"[OK] git-lfs found\")\n", + "\n", + "# Clone from correct branch (skip if already exists)\n", + "if not os.path.exists(RAG_REPO_DIR):\n", + " subprocess.run(f\"git clone {RAG_REPO_URL} {RAG_REPO_DIR}\", shell=True, check=True)\n", + "else:\n", + " print(f\"[OK] RAG repo already exists: {RAG_REPO_DIR}\")\n", + "subprocess.run(\"git lfs pull\", shell=True, cwd=RAG_REPO_DIR, check=True)\n", + "\n", + "# Verify\n", + "for path in [\"deploy/compose\", \"examples/rag_event_ingest/kafka_consumer\", \"examples/rag_event_ingest/data\"]:\n", + " status = \"[OK]\" if os.path.exists(os.path.join(RAG_REPO_DIR, path)) else \"[MISSING]\"\n", + " print(f\" {status} {path}\")\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Install Dependencies\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! python3 -m ensurepip --upgrade" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Ensure pip is available (some minimal Python installs lack it)\n", + "subprocess.run([sys.executable, \"-m\", \"ensurepip\", \"--upgrade\"], capture_output=True)\n", + "\n", + "def check_install_system_pkg(cmd: str, install_cmd: str):\n", + " if shutil.which(cmd):\n", + " print(f\" [OK] {cmd} found\")\n", + " return True\n", + " print(f\" [INSTALLING] {cmd}...\")\n", + " result = subprocess.run(install_cmd, shell=True, capture_output=True, text=True)\n", + " if result.returncode == 0:\n", + " print(f\" [OK] {cmd} installed\")\n", + " return True\n", + " print(f\" [ERROR] Failed to install {cmd}. Please install manually: {install_cmd}\")\n", + " return False\n", + "\n", + "check_install_system_pkg(\"git\", \"sudo apt-get update && sudo apt-get install -y git\")\n", + "\n", + "# Install Python packages\n", + "subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"minio\", \"aiohttp\", \"requests\", \"python-dotenv\", \"pyyaml\"])\n", + "print(\"[OK] Ready\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Set API Keys\n", + "\n", + "Configure NGC API key for NIM deployments.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "\n", + "def set_api_key(env_var: str, prompt: str, required: bool = True):\n", + " if os.environ.get(env_var):\n", + " print(f\" [OK] {env_var} already set ({os.environ[env_var][:10]}...)\")\n", + " return True\n", + " key = getpass.getpass(prompt)\n", + " if key:\n", + " os.environ[env_var] = key\n", + " print(f\" [OK] {env_var} set\")\n", + " return True\n", + " if required:\n", + " print(f\" [ERROR] {env_var} is required\")\n", + " return False\n", + " print(f\" [SKIP] {env_var} (optional)\")\n", + " return True\n", + "\n", + "set_api_key(\"NGC_API_KEY\", \"Enter NGC_API_KEY (starts with 'nvapi-'): \", required=True)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Helper Functions\n", + "\n", + "Shared utilities for deployment, file upload, status checks, and RAG queries.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Install dependencies\n", + "import sys\n", + "!{sys.executable} -m pip install -q minio aiohttp requests python-dotenv" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os, sys, json, re, subprocess, time, socket, asyncio\n", + "import aiohttp, requests\n", + "from typing import List, Optional, Dict\n", + "\n", + "try:\n", + " from minio import Minio\n", + " from minio.error import S3Error\n", + "except ImportError:\n", + " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"minio\"])\n", + " from minio import Minio\n", + " from minio.error import S3Error\n", + "\n", + "# =============================================================================\n", + "# CONFIGURATION\n", + "# =============================================================================\n", + "\n", + "# Paths relative to RAG repo root\n", + "RAG_REPO_DIR = os.path.expanduser(\"~/rag\")\n", + "EXAMPLE_DIR = os.path.join(RAG_REPO_DIR, \"examples/rag_event_ingest\")\n", + "AIDP_COMPOSE_FILE = os.path.join(EXAMPLE_DIR, \"deploy/docker-compose.yaml\")\n", + "DATA_DIR = os.path.join(EXAMPLE_DIR, \"data\")\n", + "RAG_SERVER_URL = \"http://localhost:8081\"\n", + "INGESTOR_URL = \"http://localhost:8082\"\n", + "\n", + "LOCAL_NIM_CACHE = os.path.expanduser(\"~/.cache/nim\")\n", + "\n", + "MINIO_ENDPOINT = \"localhost:9201\"\n", + "MINIO_ACCESS_KEY = \"minioadmin\"\n", + "MINIO_SECRET_KEY = \"minioadmin\"\n", + "MINIO_BUCKET = \"aidp-bucket\"\n", + "MINIO_COLLECTION = \"aidp_bucket\"\n", + "MINIO_CONSOLE_PORT = 9211\n", + "\n", + "# =============================================================================\n", + "# SHARED UTILITIES\n", + "# =============================================================================\n", + "\n", + "def run_command(cmd: str, capture: bool = False) -> Optional[str]:\n", + " \"\"\"Execute a shell command and print it.\"\"\"\n", + " print(f\"$ {cmd}\")\n", + " result = subprocess.run(cmd, shell=True, capture_output=capture, text=True)\n", + " return result.stdout if capture else None\n", + "\n", + "def get_host_ip() -> str:\n", + " \"\"\"Get host IP address for external access URLs.\"\"\"\n", + " try:\n", + " s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n", + " s.connect((\"8.8.8.8\", 80))\n", + " ip = s.getsockname()[0]\n", + " s.close()\n", + " return ip\n", + " except OSError:\n", + " return \"localhost\"\n", + "\n", + "def get_minio_client() -> Minio:\n", + " \"\"\"Create MinIO client for AIDP bucket operations.\"\"\"\n", + " return Minio(MINIO_ENDPOINT, access_key=MINIO_ACCESS_KEY, secret_key=MINIO_SECRET_KEY, secure=False)\n", + "\n", + "def upload_file(local_path: str, object_name: Optional[str] = None) -> bool:\n", + " \"\"\"Upload a local file to MinIO AIDP bucket.\"\"\"\n", + " if not os.path.exists(local_path):\n", + " print(f\"[ERROR] File not found: {local_path}\")\n", + " return False\n", + " obj = object_name or os.path.basename(local_path)\n", + " try:\n", + " client = get_minio_client()\n", + " if not client.bucket_exists(MINIO_BUCKET):\n", + " client.make_bucket(MINIO_BUCKET)\n", + " client.fput_object(MINIO_BUCKET, obj, local_path)\n", + " print(f\"[OK] Uploaded: {obj}\")\n", + " return True\n", + " except S3Error as e:\n", + " print(f\"[ERROR] {e}\")\n", + " return False\n", + "\n", + "def verify_file_in_storage(object_name: str, bucket: str = MINIO_BUCKET) -> bool:\n", + " \"\"\"Check if a file exists in MinIO bucket and print verification status.\"\"\"\n", + " try:\n", + " client = get_minio_client()\n", + " stat = client.stat_object(bucket, object_name)\n", + " print(f\"[OK] File verified in storage:\")\n", + " print(f\" Bucket: {bucket}\")\n", + " print(f\" Object: {object_name}\")\n", + " print(f\" Size: {stat.size:,} bytes\")\n", + " print(f\" Modified: {stat.last_modified}\")\n", + " return True\n", + " except S3Error as e:\n", + " print(f\"[ERROR] File not found in storage: {object_name}\")\n", + " print(f\" Error: {e}\")\n", + " return False\n", + "\n", + "def get_consumer_logs(lines: int = 30) -> None:\n", + " \"\"\"Show recent Kafka consumer logs.\"\"\"\n", + " run_command(f\"docker logs kafka-consumer --tail {lines}\")\n", + "\n", + "async def query_rag(question: str, collection: str = None) -> Optional[str]:\n", + " \"\"\"Query RAG system and print the answer.\"\"\"\n", + " coll = collection or MINIO_COLLECTION\n", + " print(f\"Q: {question}\\nCollection: {coll}\\n\" + \"-\" * 40)\n", + "\n", + " payload = {\n", + " \"messages\": [{\"role\": \"user\", \"content\": question}],\n", + " \"use_knowledge_base\": True,\n", + " \"collection_names\": [coll],\n", + " }\n", + " try:\n", + " async with aiohttp.ClientSession() as session:\n", + " async with session.post(\n", + " f\"{RAG_SERVER_URL}/generate\", json=payload,\n", + " timeout=aiohttp.ClientTimeout(total=120),\n", + " ) as resp:\n", + " text = await resp.text()\n", + " # Parse SSE response: extract content from each \"data: {...}\" line\n", + " chunks = []\n", + " for line in text.split(\"\\n\"):\n", + " if not line.startswith(\"data: \") or line[6:] == \"[DONE]\":\n", + " continue\n", + " try:\n", + " msg = json.loads(line[6:]).get(\"choices\", [{}])[0].get(\"message\", {})\n", + " if msg.get(\"content\"):\n", + " chunks.append(msg[\"content\"])\n", + " except json.JSONDecodeError:\n", + " pass\n", + " answer = \"\".join(chunks)\n", + " print(f\"Answer: {answer}\")\n", + " return answer\n", + " except aiohttp.ClientError as e:\n", + " print(f\"[ERROR] {e}\")\n", + " return None\n", + "\n", + "print(f\"[OK] Helpers loaded | Host IP: {get_host_ip()}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deploy NVIDIA RAG\n", + "\n", + "Deploy the NVIDIA RAG: NIMs (LLM, Embedding, Reranker), Milvus vector database, Ingestor server, and RAG server.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ngc_key = os.environ.get(\"NGC_API_KEY\")\n", + "if not ngc_key:\n", + " raise RuntimeError(\"NGC_API_KEY not set! Run the API keys cell first.\")\n", + "\n", + "os.chdir(RAG_REPO_DIR)\n", + "\n", + "# Set env vars needed by docker compose\n", + "os.environ[\"NGC_API_KEY\"] = ngc_key\n", + "os.environ[\"USERID\"] = f\"{os.getuid()}:{os.getgid()}\"\n", + "os.environ[\"COLLECTION_NAME\"] = MINIO_COLLECTION\n", + "\n", + "# Load RAG .env defaults (MODEL_DIRECTORY, etc.)\n", + "from dotenv import load_dotenv\n", + "env_file = os.path.join(RAG_REPO_DIR, \"deploy/compose/.env\")\n", + "if os.path.exists(env_file):\n", + " load_dotenv(env_file, override=False)\n", + "\n", + "# Login to nvcr.io\n", + "subprocess.run(f\"echo {ngc_key} | docker login nvcr.io -u '$oauthtoken' --password-stdin\",\n", + " shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + "\n", + "# Deploy components\n", + "for label, compose_file in [\n", + " (\"NIMs\", \"deploy/compose/nims.yaml\"),\n", + " (\"Vector DB\", \"deploy/compose/vectordb.yaml\"),\n", + "]:\n", + " print(f\"Deploying {label}...\")\n", + " run_command(f\"USERID=$(id -u) docker compose -f {compose_file} up -d\")\n", + "\n", + "print(\"Waiting 30s for Milvus...\")\n", + "time.sleep(30)\n", + "\n", + "for label, compose_file in [\n", + " (\"Ingestor\", \"deploy/compose/docker-compose-ingestor-server.yaml\"),\n", + " (\"RAG Server\", \"deploy/compose/docker-compose-rag-server.yaml\"),\n", + "]:\n", + " print(f\"Deploying {label}...\")\n", + " run_command(f\"docker compose -f {compose_file} up -d\")\n", + "\n", + "ip = get_host_ip()\n", + "print(f\"\\nRAG deployed: http://{ip}:8081 (server) | http://{ip}:8082 (ingestor) | http://{ip}:8090 (UI)\")\n", + "print(f\"COLLECTION_NAME: {MINIO_COLLECTION}\")\n", + "print(\"Wait ~10 minutes for NIMs to load models, then run the status check cell.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify RAG services are healthy. Wait ~10 minutes for NIMs to load models.\n", + "\n", + "The deployment status should be:\n", + "```\n", + "NAMES STATUS\n", + "rag-frontend Up About a minute\n", + "rag-server Up About a minute\n", + "ingestor-server Up About a minute\n", + "milvus-standalone Up 2 minutes (healthy)\n", + "milvus-etcd Up 2 minutes (healthy)\n", + "milvus-minio Up 2 minutes (healthy)\n", + "nim-llm-ms Up 2 minutes (healthy)\n", + "nemotron-embedding-ms Up 2 minutes (healthy)\n", + "nemotron-ranking-ms Up 2 minutes (healthy)\n", + "```\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check service status and print access URLs\n", + "print(\"Wait ~10 minutes for services to become healthy.\")\n", + "print(\"Run this cell again after waiting.\\n\")\n", + "\n", + "ip = get_host_ip()\n", + "for name, port, path in [\n", + " (\"RAG Server\", 8081, \"/health\"), (\"Ingestor\", 8082, \"/health\"),\n", + " (\"Frontend\", 8090, \"/\"), (\"Milvus\", 19530, \"/v1/vector/collections\"),\n", + "]:\n", + " try:\n", + " s = \"[OK]\" if requests.get(f\"http://localhost:{port}{path}\", timeout=10).status_code == 200 else \"[WARN]\"\n", + " except requests.ConnectionError:\n", + " s = \"[DOWN]\"\n", + " except requests.Timeout:\n", + " s = \"[TIMEOUT]\"\n", + " print(f\" {s} {name}: http://{ip}:{port}\")\n", + "run_command(\"docker ps --format 'table {{.Names}}\\t{{.Status}}' | grep -E '(rag|milvus|ingestor|nim|nemotron|NAMES)'\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Deploy Continuous Ingestion from emulated object storage\n", + "\n", + "Deploy the Continuous Ingestion: Kafka message broker, MinIO object storage, and Kafka consumer for automated ingestion.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Deploy Services\n", + "\n", + "Deploy Kafka, MinIO, and the Kafka consumer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Verify prerequisites\n", + "net_check = subprocess.run(\"docker network inspect nvidia-rag\", shell=True, capture_output=True)\n", + "if net_check.returncode != 0:\n", + " raise RuntimeError(\"nvidia-rag network not found. Deploy RAG first.\")\n", + "\n", + "ngc_key = os.environ.get(\"NGC_API_KEY\", \"\")\n", + "if not ngc_key:\n", + " raise RuntimeError(\"NGC_API_KEY not set!\")\n", + "\n", + "host_ip = get_host_ip()\n", + "\n", + "# Set environment variables for docker compose\n", + "os.environ[\"HOST_IP\"] = host_ip\n", + "\n", + "# Login + pull + build\n", + "subprocess.run(f\"echo {ngc_key} | docker login nvcr.io -u '$oauthtoken' --password-stdin\",\n", + " shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + "\n", + "compose = f\"docker compose -f {AIDP_COMPOSE_FILE}\"\n", + "subprocess.run(f\"{compose} pull --ignore-pull-failures\", shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + "subprocess.run(f\"{compose} up -d --build\", shell=True, capture_output=True, text=True, executable=\"/bin/bash\")\n", + "\n", + "print(f\"Continuous Ingestion deployed:\")\n", + "print(f\" Kafka UI: http://{host_ip}:8080\")\n", + "print(f\" MinIO Console: http://{host_ip}:{MINIO_CONSOLE_PORT}\")\n", + "print(f\" Credentials: minioadmin / minioadmin\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify continuous ingestion services are running.\n", + "\n", + "The deployment status should be:\n", + "```\n", + "NAMES STATUS\n", + "kafka-consumer Up About a minute\n", + "aidp-kafka-ui Up About a minute\n", + "aidp-minio-mc Up About a minute\n", + "aidp-minio Up About a minute (healthy)\n", + "kafka Up About a minute (healthy)\n", + "```\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check service status and print access URLs\n", + "ip = get_host_ip()\n", + "for name, port, path in [\n", + " (\"Kafka UI\", 8080, \"/\"),\n", + " (\"MinIO Console\", MINIO_CONSOLE_PORT, \"/\"),\n", + "]:\n", + " try:\n", + " s = \"[OK]\" if requests.get(f\"http://localhost:{port}{path}\", timeout=10).status_code == 200 else \"[WARN]\"\n", + " except requests.ConnectionError:\n", + " s = \"[DOWN]\"\n", + " except requests.Timeout:\n", + " s = \"[TIMEOUT]\"\n", + " print(f\" {s} {name}: http://{ip}:{port}\")\n", + "\n", + "# Check kafka-consumer container status\n", + "result = subprocess.run(\"docker inspect -f '{{.State.Status}}' kafka-consumer 2>/dev/null\",\n", + " shell=True, capture_output=True, text=True)\n", + "status = result.stdout.strip()\n", + "s = \"[OK]\" if status == \"running\" else \"[DOWN]\"\n", + "print(f\" {s} Kafka Consumer: {status or 'not found'}\")\n", + "\n", + "run_command(\"docker ps --format 'table {{.Names}}\\t{{.Status}}' | grep -E '(kafka|minio|NAMES)'\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Testing\n", + "\n", + "Test the deployment by uploading documents, then querying via RAG.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Document Upload\n", + "\n", + "Upload a PDF document to MinIO, which triggers automatic ingestion via Kafka consumer.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.1 Upload to Storage\n", + "\n", + "Upload the document to MinIO object storage.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Sample documents are included in the repo under examples/rag_event_ingest/data/\n", + "pdf_path = os.path.join(DATA_DIR, \"documents\", \"Seahawks-Patriots in Super Bowl LX_ What We Learned from Seattle's 29-13 win.pdf\")\n", + "upload_file(pdf_path, \"Seahawks-Patriots_SuperBowl_LX_Analysis.pdf\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.2 Verify Document Ingestion" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check consumer logs to verify document processing status.\n", + "\n", + "The logs should show the document being picked up and successfully ingested:\n", + "```\n", + "services.document_indexer - INFO - Task ...: PENDING (0s)\n", + "services.document_indexer - INFO - Task ...: PENDING (5s)\n", + "handlers.base - INFO - [DocumentHandler] ✓ Seahawks-Patriots_SuperBowl_LX_Analysis.pdf → SUCCESS\n", + "consumer - INFO - ✓ SUMMARY: Seahawks-Patriots_SuperBowl_LX_Analysis.pdf | Collection: aidp_bucket | Duration: 12.76s | Status: SUCCESS\n", + "```\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Verify file landed in object storage\n", + "verify_file_in_storage(\"Seahawks-Patriots_SuperBowl_LX_Analysis.pdf\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.3 Verify Document Ingestion\n", + "\n", + "Check consumer logs to verify document processing status.\n", + "\n", + "The logs should show the document being picked up and successfully ingested:\n", + "```\n", + "services.document_indexer - INFO - Task ...: PENDING (0s)\n", + "services.document_indexer - INFO - Task ...: PENDING (5s)\n", + "handlers.base - INFO - [DocumentHandler] ✓ Seahawks-Patriots_SuperBowl_LX_Analysis.pdf → SUCCESS\n", + "consumer - INFO - ✓ SUMMARY: Seahawks-Patriots_SuperBowl_LX_Analysis.pdf | Collection: aidp_bucket | Duration: 12.76s | Status: SUCCESS\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Check consumer logs for ingestion status\n", + "print(\"Waiting for document processing...\")\n", + "get_consumer_logs(50)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.4 Query Document via RAG\n", + "\n", + "You can query the ingested document either **programmatically** below or via the **RAG Frontend UI**.\n", + "\n", + "> **💡 RAG Frontend**: Open `http://:8090` in your browser for an interactive Q&A interface.\n", + "> Make sure to select the collection **`aidp_bucket`** in the UI.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Query the document\n", + "await query_rag(\"What was the final score and who won Super Bowl LX?\", MINIO_COLLECTION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ask another question about the document.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Query about key takeaways\n", + "await query_rag(\"What were the key lessons learned from Seattle's victory in Super Bowl LX?\", MINIO_COLLECTION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Clean Up\n", + "\n", + "Stop all services and clean up ingested data.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Stop RAG Deployment\n", + "\n", + "Stop all RAG services (NIMs, Milvus, Ingestor, RAG server).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "os.chdir(RAG_REPO_DIR)\n", + "for f in [\n", + " \"deploy/compose/docker-compose-rag-server.yaml\",\n", + " \"deploy/compose/docker-compose-ingestor-server.yaml\",\n", + " \"deploy/compose/vectordb.yaml\",\n", + " \"deploy/compose/nims.yaml\",\n", + "]:\n", + " run_command(f\"docker compose -f {f} down\")\n", + "print(\"[OK] RAG stopped\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Stop Continuous ingestion Deployment\n", + "\n", + "Stop Continuous ingestion services (Kafka, MinIO, Consumer).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "run_command(f\"docker compose -f {AIDP_COMPOSE_FILE} down\")\n", + "print(\"[OK] Continuous ingestion stopped\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 } From 80b00829a0144d77c1fe9fb6c6c8421afe9a0b12 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Wed, 11 Mar 2026 15:26:56 -0500 Subject: [PATCH 55/81] removed minio deps --- pyproject.toml | 12 +- src/nvidia_rag/ingestor_server/Dockerfile | 2 +- src/nvidia_rag/ingestor_server/health.py | 21 +- src/nvidia_rag/ingestor_server/main.py | 51 ++- src/nvidia_rag/rag_server/Dockerfile | 2 +- src/nvidia_rag/rag_server/health.py | 21 +- .../rag_server/response_generator.py | 34 +- src/nvidia_rag/utils/configuration.py | 5 + src/nvidia_rag/utils/minio_operator.py | 34 +- .../utils/vdb/oracle/oracle_queries.py | 19 +- src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py | 75 +++- uv.lock | 372 +++++++++++------- 12 files changed, 418 insertions(+), 230 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2fddea4a3..3a62506a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,6 @@ dependencies = [ "langchain-community>=0.4", "langchain-milvus>=0.3.0", "langchain-nvidia-ai-endpoints>=1.0.3", - "minio>=7.2,<8.0", "pdfplumber>=0.11.9", "pydantic>=2.11,<3.0", "pymilvus[milvus_lite]>=2.6.7,<3.0", @@ -59,8 +58,8 @@ rag = [ ] ingest = [ # nv-ingest dependencies (required for ingestion operations) - "nv-ingest-api==25.9.0", - "nv-ingest-client==25.9.0", + "nv-ingest-api==26.1.2", + "nv-ingest-client==26.1.2", "tritonclient==2.57.0", # Other ingest dependencies "langchain-openai>=0.2", @@ -81,8 +80,8 @@ ingest = [ ] all = [ # nv-ingest dependencies (required for ingestion operations) - "nv-ingest-api==25.9.0", - "nv-ingest-client==25.9.0", + "nv-ingest-api==26.1.2", + "nv-ingest-client==26.1.2", "tritonclient==2.57.0", # RAG + Ingest dependencies "langchain-openai>=0.2", @@ -111,6 +110,9 @@ oracle = [ "oracledb>=2.0.0", "langchain-community>=0.4", ] +minio = [ + "minio>=7.2,<8.0", +] [tool.uv.sources] nvidia-rag = { workspace = true } diff --git a/src/nvidia_rag/ingestor_server/Dockerfile b/src/nvidia_rag/ingestor_server/Dockerfile index eccfeffd6..ed481b832 100644 --- a/src/nvidia_rag/ingestor_server/Dockerfile +++ b/src/nvidia_rag/ingestor_server/Dockerfile @@ -71,7 +71,7 @@ WORKDIR /workspace COPY --from=builder /build/pyproject.toml /workspace/ COPY --from=builder /build/uv.lock /workspace/ -RUN uv sync --locked --no-install-project --no-dev --extra ingest --extra oracle +RUN uv sync --locked --no-install-project --no-dev --extra ingest --extra oracle --extra minio COPY --from=builder /build/dist/*.whl /workspace/ diff --git a/src/nvidia_rag/ingestor_server/health.py b/src/nvidia_rag/ingestor_server/health.py index 55c3e0963..c08d418d3 100644 --- a/src/nvidia_rag/ingestor_server/health.py +++ b/src/nvidia_rag/ingestor_server/health.py @@ -378,16 +378,17 @@ async def check_all_services_health( task_management: list[TaskManagementHealthInfo] = [] # MinIO health check - minio_endpoint = config.minio.endpoint - minio_access_key = config.minio.access_key.get_secret_value() - minio_secret_key = config.minio.secret_key.get_secret_value() - if minio_endpoint: - minio_result = await check_minio_health( - endpoint=minio_endpoint, - access_key=minio_access_key, - secret_key=minio_secret_key, - ) - object_storage.append(minio_result) + if config.minio.enabled: + minio_endpoint = config.minio.endpoint + minio_access_key = config.minio.access_key.get_secret_value() + minio_secret_key = config.minio.secret_key.get_secret_value() + if minio_endpoint: + minio_result = await check_minio_health( + endpoint=minio_endpoint, + access_key=minio_access_key, + secret_key=minio_secret_key, + ) + object_storage.append(minio_result) # Vector DB health check try: diff --git a/src/nvidia_rag/ingestor_server/main.py b/src/nvidia_rag/ingestor_server/main.py index f28750e38..d7269fd8a 100644 --- a/src/nvidia_rag/ingestor_server/main.py +++ b/src/nvidia_rag/ingestor_server/main.py @@ -158,24 +158,28 @@ def __init__( ) # Initialize MinIO operator - handle failures gracefully - try: - if self.mode == Mode.LITE: - raise ValueError("MinIO operations are not supported in RAG Lite mode") - self.minio_operator = get_minio_operator(config=self.config) - # Ensure default bucket exists (idempotent operation) - try: - self.minio_operator._make_bucket(bucket_name="a-bucket") - logger.debug("Ensured 'a-bucket' exists in MinIO") - except Exception as bucket_err: - # Log specific exception for debugging bucket creation issues - logger.debug("Could not ensure bucket exists: %s", bucket_err) - except Exception as e: + if not self.config.minio.enabled: self.minio_operator = None - # Error already logged in MinioOperator.__init__, just note it here - logger.debug( - "MinIO operator set to None due to initialization failure, reason: %s", - e, - ) + else: + try: + if self.mode == Mode.LITE: + raise ValueError("MinIO operations are not supported in RAG Lite mode") + self.minio_operator = get_minio_operator(config=self.config) + # Ensure default bucket exists (idempotent operation) + if self.minio_operator is not None: + try: + self.minio_operator._make_bucket(bucket_name="a-bucket") + logger.debug("Ensured 'a-bucket' exists in MinIO") + except Exception as bucket_err: + # Log specific exception for debugging bucket creation issues + logger.debug("Could not ensure bucket exists: %s", bucket_err) + except Exception as e: + self.minio_operator = None + # Error already logged in MinioOperator.__init__, just note it here + logger.debug( + "MinIO operator set to None due to initialization failure, reason: %s", + e, + ) if self.vdb_op is not None: if not (isinstance(self.vdb_op, VDBRag) or isinstance(self.vdb_op, VDB)): @@ -1767,6 +1771,8 @@ def delete_minio_metadata(docs_to_delete: list[str]) -> None: logger.info( f"Recalculated collection info for {collection_name} after document deletion" ) + # Delete MinIO metadata for successfully deleted documents + delete_minio_metadata(deleted_docs) elif hasattr(vdb_op, "_es_connection"): # Elasticsearch: Delete first, then add without aggregation # Lazy import to avoid requiring elasticsearch when not used @@ -1798,6 +1804,14 @@ def delete_minio_metadata(docs_to_delete: list[str]) -> None: logger.info( f"Recalculated collection info for {collection_name} after document deletion" ) + # Delete MinIO metadata for successfully deleted documents + delete_minio_metadata(deleted_docs) + elif hasattr(vdb_op, "_oracle_cs"): + # Oracle 26ai: Replace collection info directly without aggregation + vdb_op.set_collection_info(collection_name, updated_collection_info) + logger.info( + f"Recalculated collection info for {collection_name} after document deletion" + ) else: # Fallback: Use add_document_info (may cause double-aggregation, but better than nothing) logger.warning( @@ -1819,9 +1833,6 @@ def delete_minio_metadata(docs_to_delete: list[str]) -> None: "documents": [], } - # Delete MinIO metadata for successfully deleted documents - delete_minio_metadata(deleted_docs) - # Build documents response with metadata and document_info from fetched data documents = [] for doc_name in deleted_docs: diff --git a/src/nvidia_rag/rag_server/Dockerfile b/src/nvidia_rag/rag_server/Dockerfile index 286d120b4..d203cfd7c 100644 --- a/src/nvidia_rag/rag_server/Dockerfile +++ b/src/nvidia_rag/rag_server/Dockerfile @@ -69,7 +69,7 @@ WORKDIR /workspace COPY --from=builder /build/pyproject.toml /workspace/ COPY --from=builder /build/uv.lock /workspace/ -RUN uv sync --locked --no-install-project --no-dev --extra rag --extra oracle +RUN uv sync --locked --no-install-project --no-dev --extra rag --extra oracle --extra minio COPY --from=builder /build/dist/*.whl /workspace/ diff --git a/src/nvidia_rag/rag_server/health.py b/src/nvidia_rag/rag_server/health.py index b0e850e25..78a526ef4 100644 --- a/src/nvidia_rag/rag_server/health.py +++ b/src/nvidia_rag/rag_server/health.py @@ -225,16 +225,17 @@ async def check_all_services_health( nim: list[NIMServiceHealthInfo] = [] # MinIO health check - minio_endpoint = config.minio.endpoint - minio_access_key = config.minio.access_key.get_secret_value() - minio_secret_key = config.minio.secret_key.get_secret_value() - if minio_endpoint: - minio_result = await check_minio_health( - endpoint=minio_endpoint, - access_key=minio_access_key, - secret_key=minio_secret_key, - ) - object_storage.append(minio_result) + if config.minio.enabled: + minio_endpoint = config.minio.endpoint + minio_access_key = config.minio.access_key.get_secret_value() + minio_secret_key = config.minio.secret_key.get_secret_value() + if minio_endpoint: + minio_result = await check_minio_health( + endpoint=minio_endpoint, + access_key=minio_access_key, + secret_key=minio_secret_key, + ) + object_storage.append(minio_result) # Vector DB health check try: diff --git a/src/nvidia_rag/rag_server/response_generator.py b/src/nvidia_rag/rag_server/response_generator.py index a274a5898..f3f3df7f5 100644 --- a/src/nvidia_rag/rag_server/response_generator.py +++ b/src/nvidia_rag/rag_server/response_generator.py @@ -92,16 +92,28 @@ def __init__(self, generator, status_code: int = 200): ) MINIO_OPERATOR = None +_MINIO_OPERATOR_INITIALIZED = False def get_minio_operator_instance(): - """Lazy initialize the MinioOperator instance""" - global MINIO_OPERATOR - if MINIO_OPERATOR is None: + """Lazy initialize the MinioOperator instance. Returns None when MinIO is disabled.""" + global MINIO_OPERATOR, _MINIO_OPERATOR_INITIALIZED + if not _MINIO_OPERATOR_INITIALIZED: MINIO_OPERATOR = get_minio_operator() + _MINIO_OPERATOR_INITIALIZED = True return MINIO_OPERATOR +def _get_minio_payload(object_name: str | None) -> dict: + """Fetch a payload from MinIO, returning empty dict when MinIO is unavailable or disabled.""" + if not object_name: + return {} + operator = get_minio_operator_instance() + if operator is None: + return {} + return operator.get_payload(object_name=object_name) + + class Usage(BaseModel): """Token usage information.""" @@ -826,9 +838,7 @@ def prepare_citations( location=location, metadata=doc.metadata, ) - payload = get_minio_operator_instance().get_payload( - object_name=unique_thumbnail_id - ) + payload = _get_minio_payload(unique_thumbnail_id) content = payload.get("content", "") source_metadata = SourceMetadata( page_number=page_number, @@ -1018,9 +1028,7 @@ async def retrieve_summary( location=[], ) - payload = get_minio_operator_instance().get_payload( - object_name=unique_thumbnail_id - ) + payload = _get_minio_payload(unique_thumbnail_id) if payload: return { @@ -1091,9 +1099,7 @@ async def _wait_for_summary_completion( page_number=0, location=[], ) - payload = get_minio_operator_instance().get_payload( - object_name=unique_thumbnail_id - ) + payload = _get_minio_payload(unique_thumbnail_id) if payload: return { "message": "Summary retrieved successfully.", @@ -1117,9 +1123,7 @@ async def _wait_for_summary_completion( page_number=0, location=[], ) - payload = get_minio_operator_instance().get_payload( - object_name=unique_thumbnail_id - ) + payload = _get_minio_payload(unique_thumbnail_id) if payload: return { "message": "Summary retrieved successfully.", diff --git a/src/nvidia_rag/utils/configuration.py b/src/nvidia_rag/utils/configuration.py index 9019b7a84..53e2a12f6 100644 --- a/src/nvidia_rag/utils/configuration.py +++ b/src/nvidia_rag/utils/configuration.py @@ -856,6 +856,11 @@ def normalize_url(cls, v: Any) -> Any: class MinioConfig(_ConfigBase): """Minio configuration.""" + enabled: bool = Field( + default=True, + env="ENABLE_MINIO", + description="Enable MinIO object storage for multimodal citations. Set to False for deployments without Milvus (e.g. Oracle 26ai).", + ) endpoint: str = Field( default="localhost:9010", env="MINIO_ENDPOINT", diff --git a/src/nvidia_rag/utils/minio_operator.py b/src/nvidia_rag/utils/minio_operator.py index c1c3a4edd..7aa90188c 100644 --- a/src/nvidia_rag/utils/minio_operator.py +++ b/src/nvidia_rag/utils/minio_operator.py @@ -27,8 +27,15 @@ import logging from io import BytesIO -from minio import Minio -from minio.commonconfig import SnowballObject +try: + from minio import Minio + from minio.commonconfig import SnowballObject + + _MINIO_AVAILABLE = True +except ImportError: + Minio = None # type: ignore[assignment,misc] + SnowballObject = None # type: ignore[assignment,misc] + _MINIO_AVAILABLE = False from nvidia_rag.utils.configuration import NvidiaRAGConfig @@ -47,6 +54,11 @@ def __init__( secret_key: str, default_bucket_name: str = DEFAULT_BUCKET_NAME, ): + if not _MINIO_AVAILABLE: + raise ImportError( + "minio package is not installed. Install with: pip install 'nvidia_rag[minio]'" + ) + assert Minio is not None # noqa: S101 — narrowing for type checker self.client = Minio( endpoint, access_key=access_key, secret_key=secret_key, secure=False ) @@ -88,6 +100,7 @@ def put_payloads_bulk(self, payloads: list[dict], object_names: list[str]): """Put list of dictionaries to S3 storage using minio client""" json_datas = [json.dumps(payload).encode("utf-8") for payload in payloads] + assert SnowballObject is not None # noqa: S101 — narrowing for type checker snowball_objects = [] for object_name, json_data in zip(object_names, json_datas, strict=False): snowball_objects.append( @@ -134,20 +147,31 @@ def delete_payloads(self, object_names: list[str]) -> None: def get_minio_operator( default_bucket_name: str = DEFAULT_BUCKET_NAME, config: NvidiaRAGConfig | None = None, -) -> MinioOperator: +) -> MinioOperator | None: """ - Prepares and return MinioOperator object + Prepares and return MinioOperator object, or None when MinIO is disabled. Args: default_bucket_name: Default bucket name config: NvidiaRAGConfig instance. If None, creates a new one. Returns: - - minio_operator: MinioOperator + - minio_operator: MinioOperator, or None if ENABLE_MINIO=false or package not installed """ if config is None: config = NvidiaRAGConfig() + if not config.minio.enabled: + logger.info("MinIO is disabled (ENABLE_MINIO=false). Multimodal citations will be unavailable.") + return None + + if not _MINIO_AVAILABLE: + logger.warning( + "minio package is not installed. Multimodal citations will be unavailable. " + "Install with: pip install 'nvidia_rag[minio]', or set ENABLE_MINIO=false to suppress this warning." + ) + return None + minio_operator = MinioOperator( endpoint=config.minio.endpoint, access_key=config.minio.access_key.get_secret_value(), diff --git a/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py b/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py index a246f4e1e..f5b91bc0b 100644 --- a/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py +++ b/src/nvidia_rag/utils/vdb/oracle/oracle_queries.py @@ -168,16 +168,15 @@ def get_unique_sources_query(table_name: str) -> str: """ return f""" WITH unique_sources AS ( - SELECT source, + SELECT source, content_metadata, ROW_NUMBER() OVER (PARTITION BY source ORDER BY created_at DESC) as rn FROM {table_name} WHERE source IS NOT NULL ) - SELECT us.source, t.content_metadata - FROM unique_sources us - JOIN {table_name} t ON us.source = t.source - WHERE us.rn = 1 - ORDER BY us.source + SELECT source, content_metadata + FROM unique_sources + WHERE rn = 1 + ORDER BY source """ @@ -185,6 +184,11 @@ def get_delete_docs_query(table_name: str) -> str: """ Generate parameterized delete query for documents by source. + The source column stores a JSON object with a source_name field + (e.g. {"source_name": "/tmp/.../file.pdf"}). We match against the + extracted source_name value; the fallback OR clause handles any rows + where source was stored as a plain string. + Args: table_name: Name of the collection table @@ -193,7 +197,8 @@ def get_delete_docs_query(table_name: str) -> str: """ return f""" DELETE FROM {table_name} - WHERE source = :source_value + WHERE JSON_VALUE(source, '$.source_name') = :source_value + OR source = :source_value """ diff --git a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py index 47e03f680..34d999093 100644 --- a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py +++ b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py @@ -97,6 +97,13 @@ def __init__( # Cache embedding dimension to avoid re-embedding per query self._embedding_dim = self.get_embedding_dimension() + @staticmethod + def _get_clob_value(lob_or_str) -> str: + """Read a CLOB/LOB column value to a plain string.""" + if hasattr(lob_or_str, "read"): + return lob_or_str.read() + return lob_or_str or "" + def similarity_search_by_vector_with_relevance_scores( self, embedding, @@ -599,7 +606,15 @@ def get_documents(self, collection_name: str) -> list[dict[str, Any]]: cursor.execute(get_unique_sources_query(table_name)) for row in cursor: source_name = json.loads(row[0]).get('source_name') - content_metadata = row[1] if row[1] else {} + raw_cm = row[1] + if raw_cm is None: + content_metadata = {} + elif hasattr(raw_cm, 'read'): + content_metadata = json.loads(raw_cm.read()) or {} + elif isinstance(raw_cm, str): + content_metadata = json.loads(raw_cm) or {} + else: + content_metadata = raw_cm or {} metadata_dict = {} for item in metadata_schema: @@ -789,6 +804,59 @@ def add_document_info( logger.info(f"Added document info for {table_name}/{document_name}") + def set_collection_info( + self, + collection_name: str, + info_value: dict[str, Any], + ) -> None: + """Directly replace the collection-level document_info entry without aggregation. + + Unlike add_document_info, this bypasses _get_aggregated_document_info so the + caller's pre-computed value is stored as-is. Use this after document deletion + when the caller has already recalculated the correct aggregated state. + """ + table_name = collection_name.upper() + + with self._get_connection() as conn: + with conn.cursor() as cursor: + cursor.execute( + get_delete_document_info_query(), + { + "collection_name": table_name, + "document_name": "NA", + "info_type": "collection", + }, + ) + cursor.execute( + """ + INSERT INTO document_info (collection_name, info_type, document_name, info_value) + VALUES (:collection_name, :info_type, :document_name, :info_value) + """, + { + "collection_name": table_name, + "info_type": "collection", + "document_name": "NA", + "info_value": json.dumps(info_value), + }, + ) + conn.commit() + + logger.info(f"Set collection info for {table_name}") + + @staticmethod + def _read_clob_json(value) -> dict[str, Any]: + """Normalise a CLOB column value to a dict. + + oracledb returns CLOB columns as LOB objects; read() converts them to + str, then json.loads converts to dict. If the value is already a dict + (e.g. when Oracle returns native JSON), it is returned as-is. + """ + if hasattr(value, "read"): + value = value.read() + if isinstance(value, str): + return json.loads(value) + return value or {} + def _get_aggregated_document_info( self, collection_name: str, @@ -804,7 +872,8 @@ def _get_aggregated_document_info( ) row = cursor.fetchone() if row and row[0]: - return perform_document_info_aggregation(row[0][0], info_value) + existing = self._read_clob_json(row[0]) + return perform_document_info_aggregation(existing, info_value) except Exception as e: logger.warning(f"Error getting aggregated info: {e}") @@ -831,7 +900,7 @@ def get_document_info( ) row = cursor.fetchone() if row and row[0]: - return row[0] + return self._read_clob_json(row[0]) return {} diff --git a/uv.lock b/uv.lock index 2823fa268..08bc8968a 100644 --- a/uv.lock +++ b/uv.lock @@ -292,6 +292,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cd/3a/577b549de0cc09d95f11087ee63c739bba856cd3952697eec4c4bb91350a/bleach-6.3.0-py3-none-any.whl", hash = "sha256:fe10ec77c93ddf3d13a73b035abaac7a9f5e436513864ccdad516693213c65d6", size = 164437, upload-time = "2025-10-27T17:57:37.538Z" }, ] +[[package]] +name = "boltons" +version = "25.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/63/54/71a94d8e02da9a865587fb3fff100cb0fc7aa9f4d5ed9ed3a591216ddcc7/boltons-25.0.0.tar.gz", hash = "sha256:e110fbdc30b7b9868cb604e3f71d4722dd8f4dcb4a5ddd06028ba8f1ab0b5ace", size = 246294, upload-time = "2025-02-03T05:57:59.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/7f/0e961cf3908bc4c1c3e027de2794f867c6c89fb4916fc7dba295a0e80a2d/boltons-25.0.0-py3-none-any.whl", hash = "sha256:dc9fb38bf28985715497d1b54d00b62ea866eca3938938ea9043e254a3a6ca62", size = 194210, upload-time = "2025-02-03T05:57:56.705Z" }, +] + [[package]] name = "build" version = "1.4.0" @@ -533,24 +542,24 @@ wheels = [ ] [[package]] -name = "deprecated" -version = "1.3.1" +name = "decorator" +version = "5.2.1" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } +sdist = { url = "https://files.pythonhosted.org/packages/43/fa/6d96a0978d19e17b68d634497769987b16c8f4cd0a7a05048bec693caa6b/decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360", size = 56711, upload-time = "2025-02-24T04:41:34.073Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, + { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" }, ] [[package]] -name = "dirtyjson" -version = "1.0.8" +name = "deprecation" +version = "2.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/db/04/d24f6e645ad82ba0ef092fa17d9ef7a21953781663648a01c9371d9e8e98/dirtyjson-1.0.8.tar.gz", hash = "sha256:90ca4a18f3ff30ce849d100dcf4a003953c79d3a2348ef056f1d9c22231a25fd", size = 30782, upload-time = "2022-11-28T23:32:33.319Z" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788, upload-time = "2020-04-20T14:23:38.738Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/68/69/1bcf70f81de1b4a9f21b3a62ec0c83bdff991c88d6cc2267d02408457e88/dirtyjson-1.0.8-py3-none-any.whl", hash = "sha256:125e27248435a58acace26d5c2c4c11a1c0de0a9c5124c5a94ba78e517d74f53", size = 25197, upload-time = "2022-11-28T23:32:31.219Z" }, + { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178, upload-time = "2020-04-20T14:23:36.581Z" }, ] [[package]] @@ -604,6 +613,18 @@ vectorstore-mmr = [ { name = "simsimd" }, ] +[[package]] +name = "face" +version = "26.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "boltons" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/24/4e/0e106b0ba486cc38c858fb5efe899002f2ec4765e0808b298d8e19a16efb/face-26.0.0.tar.gz", hash = "sha256:ae12136ff0052f124811f5319670a8d9d29b7d2caaaabe542813690967cc6bca", size = 49862, upload-time = "2026-02-14T00:17:12.576Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/1d/c2f7a4334f7501a3474766b5bc0948e8e0b0916217a54d092dd700a5ed3c/face-26.0.0-py3-none-any.whl", hash = "sha256:6ec9cf271d8ee2447f04b14264209a09ec9cbe8252255e61fb7ab6b154e300f9", size = 54825, upload-time = "2026-02-14T00:17:11.519Z" }, +] + [[package]] name = "fastapi" version = "0.128.0" @@ -619,6 +640,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5c/05/5cbb59154b093548acd0f4c7c474a118eda06da25aa75c616b72d8fcd92a/fastapi-0.128.0-py3-none-any.whl", hash = "sha256:aebd93f9716ee3b4f4fcfe13ffb7cf308d99c9f3ab5622d8877441072561582d", size = 103094, upload-time = "2025-12-27T15:21:12.154Z" }, ] +[[package]] +name = "ffmpeg-python" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "future" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/5e/d5f9105d59c1325759d838af4e973695081fbbc97182baf73afc78dec266/ffmpeg-python-0.2.0.tar.gz", hash = "sha256:65225db34627c578ef0e11c8b1eb528bb35e024752f6f10b78c011f6f64c4127", size = 21543, upload-time = "2019-07-06T00:19:08.989Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/0c/56be52741f75bad4dc6555991fabd2e07b432d333da82c11ad701123888a/ffmpeg_python-0.2.0-py3-none-any.whl", hash = "sha256:ac441a0404e053f8b6a1113a77c0f452f1cfc62f6344a769475ffdc0f56c23c5", size = 25024, upload-time = "2019-07-06T00:19:07.215Z" }, +] + [[package]] name = "filelock" version = "3.20.3" @@ -727,6 +760,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/01/c9/97cc5aae1648dcb851958a3ddf73ccd7dbe5650d95203ecb4d7720b4cdbf/fsspec-2026.1.0-py3-none-any.whl", hash = "sha256:cb76aa913c2285a3b49bdd5fc55b1d7c708d7208126b60f2eb8194fe1b4cbdcc", size = 201838, upload-time = "2026-01-09T15:21:34.041Z" }, ] +[[package]] +name = "future" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/b2/4140c69c6a66432916b26158687e821ba631a4c9273c474343badf84d3ba/future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05", size = 1228490, upload-time = "2024-02-21T11:52:38.461Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/71/ae30dadffc90b9006d77af76b393cb9dfbfc9629f339fc1574a1c52e6806/future-1.0.0-py3-none-any.whl", hash = "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216", size = 491326, upload-time = "2024-02-21T11:52:35.956Z" }, +] + +[[package]] +name = "glom" +version = "25.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "boltons" }, + { name = "face" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/74/8387f95565ba7c30cd152a585b275ebb9a834d1d32782425c5d2fe0a102c/glom-25.12.0.tar.gz", hash = "sha256:1ae7da88be3693df40ad27bdf57a765a55c075c86c971bcddd67927403eb0069", size = 196128, upload-time = "2025-12-29T06:29:07.274Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/e6/4129d9a3baa72d747533bb33376543ccadd9a7f9944e5a6e3ae2e245f5d6/glom-25.12.0-py3-none-any.whl", hash = "sha256:b9f21e77f71a6576a43864e85066b8cc3f0f778d0d50961563f8981377a6dcb1", size = 103295, upload-time = "2025-12-29T06:29:06.074Z" }, +] + [[package]] name = "googleapis-common-protos" version = "1.72.0" @@ -951,6 +1007,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, ] +[[package]] +name = "imageio" +version = "2.37.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "pillow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/84/93bcd1300216ea50811cee96873b84a1bebf8d0489ffaf7f2a3756bab866/imageio-2.37.3.tar.gz", hash = "sha256:bbb37efbfc4c400fcd534b367b91fcd66d5da639aaa138034431a1c5e0a41451", size = 389673, upload-time = "2026-03-09T11:31:12.573Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/fa/391e437a34e55095173dca5f24070d89cbc233ff85bf1c29c93248c6588d/imageio-2.37.3-py3-none-any.whl", hash = "sha256:46f5bb8522cd421c0f5ae104d8268f569d856b29eb1a13b92829d1970f32c9f0", size = 317646, upload-time = "2026-03-09T11:31:10.771Z" }, +] + +[[package]] +name = "imageio-ffmpeg" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/44/bd/c3343c721f2a1b0c9fc71c1aebf1966a3b7f08c2eea8ed5437a2865611d6/imageio_ffmpeg-0.6.0.tar.gz", hash = "sha256:e2556bed8e005564a9f925bb7afa4002d82770d6b08825078b7697ab88ba1755", size = 25210, upload-time = "2025-01-16T21:34:32.747Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/58/87ef68ac83f4c7690961bce288fd8e382bc5f1513860fc7f90a9c1c1c6bf/imageio_ffmpeg-0.6.0-py3-none-macosx_10_9_intel.macosx_10_9_x86_64.whl", hash = "sha256:9d2baaf867088508d4a3458e61eeb30e945c4ad8016025545f66c4b5aaef0a61", size = 24932969, upload-time = "2025-01-16T21:34:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/40/5c/f3d8a657d362cc93b81aab8feda487317da5b5d31c0e1fdfd5e986e55d17/imageio_ffmpeg-0.6.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b1ae3173414b5fc5f538a726c4e48ea97edc0d2cdc11f103afee655c463fa742", size = 21113891, upload-time = "2025-01-16T21:34:00.277Z" }, + { url = "https://files.pythonhosted.org/packages/33/e7/1925bfbc563c39c1d2e82501d8372734a5c725e53ac3b31b4c2d081e895b/imageio_ffmpeg-0.6.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1d47bebd83d2c5fc770720d211855f208af8a596c82d17730aa51e815cdee6dc", size = 25632706, upload-time = "2025-01-16T21:33:53.475Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2d/43c8522a2038e9d0e7dbdf3a61195ecc31ca576fb1527a528c877e87d973/imageio_ffmpeg-0.6.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:c7e46fcec401dd990405049d2e2f475e2b397779df2519b544b8aab515195282", size = 29498237, upload-time = "2025-01-16T21:34:13.726Z" }, + { url = "https://files.pythonhosted.org/packages/a0/13/59da54728351883c3c1d9fca1710ab8eee82c7beba585df8f25ca925f08f/imageio_ffmpeg-0.6.0-py3-none-win32.whl", hash = "sha256:196faa79366b4a82f95c0f4053191d2013f4714a715780f0ad2a68ff37483cc2", size = 19652251, upload-time = "2025-01-16T21:34:06.812Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c6/fa760e12a2483469e2bf5058c5faff664acf66cadb4df2ad6205b016a73d/imageio_ffmpeg-0.6.0-py3-none-win_amd64.whl", hash = "sha256:02fa47c83703c37df6bfe4896aab339013f62bf02c5ebf2dce6da56af04ffc0a", size = 31246824, upload-time = "2025-01-16T21:34:28.6Z" }, +] + [[package]] name = "imagesize" version = "1.4.1" @@ -1053,15 +1136,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2f/9c/6753e6522b8d0ef07d3a3d239426669e984fb0eba15a315cdbc1253904e4/jiter-0.12.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c24e864cb30ab82311c6425655b0cdab0a98c5d973b065c66a3f020740c2324c", size = 346110, upload-time = "2025-11-09T20:49:21.817Z" }, ] -[[package]] -name = "joblib" -version = "1.5.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, -] - [[package]] name = "jsonpatch" version = "1.33" @@ -1083,6 +1157,56 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/92/5e77f98553e9e75130c78900d000368476aed74276eb8ae8796f65f00918/jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942", size = 7595, upload-time = "2024-06-10T19:24:40.698Z" }, ] +[[package]] +name = "lance-namespace" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lance-namespace-urllib3-client" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2b/c6/aec0d7752e15536564b50cf9a8926f0e5d7780aa3ab8ce8bca46daa55659/lance_namespace-0.5.2.tar.gz", hash = "sha256:566cc33091b5631793ab411f095d46c66391db0a62343cd6b4470265bb04d577", size = 10274, upload-time = "2026-02-20T03:14:31.777Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/3d/737c008d8fb2861e7ce260e2ffab0d5058eae41556181f80f1a1c3b52ef5/lance_namespace-0.5.2-py3-none-any.whl", hash = "sha256:6ccaf5649bf6ee6aa92eed9c535a114b7b4eb08e89f40426f58bc1466cbcffa3", size = 12087, upload-time = "2026-02-20T03:14:35.261Z" }, +] + +[[package]] +name = "lance-namespace-urllib3-client" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/64/51622c93ec8c164483c83b68764e5e76e52286c0137a8247bc6a7fac25f4/lance_namespace_urllib3_client-0.5.2.tar.gz", hash = "sha256:8a3a238006e6eabc01fc9d385ac3de22ba933aef0ae8987558f3c3199c9b3799", size = 172578, upload-time = "2026-02-20T03:14:33.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/10/f86d994498b37f7f35d0b8c2f7626a16fe4cb1949b518c1e5d5052ecf95f/lance_namespace_urllib3_client-0.5.2-py3-none-any.whl", hash = "sha256:83cefb6fd6e5df0b99b5e866ee3d46300d375b75e8af32c27bc16fbf7c1a5978", size = 300351, upload-time = "2026-02-20T03:14:34.236Z" }, +] + +[[package]] +name = "lancedb" +version = "0.29.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecation" }, + { name = "lance-namespace" }, + { name = "numpy" }, + { name = "overrides", marker = "python_full_version < '3.12'" }, + { name = "packaging" }, + { name = "pyarrow" }, + { name = "pydantic" }, + { name = "tqdm" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/77/fbb25946a234928958e016c5448343fd314bd601315f9587568321591a17/lancedb-0.29.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:bc1faf2e12addb9585569d0fb114ecc25ec3867e4e1aa6934e9343cfb5265ee4", size = 42341708, upload-time = "2026-02-09T06:21:31.677Z" }, + { url = "https://files.pythonhosted.org/packages/cd/95/d3a7b6d0237e343ad5b2afef2bdb99423746d5c3e882a9cab68dc041c2d0/lancedb-0.29.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fec19cfc52a5b9d98e060bd2f02a1c9df6a0bfd15b36021b6017327a41893a3", size = 44147347, upload-time = "2026-02-09T06:31:02.567Z" }, + { url = "https://files.pythonhosted.org/packages/66/21/153a42294279c5b66d763f357808dde0899b71c5c8e41ad5ecbeeb8728df/lancedb-0.29.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:636939ab9225d435020ba17c231f5eaba15312a07813bcebcd71128204cc039f", size = 47186355, upload-time = "2026-02-09T06:34:47.726Z" }, + { url = "https://files.pythonhosted.org/packages/a2/f7/f7041ae7d7730332b2754fe7adc2e0bd496f92bf526ac710b7eb3caf1d0a/lancedb-0.29.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f79b32083fcab139009db521d2f7fcd6afe4cca98a78c06c5940ff00a170cc1a", size = 44172354, upload-time = "2026-02-09T06:31:03.834Z" }, + { url = "https://files.pythonhosted.org/packages/72/6f/c152497c18cea0f36b523fc03b8e0a48be2b120276cc15a86d79b8b83cde/lancedb-0.29.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:991043a28c1f49f14df2479b554a95c759a85666dc58573cc86c1b9df05db794", size = 47228009, upload-time = "2026-02-09T06:34:40.872Z" }, + { url = "https://files.pythonhosted.org/packages/66/50/bd47bca59a87a88a4ca291a0718291422440750d84b34318048c70a537c2/lancedb-0.29.2-cp39-abi3-win_amd64.whl", hash = "sha256:101eb0ac018bb0b643dd9ea22065f6f2102e9d44c9ac58a197477ccbfbc0b9fa", size = 52028768, upload-time = "2026-02-09T07:00:02.272Z" }, +] + [[package]] name = "langchain" version = "1.2.7" @@ -1307,65 +1431,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, ] -[[package]] -name = "llama-index-core" -version = "0.10.45" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "dataclasses-json" }, - { name = "deprecated" }, - { name = "dirtyjson" }, - { name = "fsspec" }, - { name = "httpx" }, - { name = "llamaindex-py-client" }, - { name = "nest-asyncio" }, - { name = "networkx" }, - { name = "nltk" }, - { name = "numpy" }, - { name = "openai" }, - { name = "pandas" }, - { name = "pillow" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "sqlalchemy", extra = ["asyncio"] }, - { name = "tenacity" }, - { name = "tiktoken" }, - { name = "tqdm" }, - { name = "typing-extensions" }, - { name = "typing-inspect" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a7/6b/1cd00e76b5bdc0060567fdc2c41e9ba6f29a17ec90c36afc5ac9540a4306/llama_index_core-0.10.45.tar.gz", hash = "sha256:f32d0448e7193ff45c8e84abd49493be030998fc8f1a0cab069387deef3e577c", size = 15109807, upload-time = "2024-06-14T22:43:47.813Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/79/fc/75b116bafcf13346378fbaa138dc3c96226c672ccb123b32c4abba6ba3b3/llama_index_core-0.10.45-py3-none-any.whl", hash = "sha256:8c800c7221322b8e1cbbbc13325039b5fe3575d4b0e0be14ac9a8f1e5d14fee3", size = 15436131, upload-time = "2024-06-14T22:43:39.897Z" }, -] - -[[package]] -name = "llama-index-embeddings-nvidia" -version = "0.1.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llama-index-core" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c5/7b/01680981eda1d3f662451d86e9e2ab4cd1c64777f948c1c124473208b62a/llama_index_embeddings_nvidia-0.1.5.tar.gz", hash = "sha256:1b6d4d679a2176b8a174af21724c90d560ff417375b20e6bd8045200b45ccd54", size = 5579, upload-time = "2024-08-12T14:46:51.979Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/f7/3a5b84110761c7e4a5259b272775b98bcb41c7b5856ac19be1fd8b9c2bfd/llama_index_embeddings_nvidia-0.1.5-py3-none-any.whl", hash = "sha256:6fbe1efba56d96e67d434b77043f98f585a01d11067c1ea8fb52219609b6b1c3", size = 5885, upload-time = "2024-08-12T14:46:50.646Z" }, -] - -[[package]] -name = "llamaindex-py-client" -version = "0.1.19" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a0/1b/9dbd37e1c2efb21e91047b2ae7379e309e45a884ed35aa5d3e754ea69cb3/llamaindex_py_client-0.1.19.tar.gz", hash = "sha256:73f74792bb8c092bae6dc626627a09ac13a099fa8d10f8fcc83e17a2b332cca7", size = 54097, upload-time = "2024-04-26T15:59:05.413Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/9c/a1a3086f023a8957ebd527c98d9356b6dd9409edef827ffdeee484f47abf/llamaindex_py_client-0.1.19-py3-none-any.whl", hash = "sha256:fd9416fd78b97209bf323bc3c7fab314499778563e7274f10853ad560563d10e", size = 141926, upload-time = "2024-04-26T15:59:02.888Z" }, -] - [[package]] name = "markdown-it-py" version = "4.0.0" @@ -1463,20 +1528,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] -[[package]] -name = "milvus-lite" -version = "2.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tqdm" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/b2/acc5024c8e8b6a0b034670b8e8af306ebd633ede777dcbf557eac4785937/milvus_lite-2.5.1-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:6b014453200ba977be37ba660cb2d021030375fa6a35bc53c2e1d92980a0c512", size = 27934713, upload-time = "2025-06-30T04:23:37.028Z" }, - { url = "https://files.pythonhosted.org/packages/9b/2e/746f5bb1d6facd1e73eb4af6dd5efda11125b0f29d7908a097485ca6cad9/milvus_lite-2.5.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2e031088bf308afe5f8567850412d618cfb05a65238ed1a6117f60decccc95a", size = 24421451, upload-time = "2025-06-30T04:23:51.747Z" }, - { url = "https://files.pythonhosted.org/packages/2e/cf/3d1fee5c16c7661cf53977067a34820f7269ed8ba99fe9cf35efc1700866/milvus_lite-2.5.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:a13277e9bacc6933dea172e42231f7e6135bd3bdb073dd2688ee180418abd8d9", size = 45337093, upload-time = "2025-06-30T04:24:06.706Z" }, - { url = "https://files.pythonhosted.org/packages/d3/82/41d9b80f09b82e066894d9b508af07b7b0fa325ce0322980674de49106a0/milvus_lite-2.5.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25ce13f4b8d46876dd2b7ac8563d7d8306da7ff3999bb0d14b116b30f71d706c", size = 55263911, upload-time = "2025-06-30T04:24:19.434Z" }, -] - [[package]] name = "minio" version = "7.2.20" @@ -1493,6 +1544,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3e/9a/b697530a882588a84db616580f2ba5d1d515c815e11c30d219145afeec87/minio-7.2.20-py3-none-any.whl", hash = "sha256:eb33dd2fb80e04c3726a76b13241c6be3c4c46f8d81e1d58e757786f6501897e", size = 93751, upload-time = "2025-11-27T00:37:13.993Z" }, ] +[[package]] +name = "moviepy" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "decorator" }, + { name = "imageio" }, + { name = "imageio-ffmpeg" }, + { name = "numpy" }, + { name = "pillow" }, + { name = "proglog" }, + { name = "python-dotenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/61/15f9476e270f64c78a834e7459ca045d669f869cec24eed26807b8cd479d/moviepy-2.2.1.tar.gz", hash = "sha256:c80cb56815ece94e5e3e2d361aa40070eeb30a09d23a24c4e684d03e16deacb1", size = 58431438, upload-time = "2025-05-21T19:31:52.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/73/7d3b2010baa0b5eb1e4dfa9e4385e89b6716be76f2fa21a6c0fe34b68e5a/moviepy-2.2.1-py3-none-any.whl", hash = "sha256:6b56803fec2ac54b557404126ac1160e65448e03798fa282bd23e8fab3795060", size = 129871, upload-time = "2025-05-21T19:31:50.11Z" }, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -1610,39 +1679,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d3/ac/686789b9145413f1a61878c407210e41bfdb097976864e0913078b24098c/myst_parser-5.0.0-py3-none-any.whl", hash = "sha256:ab31e516024918296e169139072b81592336f2fef55b8986aa31c9f04b5f7211", size = 84533, upload-time = "2026-01-15T09:08:16.788Z" }, ] -[[package]] -name = "nest-asyncio" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, -] - -[[package]] -name = "networkx" -version = "3.6.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, -] - -[[package]] -name = "nltk" -version = "3.9.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "joblib" }, - { name = "regex" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e1/8f/915e1c12df07c70ed779d18ab83d065718a926e70d3ea33eb0cd66ffb7c0/nltk-3.9.3.tar.gz", hash = "sha256:cb5945d6424a98d694c2b9a0264519fab4363711065a46aa0ae7a2195b92e71f", size = 2923673, upload-time = "2026-02-24T12:05:53.833Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/7e/9af5a710a1236e4772de8dfcc6af942a561327bb9f42b5b4a24d0cf100fd/nltk-3.9.3-py3-none-any.whl", hash = "sha256:60b3db6e9995b3dd976b1f0fa7dec22069b2677e759c28eb69b62ddd44870522", size = 1525385, upload-time = "2026-02-24T12:05:46.54Z" }, -] - [[package]] name = "numpy" version = "2.4.1" @@ -1703,23 +1739,28 @@ wheels = [ [[package]] name = "nv-ingest-api" -version = "25.9.0" +version = "26.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff" }, + { name = "ffmpeg-python" }, + { name = "fsspec" }, + { name = "glom" }, + { name = "moviepy" }, { name = "pandas" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "tritonclient" }, + { name = "universal-pathlib" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8b/84/70a7d089f4b7961eff4dfa8e4d0a485a7fba5c4403bb061f81789f25578a/nv_ingest_api-25.9.0.tar.gz", hash = "sha256:45f611e51560b8175ce06097f9b954500ae19aba4322a59796f1fb533e6d5fc2", size = 209566, upload-time = "2025-09-16T21:06:10.994Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/bd/e6e885cca94b89723468d4c32d52d30ccc0235ebe2f1db33b0605402d6b8/nv_ingest_api-26.1.2.tar.gz", hash = "sha256:fea08f9bda064938a5876f1610ef0b92c6a1e4943130c564f329b0c87efa3daf", size = 259604, upload-time = "2026-01-21T14:06:27.092Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/32/38/9ec1cb9d594f64b6eb88ad1da31c891c70444b742e7f5645b5ef4646e509/nv_ingest_api-25.9.0-py3-none-any.whl", hash = "sha256:f39fcafbcd2842be9fce9485e65afd301992bdb2d42f31d8c6736dd8ba6ba4fe", size = 296295, upload-time = "2025-09-16T21:06:08.344Z" }, + { url = "https://files.pythonhosted.org/packages/78/66/21e30e658578b7e5ab30857b99e9a0a5c91728ffdca13dadc3d3dba58b98/nv_ingest_api-26.1.2-py3-none-any.whl", hash = "sha256:8e7539a6b7d52afd821c0030e3197cfffddc011d26a8b093cf7b5ffa8addf02d", size = 357537, upload-time = "2026-01-21T14:06:24.321Z" }, ] [[package]] name = "nv-ingest-client" -version = "25.9.0" +version = "26.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "build" }, @@ -1727,17 +1768,16 @@ dependencies = [ { name = "click" }, { name = "fsspec" }, { name = "httpx" }, - { name = "llama-index-embeddings-nvidia" }, - { name = "nv-ingest-api" }, + { name = "lancedb" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "requests" }, { name = "setuptools" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4b/6c/04037168ac68ec9f6168e5c50572e84238225ab58e4c31cf80530cc7ad82/nv_ingest_client-25.9.0.tar.gz", hash = "sha256:1e43f925b28843f9e47108e37b22c0a3c2a77af15be4ed4b5977a83af1f55a09", size = 99358, upload-time = "2025-09-16T21:06:15.419Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/62/386bc4a336b91df9c65afc18d905bb1fe3dd44ef1ce038895a701cba6035/nv_ingest_client-26.1.2.tar.gz", hash = "sha256:7ea4a35d4e7051031c273eb2b15170a0555462b702602e5c4fdce947bd39d446", size = 126061, upload-time = "2026-01-21T14:06:31.836Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/89/96/548b87f4e14bc014fccdc87edc2fa8ae70af299a36084bdd23a9f1884452/nv_ingest_client-25.9.0-py3-none-any.whl", hash = "sha256:ab07fdff253d678986db1bc2b083b7c94243be5d327155dbbc19883f49675e41", size = 114937, upload-time = "2025-09-16T21:06:12.764Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a0/6f082f42ba1ba4e3de751deee98099114405489389d82b840024fa26512e/nv_ingest_client-26.1.2-py3-none-any.whl", hash = "sha256:49ec81c2e2470509fc527d778d886caebe05a7f3012918eef2fd67922ea9b8b4", size = 146091, upload-time = "2026-01-21T14:06:28.586Z" }, ] [[package]] @@ -1757,12 +1797,11 @@ dependencies = [ { name = "langchain-milvus" }, { name = "langchain-nvidia-ai-endpoints" }, { name = "lark" }, - { name = "minio" }, { name = "oracledb" }, { name = "pdfplumber" }, { name = "protobuf" }, { name = "pydantic" }, - { name = "pymilvus", extra = ["milvus-lite"] }, + { name = "pymilvus" }, { name = "pymilvus-model" }, { name = "python-dateutil" }, { name = "python-multipart" }, @@ -1816,6 +1855,9 @@ ingest = [ { name = "tqdm" }, { name = "tritonclient" }, ] +minio = [ + { name = "minio" }, +] oracle = [ { name = "langchain-community" }, { name = "oracledb" }, @@ -1876,11 +1918,11 @@ requires-dist = [ { name = "langchain-openai", marker = "extra == 'ingest'", specifier = ">=0.2" }, { name = "langchain-openai", marker = "extra == 'rag'", specifier = ">=0.2" }, { name = "lark", specifier = ">=1.2.2" }, - { name = "minio", specifier = ">=7.2,<8.0" }, - { name = "nv-ingest-api", marker = "extra == 'all'", specifier = "==25.9.0" }, - { name = "nv-ingest-api", marker = "extra == 'ingest'", specifier = "==25.9.0" }, - { name = "nv-ingest-client", marker = "extra == 'all'", specifier = "==25.9.0" }, - { name = "nv-ingest-client", marker = "extra == 'ingest'", specifier = "==25.9.0" }, + { name = "minio", marker = "extra == 'minio'", specifier = ">=7.2,<8.0" }, + { name = "nv-ingest-api", marker = "extra == 'all'", specifier = "==26.1.2" }, + { name = "nv-ingest-api", marker = "extra == 'ingest'", specifier = "==26.1.2" }, + { name = "nv-ingest-client", marker = "extra == 'all'", specifier = "==26.1.2" }, + { name = "nv-ingest-client", marker = "extra == 'ingest'", specifier = "==26.1.2" }, { name = "opentelemetry-api", marker = "extra == 'all'", specifier = ">=1.29,<2.0" }, { name = "opentelemetry-api", marker = "extra == 'ingest'", specifier = ">=1.29,<2.0" }, { name = "opentelemetry-api", marker = "extra == 'rag'", specifier = ">=1.29,<2.0" }, @@ -1918,7 +1960,7 @@ requires-dist = [ { name = "pyarrow", marker = "extra == 'ingest'", specifier = ">=21.0,<22.0" }, { name = "pyarrow", marker = "extra == 'rag'", specifier = ">=21.0,<22.0" }, { name = "pydantic", specifier = ">=2.11,<3.0" }, - { name = "pymilvus", extras = ["milvus-lite"], specifier = ">=2.6.7,<3.0" }, + { name = "pymilvus", specifier = ">=2.6.7,<3.0" }, { name = "pymilvus-model", specifier = ">=0.3,<1.0" }, { name = "python-dateutil", specifier = ">=2.9.0.post0" }, { name = "python-multipart", specifier = ">=0.0.22,<1.0" }, @@ -1931,7 +1973,7 @@ requires-dist = [ { name = "tritonclient", marker = "extra == 'ingest'", specifier = "==2.57.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.32,<1.0" }, ] -provides-extras = ["rag", "ingest", "all", "elasticsearch", "oracle"] +provides-extras = ["rag", "ingest", "all", "elasticsearch", "oracle", "minio"] [package.metadata.requires-dev] docs = [ @@ -2418,6 +2460,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/17/e756653095a083d8a37cbd816cb87148debcfcd920129b25f99dd8d04271/pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc", size = 13199233, upload-time = "2025-09-29T23:24:24.876Z" }, ] +[[package]] +name = "pathlib-abc" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/cb/448649d7f25d228bf0be3a04590ab7afa77f15e056f8fa976ed05ec9a78f/pathlib_abc-0.5.2.tar.gz", hash = "sha256:fcd56f147234645e2c59c7ae22808b34c364bb231f685ddd9f96885aed78a94c", size = 33342, upload-time = "2025-10-10T18:37:20.524Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/29/c028a0731e202035f0e2e0bfbf1a3e46ad6c628cbb17f6f1cc9eea5d9ff1/pathlib_abc-0.5.2-py3-none-any.whl", hash = "sha256:4c9d94cf1b23af417ce7c0417b43333b06a106c01000b286c99de230d95eefbb", size = 19070, upload-time = "2025-10-10T18:37:19.437Z" }, +] + [[package]] name = "pdfminer-six" version = "20251230" @@ -2507,6 +2558,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/26/c56ce33ca856e358d27fda9676c055395abddb82c35ac0f593877ed4562e/pillow-12.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:cb9bb857b2d057c6dfc72ac5f3b44836924ba15721882ef103cecb40d002d80e", size = 7029880, upload-time = "2026-02-11T04:23:04.783Z" }, ] +[[package]] +name = "proglog" +version = "0.1.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/af/c108866c452eda1132f3d6b3cb6be2ae8430c97e9309f38ca9dbd430af37/proglog-0.1.12.tar.gz", hash = "sha256:361ee074721c277b89b75c061336cb8c5f287c92b043efa562ccf7866cda931c", size = 8794, upload-time = "2025-05-09T14:36:18.316Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/1b/f7ea6cde25621cd9236541c66ff018f4268012a534ec31032bcb187dc5e7/proglog-0.1.12-py3-none-any.whl", hash = "sha256:ccaafce51e80a81c65dc907a460c07ccb8ec1f78dc660cfd8f9ec3a22f01b84c", size = 6337, upload-time = "2025-05-09T14:36:16.798Z" }, +] + [[package]] name = "prometheus-client" version = "0.24.1" @@ -2819,11 +2882,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/0c/777cf174b8f4f12d458857ad5ede981be0a4b5dc160a98da2654832e34e5/pymilvus-2.6.7-py3-none-any.whl", hash = "sha256:a9fe5f8ea4191c35b53fd99820f802dca3a6efc336c894c59b19e5472545383a", size = 298693, upload-time = "2026-01-26T13:20:30.308Z" }, ] -[package.optional-dependencies] -milvus-lite = [ - { name = "milvus-lite", marker = "sys_platform != 'win32'" }, -] - [[package]] name = "pymilvus-model" version = "0.3.2" @@ -3520,11 +3578,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/e1/3ccb13c643399d22289c6a9786c1a91e3dcbb68bce4beb44926ac2c557bf/sqlalchemy-2.0.45-py3-none-any.whl", hash = "sha256:5225a288e4c8cc2308dbdd874edad6e7d0fd38eac1e9e5f23503425c8eee20d0", size = 1936672, upload-time = "2025-12-09T21:54:52.608Z" }, ] -[package.optional-dependencies] -asyncio = [ - { name = "greenlet" }, -] - [[package]] name = "starlette" version = "0.50.0" @@ -3744,6 +3797,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1", size = 348521, upload-time = "2025-12-13T17:45:33.889Z" }, ] +[[package]] +name = "universal-pathlib" +version = "0.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fsspec" }, + { name = "pathlib-abc" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/6e/d997a70ee8f4c61f9a7e2f4f8af721cf072a3326848fc881b05187e52558/universal_pathlib-0.3.10.tar.gz", hash = "sha256:4487cbc90730a48cfb64f811d99e14b6faed6d738420cd5f93f59f48e6930bfb", size = 261110, upload-time = "2026-02-22T14:40:58.87Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/1a/5d9a402b39ec892d856bbdd9db502ff73ce28cdf4aff72eb1ce1d6843506/universal_pathlib-0.3.10-py3-none-any.whl", hash = "sha256:dfaf2fb35683d2eb1287a3ed7b215e4d6016aa6eaf339c607023d22f90821c66", size = 83528, upload-time = "2026-02-22T14:40:57.316Z" }, +] + [[package]] name = "urllib3" version = "2.6.3" From f00a69f007413c1e15a2e8259f22bc40241ab245 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Wed, 11 Mar 2026 15:59:39 -0500 Subject: [PATCH 56/81] updated uv.lock --- uv.lock | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 08bc8968a..00b0abdae 100644 --- a/uv.lock +++ b/uv.lock @@ -438,6 +438,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, ] +[[package]] +name = "circuitbreaker" +version = "2.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/ac/de7a92c4ed39cba31fe5ad9203b76a25ca67c530797f6bb420fff5f65ccb/circuitbreaker-2.1.3.tar.gz", hash = "sha256:1a4baee510f7bea3c91b194dcce7c07805fe96c4423ed5594b75af438531d084", size = 10787, upload-time = "2025-03-31T08:12:08.963Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/34/15f08edd4628f65217de1fc3c1a27c82e46fe357d60c217fc9881e12ebcc/circuitbreaker-2.1.3-py3-none-any.whl", hash = "sha256:87ba6a3ed03fdc7032bc175561c2b04d52ade9d5faf94ca2b035fbdc5e6b1dd1", size = 7737, upload-time = "2025-03-31T08:12:07.802Z" }, +] + [[package]] name = "click" version = "8.3.1" @@ -1528,6 +1537,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "milvus-lite" +version = "2.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tqdm" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/b2/acc5024c8e8b6a0b034670b8e8af306ebd633ede777dcbf557eac4785937/milvus_lite-2.5.1-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:6b014453200ba977be37ba660cb2d021030375fa6a35bc53c2e1d92980a0c512", size = 27934713, upload-time = "2025-06-30T04:23:37.028Z" }, + { url = "https://files.pythonhosted.org/packages/9b/2e/746f5bb1d6facd1e73eb4af6dd5efda11125b0f29d7908a097485ca6cad9/milvus_lite-2.5.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2e031088bf308afe5f8567850412d618cfb05a65238ed1a6117f60decccc95a", size = 24421451, upload-time = "2025-06-30T04:23:51.747Z" }, + { url = "https://files.pythonhosted.org/packages/2e/cf/3d1fee5c16c7661cf53977067a34820f7269ed8ba99fe9cf35efc1700866/milvus_lite-2.5.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:a13277e9bacc6933dea172e42231f7e6135bd3bdb073dd2688ee180418abd8d9", size = 45337093, upload-time = "2025-06-30T04:24:06.706Z" }, + { url = "https://files.pythonhosted.org/packages/d3/82/41d9b80f09b82e066894d9b508af07b7b0fa325ce0322980674de49106a0/milvus_lite-2.5.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25ce13f4b8d46876dd2b7ac8563d7d8306da7ff3999bb0d14b116b30f71d706c", size = 55263911, upload-time = "2025-06-30T04:24:19.434Z" }, +] + [[package]] name = "minio" version = "7.2.20" @@ -1801,7 +1824,7 @@ dependencies = [ { name = "pdfplumber" }, { name = "protobuf" }, { name = "pydantic" }, - { name = "pymilvus" }, + { name = "pymilvus", extra = ["milvus-lite"] }, { name = "pymilvus-model" }, { name = "python-dateutil" }, { name = "python-multipart" }, @@ -1858,6 +1881,9 @@ ingest = [ minio = [ { name = "minio" }, ] +oci = [ + { name = "oci" }, +] oracle = [ { name = "langchain-community" }, { name = "oracledb" }, @@ -1923,6 +1949,7 @@ requires-dist = [ { name = "nv-ingest-api", marker = "extra == 'ingest'", specifier = "==26.1.2" }, { name = "nv-ingest-client", marker = "extra == 'all'", specifier = "==26.1.2" }, { name = "nv-ingest-client", marker = "extra == 'ingest'", specifier = "==26.1.2" }, + { name = "oci", marker = "extra == 'oci'", specifier = ">=2.130,<3.0" }, { name = "opentelemetry-api", marker = "extra == 'all'", specifier = ">=1.29,<2.0" }, { name = "opentelemetry-api", marker = "extra == 'ingest'", specifier = ">=1.29,<2.0" }, { name = "opentelemetry-api", marker = "extra == 'rag'", specifier = ">=1.29,<2.0" }, @@ -1960,7 +1987,7 @@ requires-dist = [ { name = "pyarrow", marker = "extra == 'ingest'", specifier = ">=21.0,<22.0" }, { name = "pyarrow", marker = "extra == 'rag'", specifier = ">=21.0,<22.0" }, { name = "pydantic", specifier = ">=2.11,<3.0" }, - { name = "pymilvus", specifier = ">=2.6.7,<3.0" }, + { name = "pymilvus", extras = ["milvus-lite"], specifier = ">=2.6.7,<3.0" }, { name = "pymilvus-model", specifier = ">=0.3,<1.0" }, { name = "python-dateutil", specifier = ">=2.9.0.post0" }, { name = "python-multipart", specifier = ">=0.0.22,<1.0" }, @@ -1973,7 +2000,7 @@ requires-dist = [ { name = "tritonclient", marker = "extra == 'ingest'", specifier = "==2.57.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.32,<1.0" }, ] -provides-extras = ["rag", "ingest", "all", "elasticsearch", "oracle", "minio"] +provides-extras = ["rag", "ingest", "all", "elasticsearch", "oracle", "minio", "oci"] [package.metadata.requires-dev] docs = [ @@ -2000,6 +2027,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/79/017fab2f7167a9a9795665f894d04f77aafceca80821b51589bb4b23ff5c/nvidia_sphinx_theme-0.0.9.post1-py3-none-any.whl", hash = "sha256:21ca60206dff2f380d7783d64bbaf71a5b9cacae53c7d0686f089c16b5a3d45a", size = 143816, upload-time = "2025-11-09T23:16:55.719Z" }, ] +[[package]] +name = "oci" +version = "2.168.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "circuitbreaker" }, + { name = "cryptography" }, + { name = "pyopenssl" }, + { name = "python-dateutil" }, + { name = "pytz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/68/edf8ffbb42e97ad44d64fce85be00818d979b472dd4377dc948155f811e9/oci-2.168.1.tar.gz", hash = "sha256:b941674171b41e999b8e3adb38d4797d7b42d2bb5ff40d17c26e8ce2a7d4b605", size = 16751235, upload-time = "2026-03-10T10:50:16.244Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/3e/29e05b4f8bed3b4a89b52fc57e76ac86669fc43a59e128eb526e395eda7b/oci-2.168.1-py3-none-any.whl", hash = "sha256:d106cfffc9153b5c9de628877c967ed87bbbfbbc9d411c97feee0eba8f2e4eab", size = 34033119, upload-time = "2026-03-10T10:50:08.501Z" }, +] + [[package]] name = "onnxruntime" version = "1.23.2" @@ -2882,6 +2926,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/0c/777cf174b8f4f12d458857ad5ede981be0a4b5dc160a98da2654832e34e5/pymilvus-2.6.7-py3-none-any.whl", hash = "sha256:a9fe5f8ea4191c35b53fd99820f802dca3a6efc336c894c59b19e5472545383a", size = 298693, upload-time = "2026-01-26T13:20:30.308Z" }, ] +[package.optional-dependencies] +milvus-lite = [ + { name = "milvus-lite", marker = "sys_platform != 'win32'" }, +] + [[package]] name = "pymilvus-model" version = "0.3.2" @@ -2898,6 +2947,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/ba/b34e3f8c57c453c02daf1a1eff9c4b62a2e89fc10a29bc83fb685155fe14/pymilvus_model-0.3.2-py3-none-any.whl", hash = "sha256:df8a90519a2adc47bd40f37d8c250b0ad0c7aaafc6139b52dc896d5cfda55d35", size = 47699, upload-time = "2025-03-31T08:47:43.372Z" }, ] +[[package]] +name = "pyopenssl" +version = "25.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/be/97b83a464498a79103036bc74d1038df4a7ef0e402cfaf4d5e113fb14759/pyopenssl-25.3.0.tar.gz", hash = "sha256:c981cb0a3fd84e8602d7afc209522773b94c1c2446a3c710a75b06fe1beae329", size = 184073, upload-time = "2025-09-17T00:32:21.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/81/ef2b1dfd1862567d573a4fdbc9f969067621764fbb74338496840a1d2977/pyopenssl-25.3.0-py3-none-any.whl", hash = "sha256:1fda6fc034d5e3d179d39e59c1895c9faeaf40a79de5fc4cbbfbe0d36f4a77b6", size = 57268, upload-time = "2025-09-17T00:32:19.474Z" }, +] + [[package]] name = "pypdfium2" version = "5.3.0" From 787ddb4364cd6969e305d4731a953d7d9447a6dc Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Wed, 11 Mar 2026 16:03:47 -0500 Subject: [PATCH 57/81] updated uv.lock again --- uv.lock | 45 +-------------------------------------------- 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/uv.lock b/uv.lock index 00b0abdae..b420112a6 100644 --- a/uv.lock +++ b/uv.lock @@ -438,15 +438,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, ] -[[package]] -name = "circuitbreaker" -version = "2.1.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/df/ac/de7a92c4ed39cba31fe5ad9203b76a25ca67c530797f6bb420fff5f65ccb/circuitbreaker-2.1.3.tar.gz", hash = "sha256:1a4baee510f7bea3c91b194dcce7c07805fe96c4423ed5594b75af438531d084", size = 10787, upload-time = "2025-03-31T08:12:08.963Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/34/15f08edd4628f65217de1fc3c1a27c82e46fe357d60c217fc9881e12ebcc/circuitbreaker-2.1.3-py3-none-any.whl", hash = "sha256:87ba6a3ed03fdc7032bc175561c2b04d52ade9d5faf94ca2b035fbdc5e6b1dd1", size = 7737, upload-time = "2025-03-31T08:12:07.802Z" }, -] - [[package]] name = "click" version = "8.3.1" @@ -1881,9 +1872,6 @@ ingest = [ minio = [ { name = "minio" }, ] -oci = [ - { name = "oci" }, -] oracle = [ { name = "langchain-community" }, { name = "oracledb" }, @@ -1949,7 +1937,6 @@ requires-dist = [ { name = "nv-ingest-api", marker = "extra == 'ingest'", specifier = "==26.1.2" }, { name = "nv-ingest-client", marker = "extra == 'all'", specifier = "==26.1.2" }, { name = "nv-ingest-client", marker = "extra == 'ingest'", specifier = "==26.1.2" }, - { name = "oci", marker = "extra == 'oci'", specifier = ">=2.130,<3.0" }, { name = "opentelemetry-api", marker = "extra == 'all'", specifier = ">=1.29,<2.0" }, { name = "opentelemetry-api", marker = "extra == 'ingest'", specifier = ">=1.29,<2.0" }, { name = "opentelemetry-api", marker = "extra == 'rag'", specifier = ">=1.29,<2.0" }, @@ -2000,7 +1987,7 @@ requires-dist = [ { name = "tritonclient", marker = "extra == 'ingest'", specifier = "==2.57.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.32,<1.0" }, ] -provides-extras = ["rag", "ingest", "all", "elasticsearch", "oracle", "minio", "oci"] +provides-extras = ["rag", "ingest", "all", "elasticsearch", "oracle", "minio"] [package.metadata.requires-dev] docs = [ @@ -2027,23 +2014,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/79/017fab2f7167a9a9795665f894d04f77aafceca80821b51589bb4b23ff5c/nvidia_sphinx_theme-0.0.9.post1-py3-none-any.whl", hash = "sha256:21ca60206dff2f380d7783d64bbaf71a5b9cacae53c7d0686f089c16b5a3d45a", size = 143816, upload-time = "2025-11-09T23:16:55.719Z" }, ] -[[package]] -name = "oci" -version = "2.168.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "circuitbreaker" }, - { name = "cryptography" }, - { name = "pyopenssl" }, - { name = "python-dateutil" }, - { name = "pytz" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/68/edf8ffbb42e97ad44d64fce85be00818d979b472dd4377dc948155f811e9/oci-2.168.1.tar.gz", hash = "sha256:b941674171b41e999b8e3adb38d4797d7b42d2bb5ff40d17c26e8ce2a7d4b605", size = 16751235, upload-time = "2026-03-10T10:50:16.244Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/3e/29e05b4f8bed3b4a89b52fc57e76ac86669fc43a59e128eb526e395eda7b/oci-2.168.1-py3-none-any.whl", hash = "sha256:d106cfffc9153b5c9de628877c967ed87bbbfbbc9d411c97feee0eba8f2e4eab", size = 34033119, upload-time = "2026-03-10T10:50:08.501Z" }, -] - [[package]] name = "onnxruntime" version = "1.23.2" @@ -2947,19 +2917,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/ba/b34e3f8c57c453c02daf1a1eff9c4b62a2e89fc10a29bc83fb685155fe14/pymilvus_model-0.3.2-py3-none-any.whl", hash = "sha256:df8a90519a2adc47bd40f37d8c250b0ad0c7aaafc6139b52dc896d5cfda55d35", size = 47699, upload-time = "2025-03-31T08:47:43.372Z" }, ] -[[package]] -name = "pyopenssl" -version = "25.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/80/be/97b83a464498a79103036bc74d1038df4a7ef0e402cfaf4d5e113fb14759/pyopenssl-25.3.0.tar.gz", hash = "sha256:c981cb0a3fd84e8602d7afc209522773b94c1c2446a3c710a75b06fe1beae329", size = 184073, upload-time = "2025-09-17T00:32:21.037Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/81/ef2b1dfd1862567d573a4fdbc9f969067621764fbb74338496840a1d2977/pyopenssl-25.3.0-py3-none-any.whl", hash = "sha256:1fda6fc034d5e3d179d39e59c1895c9faeaf40a79de5fc4cbbfbe0d36f4a77b6", size = 57268, upload-time = "2025-09-17T00:32:19.474Z" }, -] - [[package]] name = "pypdfium2" version = "5.3.0" From 6d96e2eea0920502133ca4ee6321af80fbc7a38f Mon Sep 17 00:00:00 2001 From: Kurt Heiss Date: Wed, 11 Mar 2026 23:32:09 -0700 Subject: [PATCH 58/81] Kheiss/prd additions (#427) * confirming presence of switcher text in conf.py file * Update documentation to reflect name change to NeMo Retriever Library * Update api-rag.md removed GitHub markers * Update change-model.md Removed GitHub markers * Update deploy-helm.md removed GitHub markers * Update deploy-helm.md Remove GitHub markers * Update deploy-helm.md Remove GitHub markers * Update mig-deployment.md removed GitHub markers * Update deploy-helm.md removed extra space --- docs/api-ingestor.md | 4 +-- docs/api-rag.md | 2 ++ docs/audio_ingestion.md | 2 +- docs/change-model.md | 14 +++++++- docs/change-vectordb.md | 2 +- docs/conf.py | 6 ++-- docs/custom-metadata.md | 14 ++++---- docs/debugging.md | 8 ++--- docs/deploy-docker-nvidia-hosted.md | 4 +-- docs/deploy-docker-self-hosted.md | 4 +-- docs/deploy-helm-from-repo.md | 2 +- docs/deploy-helm.md | 2 +- docs/index.md | 4 +-- docs/mount-ingestor-volume.md | 2 +- docs/nemoretriever-ocr.md | 56 ++++++++++++++--------------- docs/nemotron-parse-extraction.md | 2 +- docs/notebooks.md | 2 +- docs/nv-ingest-standalone.md | 14 ++++---- docs/python-client.md | 6 ++-- docs/readme.md | 2 +- docs/release-notes.md | 8 ++--- docs/service-port-gpu-reference.md | 4 +-- docs/text_only_ingest.md | 8 ++--- docs/troubleshooting.md | 2 +- docs/vlm-embed.md | 2 +- 25 files changed, 95 insertions(+), 81 deletions(-) diff --git a/docs/api-ingestor.md b/docs/api-ingestor.md index 548841574..adeeb4cf0 100644 --- a/docs/api-ingestor.md +++ b/docs/api-ingestor.md @@ -41,7 +41,7 @@ The status response includes progress metrics updated after each batch completes For more granular progress updates during batch processing, use the `nv_ingest_status` object described below, which tracks individual document extraction progress and updates more frequently than the batch-level metrics. ::: -### NV-Ingest Extraction Status +### Extraction status The `/status` endpoint response includes an `nv_ingest_status` object that provides real-time document extraction progress, updating more frequently than batch-level metrics. This is useful for monitoring individual document processing when polling the status endpoint: @@ -53,7 +53,7 @@ The `/status` endpoint response includes an `nv_ingest_status` object that provi | Status | Description | |--------|-------------| | `not_started` | Document queued, extraction not yet initiated | -| `submitted` | Document submitted to NV-Ingest for processing | +| `submitted` | Document submitted to NeMo Retriever Library for processing | | `processing` | Document extraction is in progress | | `completed` | Document extraction completed successfully | | `failed` | Document extraction failed | diff --git a/docs/api-rag.md b/docs/api-rag.md index 518696304..7a15d8890 100644 --- a/docs/api-rag.md +++ b/docs/api-rag.md @@ -10,6 +10,8 @@ This documentation contains the OpenAPI reference for the RAG server. :::{tip} To view this documentation on docs.nvidia.com, browse to [https://docs.nvidia.com/rag/latest/api-rag](https://docs.nvidia.com/rag/latest/api-rag.html). ::: +======= +To view this documentation on docs.nvidia.com, browse to [https://docs.nvidia.com/rag/latest/api-rag](https://docs.nvidia.com/rag/latest/api-rag.html). :::{swagger-plugin} ../docs/api_reference/openapi_schema_rag_server.json diff --git a/docs/audio_ingestion.md b/docs/audio_ingestion.md index 55fcbc132..399ea7a64 100644 --- a/docs/audio_ingestion.md +++ b/docs/audio_ingestion.md @@ -132,7 +132,7 @@ When using Helm deployment, the Audio NIM service requires an additional GPU. The `APP_NVINGEST_SEGMENTAUDIO` environment variable controls whether audio segmentation is enabled during the ingestion process. -When set to `True`, NV-Ingest will segment audio files based on commas and other punctuation marks, resulting in more granular audio chunks. This can improve downstream processing and retrieval accuracy for audio content. Note that splitting on captions will occur regardless of this setting; enabling `APP_NVINGEST_SEGMENTAUDIO` simply adds additional segmentation based on punctuation. +When set to `True`, NeMo Retriever Library will segment audio files based on commas and other punctuation marks, resulting in more granular audio chunks. This can improve downstream processing and retrieval accuracy for audio content. Note that splitting on captions will occur regardless of this setting; enabling `APP_NVINGEST_SEGMENTAUDIO` simply adds additional segmentation based on punctuation. To enable audio segmentation, add the following export command to your environment configuration: diff --git a/docs/change-model.md b/docs/change-model.md index 27aa80e28..96308764f 100644 --- a/docs/change-model.md +++ b/docs/change-model.md @@ -264,7 +264,19 @@ Use this procedure to change models when you are running self-hosted NVIDIA NIM **If only the vLLM profile is available** When only a vLLM profile is available for a model, such as on H100 and RTX GPUs, you must use the vLLM engine. First [run the list-model-profiles command](model-profiles.md#list-available-profiles) to confirm which profiles are available and then apply the following configurations. - + **For Nemotron Nano Models VLLM profile** + + When deploying `nvidia/nvidia-nemotron-nano-9b-v2` or `nvidia/nemotron-3-nano`, check if `tensorrt_llm` profile is available using below command for your required model. + + ```bash + # Change model name as needed + USERID=$(id -u) docker run --rm --gpus all \ + nvcr.io/nim/nvidia/nvidia-nemotron-nano-9b-v2:latest \ + list-model-profiles + ``` + + If only `vllm` profile is available, you must use the **vLLM engine** and add these specific configurations: + ```yaml nimOperator: nim-llm: diff --git a/docs/change-vectordb.md b/docs/change-vectordb.md index a4dc993b8..36f4d4f9f 100644 --- a/docs/change-vectordb.md +++ b/docs/change-vectordb.md @@ -1001,7 +1001,7 @@ Update your [`values.yaml`](../deploy/helm/nvidia-blueprint-rag/values.yaml) fil ### Disable Default Vector Database and Add Custom Helm Chart -1. **Disable Milvus in the NV-Ingest configuration:** +1. **Disable Milvus in the NeMo Retriever Library configuration:** ```yaml nv-ingest: enabled: true diff --git a/docs/conf.py b/docs/conf.py index 6f95103b4..27aeca848 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) '2025-%Y, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,8 +23,8 @@ import os import sys -project = " NVIDIA-RAG-blueprint" -copyright = "2025, NVIDIA Corporation" +project = " NVIDIA RAG blueprint" +copyright = "'2025-%Y, NVIDIA Corporation" author = "NVIDIA Corporation" release = "2.5.0" diff --git a/docs/custom-metadata.md b/docs/custom-metadata.md index c02b80eeb..ce96b63ba 100644 --- a/docs/custom-metadata.md +++ b/docs/custom-metadata.md @@ -232,13 +232,13 @@ The system automatically manages certain metadata fields that are added to all c | Field Name | Type | Description | Auto-Populated | User Override | |------------|------|-------------|----------------|---------------| -| **`filename`** | `string` | Name of the uploaded file (case-sensitive when filtering) | ✅ RAG system | ✅ Yes - define in schema | -| **`page_number`** | `integer` | Page number where content appears (1-indexed) | ✅ nv-ingest | ✅ Yes - define in schema | -| **`start_time`** | `integer` | Start timestamp in milliseconds for audio/video segments | ✅ nv-ingest | ✅ Yes - define in schema | -| **`end_time`** | `integer` | End timestamp in milliseconds for audio/video segments | ✅ nv-ingest | ✅ Yes - define in schema | +| **`filename`** | `string` | Name of the uploaded file | ✅ RAG system | ✅ Yes - define in schema | +| **`page_number`** | `integer` | Page number where content appears (1-indexed) | ✅ NeMo Retriever Library | ✅ Yes - define in schema | +| **`start_time`** | `integer` | Start timestamp in milliseconds for audio/video segments | ✅ NeMo Retriever Library | ✅ Yes - define in schema | +| **`end_time`** | `integer` | End timestamp in milliseconds for audio/video segments | ✅ NeMo Retriever Library | ✅ Yes - define in schema | :::{note} -The following field names are **reserved** by NV-Ingest and cannot be used in custom metadata schemas: `type`, `subtype`, and `location`. These fields are exclusively managed by NV-Ingest during document processing and attempting to use them will result in a validation error. +The following field names are **reserved** by NeMo Retriever Library and cannot be used in custom metadata schemas: `type`, `subtype`, and `location`. These fields are exclusively managed by NeMo Retriever Library during document processing and attempting to use them will result in a validation error. ::: #### System-Managed Field Behavior @@ -246,7 +246,7 @@ The following field names are **reserved** by NV-Ingest and cannot be used in cu - **Auto-Addition**: These fields are automatically added to your collection schema if you don't define them - **Auto-Population**: - `filename` is populated by the RAG system during ingestion - - `page_number`, `start_time`, `end_time` are extracted and populated by nv-ingest during document processing + - `page_number`, `start_time`, `end_time` are extracted and populated by NeMo Retriever Library during document processing - **User Override**: You can define any of these fields in your schema with custom properties (e.g., different description, constraints) - If you provide a definition, your definition takes priority - If you don't provide a definition, the system auto-adds them with default settings @@ -258,7 +258,7 @@ The following field names are **reserved** by NV-Ingest and cannot be used in cu :::{note} **Example**: If you upload a multi-page PDF without defining `page_number` in your schema, the system will: 1. Automatically add the `page_number` field to your collection schema -2. nv-ingest will extract the page number from each chunk during processing +2. NeMo Retriever Library extracts the page number from each chunk during processing 3. The page number will be available for filtering (e.g., `content_metadata["page_number"] == 5`) 4. The page number will appear in citations when generating responses ::: diff --git a/docs/debugging.md b/docs/debugging.md index a9c25debd..1fb5aea0d 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -141,7 +141,7 @@ docker ps | grep -E "(ingestor-server|nv-ingest|nemoretriever-embedding|milvus|r # Check ingestor server health with all dependencies curl -X GET "http://localhost:8082/v1/health?check_dependencies=true" | jq -# Verify NV-Ingest runtime is ready for processing +# Verify NeMo Retriever Library runtime is ready for processing curl -X GET "http://localhost:7670/v1/health/ready" # Check embedding service is responding @@ -219,7 +219,7 @@ Start by examining the logs of key ingestion services to identify the specific e # Check ingestor server logs for API errors docker logs ingestor-server --tail 100 -# Check NV-Ingest runtime logs for processing errors +# Check NeMo Retriever Library runtime logs for processing errors docker logs nv-ingest-ms-runtime --tail 100 # Check embedding service logs for model issues @@ -251,9 +251,9 @@ docker logs nemotron-embedding-ms --tail 100 nvidia-smi ``` -**NV-Ingest Processing Errors:** +**NeMo Retriever Library Processing Errors:** ```bash -# Check NV-Ingest logs for processing errors +# Check NeMo Retriever Library logs for processing errors docker logs nv-ingest-ms-runtime --tail 200 | grep -i error # Check Redis connectivity for task queue diff --git a/docs/deploy-docker-nvidia-hosted.md b/docs/deploy-docker-nvidia-hosted.md index 4487edff3..2aabd06ce 100644 --- a/docs/deploy-docker-nvidia-hosted.md +++ b/docs/deploy-docker-nvidia-hosted.md @@ -111,7 +111,7 @@ Use the following procedure to start all containers needed for this blueprint. ], "processing": [ { - "service": "NV-Ingest", + "service": "NeMo Retriever Library", "status": "healthy", ... } @@ -238,7 +238,7 @@ After the first time you deploy the RAG Blueprint successfully, you can consider - If you don't have a GPU available, you can switch to CPU-only Milvus by following the instructions in [milvus-configuration.md](./milvus-configuration.md). -- If you have a requirement to build the NVIDIA Ingest runtime container from source, you can do it by following instructions [here](https://github.com/NVIDIA/nv-ingest). +- If you have a requirement to build the NeMo Retriever Library runtime container from source, you can do it by following instructions [here](https://github.com/NVIDIA/NeMo-Retriever). diff --git a/docs/deploy-docker-self-hosted.md b/docs/deploy-docker-self-hosted.md index 6607aac73..4913be36a 100644 --- a/docs/deploy-docker-self-hosted.md +++ b/docs/deploy-docker-self-hosted.md @@ -174,7 +174,7 @@ Use the following procedure to start all containers needed for this blueprint. ], "processing": [ { - "service": "NV-Ingest", + "service": "NeMo Retriever Library", "status": "healthy", ... } @@ -333,7 +333,7 @@ After the first time you deploy the RAG Blueprint successfully, you can consider - For improved accuracy, consider enabling reasoning mode. For details, refer to [Enable thinking](./enable-nemotron-thinking.md). -- NeMo Retriever OCR is now the default OCR service. To use legacy Paddle OCR instead, refer to [OCR Configuration Guide](nemoretriever-ocr.md). +- NeMo Retriever Library OCR is now the default OCR service. To use legacy Paddle OCR instead, refer to [OCR Configuration Guide](nemoretriever-ocr.md). - For advanced users who need direct filesystem access to extraction results, refer to [Ingestor Server Volume Mounting](mount-ingestor-volume.md). diff --git a/docs/deploy-helm-from-repo.md b/docs/deploy-helm-from-repo.md index 04c39829a..e57c9ea26 100644 --- a/docs/deploy-helm-from-repo.md +++ b/docs/deploy-helm-from-repo.md @@ -14,7 +14,7 @@ The following are the core services that you install: - RAG server - Ingestor server -- NV-Ingest +- NeMo Retriever Library ## Prerequisites diff --git a/docs/deploy-helm.md b/docs/deploy-helm.md index 9e7a39217..c0b8743e8 100644 --- a/docs/deploy-helm.md +++ b/docs/deploy-helm.md @@ -14,7 +14,7 @@ The following are the core services that you install: - RAG server - Ingestor server -- NV-Ingest +- NeMo Retriever Library ## Prerequisites diff --git a/docs/index.md b/docs/index.md index 941b2f9d8..1e62202a6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -141,7 +141,7 @@ After you deploy the RAG blueprint, you can customize it for your use cases. ## Blog Posts -- [NVIDIA NeMo Retriever Delivers Accurate Multimodal PDF Data Extraction 15x Faster](https://developer.nvidia.com/blog/nvidia-nemo-retriever-delivers-accurate-multimodal-pdf-data-extraction-15x-faster/) +- [NVIDIA NeMo Retriever Library Delivers Accurate Multimodal PDF Data Extraction 15x Faster](https://developer.nvidia.com/blog/nvidia-nemo-retriever-delivers-accurate-multimodal-pdf-data-extraction-15x-faster/) - [Finding the Best Chunking Strategy for Accurate AI Responses](https://developer.nvidia.com/blog/finding-the-best-chunking-strategy-for-accurate-ai-responses/) @@ -217,7 +217,7 @@ After you deploy the RAG blueprint, you can customize it for your use cases. Multimodal Embedding Support (Early Access) OCR Configuration Guide Enhanced PDF Extraction - Standalone NV-Ingest + Standalone NeMo Retriever Library Text-Only Ingestion MCP Server Usage ``` diff --git a/docs/mount-ingestor-volume.md b/docs/mount-ingestor-volume.md index a9bbb43c4..ff776e34d 100644 --- a/docs/mount-ingestor-volume.md +++ b/docs/mount-ingestor-volume.md @@ -4,7 +4,7 @@ --> # Ingestor Server Volume Mounting for NVIDIA RAG Blueprint -You can mount a host directory to access NV-Ingest extraction results directly from the filesystem when you use the [NVIDIA RAG Blueprint](readme.md). Designed for advanced developers who need programmatic access to raw extraction results for custom processing pipelines or external vector database integration. +You can mount a host directory to access extraction results from NeMo Retriever Library directly from the filesystem when you use the [NVIDIA RAG Blueprint](readme.md). Designed for advanced developers who need programmatic access to raw extraction results for custom processing pipelines or external vector database integration. ## Configuration diff --git a/docs/nemoretriever-ocr.md b/docs/nemoretriever-ocr.md index 1893da0aa..a76a7c113 100644 --- a/docs/nemoretriever-ocr.md +++ b/docs/nemoretriever-ocr.md @@ -11,17 +11,17 @@ This guide explains the OCR (Optical Character Recognition) services available i The NVIDIA RAG Blueprint supports two OCR services: -1. **NeMo Retriever OCR** (Default) - High-performance OCR service offering 2x+ faster performance +1. **NeMo Retriever Library OCR** (Default) - High-performance OCR service offering 2x+ faster performance 2. **Paddle OCR** (Legacy) - General-purpose OCR service maintained for compatibility :::{tip} -**NeMo Retriever OCR is now the default OCR service** and is recommended for all new deployments due to its superior performance and efficiency. +**NeMo Retriever Library OCR is now the default OCR service** and is recommended for all new deployments due to its superior performance and efficiency. ::: -## NeMo Retriever OCR (Default) +## NeMo Retriever Library OCR (Default) -NeMo Retriever OCR is the default and recommended OCR service for the NVIDIA RAG Blueprint, providing: +NeMo Retriever Library OCR is the default and recommended OCR service for the NVIDIA RAG Blueprint, providing: - **2x+ faster performance** compared to Paddle OCR - Optimized text extraction from documents and images @@ -38,7 +38,7 @@ NeMo Retriever OCR is the default and recommended OCR service for the NVIDIA RAG ### Default Configuration -By default, the NVIDIA RAG Blueprint is configured to use NeMo Retriever OCR with the following settings: +By default, the NVIDIA RAG Blueprint is configured to use NeMo Retriever Library OCR with the following settings: | Variable | Default Value | Description | |----------|---------------|-------------| @@ -49,11 +49,11 @@ By default, the NVIDIA RAG Blueprint is configured to use NeMo Retriever OCR wit ### Hardware Requirements -For detailed hardware requirements and GPU support, refer to the [NeMo Retriever OCR Support Matrix](https://docs.nvidia.com/nim/ingestion/image-ocr/1.2.1/support-matrix.html). +For detailed hardware requirements and GPU support, refer to the [NeMo Retriever Library OCR Support Matrix](https://docs.nvidia.com/nim/ingestion/image-ocr/1.2.0/support-matrix.html). ### Docker Configuration -The NeMo Retriever OCR service is configured in the Docker Compose file with the following key settings: +The NeMo Retriever Library OCR service is configured in the Docker Compose file with the following key settings: - **Image**: `nvcr.io/nim/nvidia/nemoretriever-ocr-v1:1.2.0` - **GPU Memory**: 8192 MB (default) @@ -72,7 +72,7 @@ export OCR_OMP_NUM_THREADS=8 # Set OpenMP threads ## Paddle OCR (Legacy) -Paddle OCR is maintained as a legacy option for compatibility with existing workflows. While still functional, it is recommended to migrate to NeMo Retriever OCR for better performance. +Paddle OCR is maintained as a legacy option for compatibility with existing workflows. While still functional, it is recommended to migrate to NeMo Retriever Library OCR for better performance. ### When to Use Paddle OCR @@ -92,7 +92,7 @@ The Paddle OCR service configuration: - **Ports**: 8009 (HTTP), 8010 (gRPC), 8011 (Metrics) :::{note} -**Legacy Service**: Paddle OCR is maintained as a legacy option. For new deployments, we recommend using the default NeMo Retriever OCR service for better performance. +**Legacy Service**: Paddle OCR is maintained as a legacy option. For new deployments, we recommend using the default NeMo Retriever Library OCR service for better performance. ::: @@ -100,9 +100,9 @@ The Paddle OCR service configuration: ### Docker Compose Deployment -#### Using NeMo Retriever OCR (Default) +#### Using NeMo Retriever Library OCR (Default) -NeMo Retriever OCR is deployed by default when you follow the standard deployment guide. No additional configuration is required. +NeMo Retriever Library OCR is deployed by default when you follow the standard deployment guide. No additional configuration is required. 1. **Prerequisites**: Follow the [deployment guide](deploy-docker-self-hosted.md) for standard setup. @@ -112,7 +112,7 @@ NeMo Retriever OCR is deployed by default when you follow the standard deploymen ``` :::{tip} - NeMo Retriever OCR is included in the default profile and will start automatically. + NeMo Retriever Library OCR is included in the default profile and will start automatically. ::: 3. **Verify Service Status**: @@ -134,7 +134,7 @@ If you need to use Paddle OCR instead: export OCR_MODEL_NAME=paddle ``` -3. **Stop NeMo Retriever OCR if running**: +3. **Stop NeMo Retriever Library OCR if running**: ```bash USERID=$(id -u) docker compose -f deploy/compose/nims.yaml down nemoretriever-ocr ``` @@ -144,7 +144,7 @@ If you need to use Paddle OCR instead: USERID=$(id -u) docker compose -f deploy/compose/nims.yaml --profile paddle up -d ``` -5. **Restart Ingestor Server and NV-Ingest Runtime**: +5. **Restart Ingestor Server and NeMo Retriever Library Runtime**: ```bash docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d ``` @@ -154,9 +154,9 @@ If you need to use Paddle OCR instead: ### NVIDIA-Hosted Deployment -#### Using NeMo Retriever OCR (Default) +#### Using NeMo Retriever Library OCR (Default) -Follow the standard [NVIDIA-hosted deployment guide](deploy-docker-nvidia-hosted.md) - NeMo Retriever OCR is the default configuration. +Follow the standard [NVIDIA-hosted deployment guide](deploy-docker-nvidia-hosted.md) - NeMo Retriever Library OCR is the default configuration. #### Using Paddle OCR with NVIDIA-Hosted Deployment @@ -176,13 +176,13 @@ Follow the standard [NVIDIA-hosted deployment guide](deploy-docker-nvidia-hosted ### Helm Deployment -#### Using NeMo Retriever OCR (Default) +#### Using NeMo Retriever Library OCR (Default) -NeMo Retriever OCR is deployed by default with Helm installations. Follow the standard [Helm Deployment Guide](deploy-helm.md) - no additional OCR configuration is required. +NeMo Retriever Library OCR is deployed by default with Helm installations. Follow the standard [Helm Deployment Guide](deploy-helm.md) - no additional OCR configuration is required. #### Using Paddle OCR with Helm -To use Paddle OCR instead of the default NeMo Retriever OCR: +To use Paddle OCR instead of the default NeMo Retriever Library OCR: Modify [`values.yaml`](../deploy/helm/nvidia-blueprint-rag/values.yaml) to override the OCR service image: @@ -214,7 +214,7 @@ For detailed Helm deployment instructions, see [Helm Deployment Guide](deploy-he ### Environment Variables -| Variable | Description | NeMo Retriever Default | Paddle Default | Required | +| Variable | Description | NeMo Retriever Library Default | Paddle Default | Required | |----------|-------------|------------------------|----------------|----------| | `OCR_GRPC_ENDPOINT` | gRPC endpoint for OCR service | `nemoretriever-ocr:8001` | `paddle:8001` | Yes (on-premises) | | `OCR_HTTP_ENDPOINT` | HTTP endpoint for OCR service | `http://nemoretriever-ocr:8000/v1/infer` | `http://paddle:8000/v1/infer` | Yes | @@ -238,16 +238,16 @@ Replace `workstation_ip` with the actual IP address of the machine running the O ## Switching Between OCR Services -### Migrating from Paddle OCR to NeMo Retriever OCR +### Migrating from Paddle OCR to NeMo Retriever Library OCR -To switch to the default NeMo Retriever OCR service: +To switch to the default NeMo Retriever Library OCR service: 1. **Stop Paddle OCR**: ```bash USERID=$(id -u) docker compose -f deploy/compose/nims.yaml down paddle ``` -2. **Configure NeMo Retriever OCR environment variables**: +2. **Configure NeMo Retriever Library OCR environment variables**: ```bash export OCR_GRPC_ENDPOINT=nemoretriever-ocr:8001 export OCR_HTTP_ENDPOINT=http://nemoretriever-ocr:8000/v1/infer @@ -255,7 +255,7 @@ To switch to the default NeMo Retriever OCR service: export OCR_MODEL_NAME=scene_text_ensemble ``` -3. **Start NeMo Retriever OCR**: +3. **Start NeMo Retriever Library OCR**: ```bash USERID=$(id -u) docker compose -f deploy/compose/nims.yaml up -d nemoretriever-ocr ``` @@ -265,14 +265,14 @@ To switch to the default NeMo Retriever OCR service: docker compose -f deploy/compose/docker-compose-ingestor-server.yaml up -d ``` -### Migrating from NeMo Retriever OCR to Paddle OCR +### Migrating from NeMo Retriever Library OCR to Paddle OCR Follow the steps in [Switching to Paddle OCR](#switching-to-paddle-ocr) above. ## Performance Comparison -| Feature | NeMo Retriever OCR | Paddle OCR | +| Feature | NeMo Retriever Library OCR | Paddle OCR | |---------|-------------------|------------| | **Performance** | 2x+ faster | Baseline | | **GPU Memory** | 8 GB (default) | 3 GB (default) | @@ -297,13 +297,13 @@ Follow the steps in [Switching to Paddle OCR](#switching-to-paddle-ocr) above. 3. **Performance Issues** - Consider increasing `OCR_CUDA_MEMORY_POOL_MB` - - Adjust `OCR_BATCH_SIZE` for NeMo Retriever OCR + - Adjust `OCR_BATCH_SIZE` for NeMo Retriever Library OCR - Verify GPU has sufficient memory ### Getting Logs ```bash -# NeMo Retriever OCR logs +# NeMo Retriever Library OCR logs docker logs nemoretriever-ocr # Paddle OCR logs diff --git a/docs/nemotron-parse-extraction.md b/docs/nemotron-parse-extraction.md index c15cacf3f..0e2fc0b11 100644 --- a/docs/nemotron-parse-extraction.md +++ b/docs/nemotron-parse-extraction.md @@ -232,7 +232,7 @@ When using Nemotron Parse for PDF extraction, consider the following: - The extraction quality may vary depending on the PDF structure and content. - Nemotron Parse is not supported on NVIDIA B200 GPUs or RTX Pro 6000 GPUs. -For detailed information about hardware requirements and supported GPUs for all NeMo Retriever extraction NIMs, refer to the [Nemotron Parse Support Matrix](https://docs.nvidia.com/nim/vision-language-models/latest/support-matrix.html#nemotron-parse). +For detailed information about hardware requirements and supported GPUs for extraction NIMs used by NeMo Retriever Library, refer to the [Nemotron Parse Support Matrix](https://docs.nvidia.com/nim/vision-language-models/latest/support-matrix.html#nemotron-parse). ## Available PDF Extraction Methods diff --git a/docs/notebooks.md b/docs/notebooks.md index 6146e4abf..beff3dc94 100644 --- a/docs/notebooks.md +++ b/docs/notebooks.md @@ -101,7 +101,7 @@ Use the following notebooks to learn comprehensive Python client usage, metadata - [rag_library_usage.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/rag_library_usage.ipynb) – Demonstrates native usage of the NVIDIA RAG Python client, including environment setup, document ingestion, collection management, and querying. This notebook provides end-to-end API usage examples for interacting directly with the RAG system from Python, covering both ingestion and retrieval workflows. -- [rag_library_lite_usage.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/rag_library_lite_usage.ipynb) – Demonstrates containerless deployment of the NVIDIA RAG Python package in lite mode. Uses Milvus Lite (embedded vector database) and NV-Ingest subprocess mode for a simplified setup without Docker containers. Leverages NVIDIA cloud APIs for embeddings, ranking, and LLM inference. **Note**: This mode does not support image/table/chart citations or document summarization. +- [rag_library_lite_usage.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/rag_library_lite_usage.ipynb) – Demonstrates containerless deployment of the NVIDIA RAG Python package in lite mode. Uses Milvus Lite (embedded vector database) and NeMo Retriever Library subprocess mode for a simplified setup without Docker containers. Leverages NVIDIA cloud APIs for embeddings, ranking, and LLM inference. **Note**: This mode does not support image/table/chart citations or document summarization. - [langchain_nvidia_retriever.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/langchain_nvidia_retriever.ipynb) – Showcases **LangChain integration** with the NVIDIA RAG Blueprint. Run [ingestion_api_usage.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/ingestion_api_usage.ipynb) first to ingest documents, then use `NVIDIARAGRetriever` for retrieval (sync/async), custom parameters, error handling, and optional RAG chaining with `ChatNVIDIA`. diff --git a/docs/nv-ingest-standalone.md b/docs/nv-ingest-standalone.md index 691a23607..14319ad94 100644 --- a/docs/nv-ingest-standalone.md +++ b/docs/nv-ingest-standalone.md @@ -3,19 +3,19 @@ SPDX-License-Identifier: Apache-2.0 --> -# Deploy NV-Ingest Standalone for NVIDIA RAG Blueprint +# Deploy NeMo Retriever Library Standalone for NVIDIA RAG Blueprint -This guide explains how to deploy and use NV-Ingest as a standalone service for [NVIDIA RAG Blueprint](readme.md) without deploying the full ingestor server. This is useful when you want to ingest documents directly using Python scripts. +This guide explains how to deploy and use NeMo Retriever Library as a standalone service for [NVIDIA RAG Blueprint](readme.md) without deploying the full ingestor server. This is useful when you want to ingest documents directly using Python scripts. For more details and advanced usage, refer to: -- [NVIDIA/nv-ingest repository](https://github.com/NVIDIA/nv-ingest) -- [Official NV-Ingest Quickstart Guide](https://github.com/NVIDIA/nv-ingest/blob/main/docs/docs/extraction/quickstart-guide.md) +- [NVIDIA/NeMo-Retriever Library repository](https://github.com/NVIDIA/NeMo-Retriever) +- [Official NeMo Retriever Library Quickstart Guide](https://docs.nvidia.com/nemo/retriever/) ## Limitations -When using NV-Ingest in standalone mode, consider the following limitations: +When using NeMo Retriever Library in standalone mode, consider the following limitations: -1. **Citations Disabled**: The RAG server's citation feature will be disabled for documents ingested through standalone NV-Ingest. This is because the citation metadata requires additional processing that is handled by the full ingestor server. +1. **Citations Disabled**: The RAG server's citation feature will be disabled for documents ingested through standalone NeMo Retriever Library. This is because the citation metadata requires additional processing that is handled by the full ingestor server. 2. **No Web UI**: The standalone deployment does not include the web-based upload interface. All document ingestion must be done through Python scripts. @@ -92,7 +92,7 @@ COLLECTION_NAME = "multimodal_data_nvingest" MILVUS_URI = "http://localhost:19530" MINIO_ENDPOINT = "localhost:9010" -# Server Mode (Create NV-Ingest client) +# Server Mode (Create NeMo Retriever Library client) client = NvIngestClient( message_client_hostname="localhost", message_client_port=7670 diff --git a/docs/python-client.md b/docs/python-client.md index e052b5e91..5c9bc33ea 100644 --- a/docs/python-client.md +++ b/docs/python-client.md @@ -170,7 +170,7 @@ nim-llm-ms Up ... (healthy) `DEPLOYMENT_MODE = "cloud"` -2. Configure NV-Ingest to use NVIDIA hosted cloud APIs using the following hosted models. +2. Configure NeMo Retriever Library to use NVIDIA hosted cloud APIs using the following hosted models. - os.environ["OCR_HTTP_ENDPOINT"] = "https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr" @@ -193,9 +193,9 @@ os.environ["YOLOX_HTTP_ENDPOINT"] = ( os.environ["YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL"] = "http" -### Setup NVIDIA Ingest Runtime and Redis Service +### Setup NeMo Retriever Library Runtime and Redis Service -Use the following command to setup your NVIDIA Ingest Runtime and Redis Service. +Use the following command to setup your NeMo Retriever Library Runtime and Redis Service. `docker compose -f ../deploy/compose/docker-compose-ingestor-server.yaml up nv-ingest-ms-runtime redis -d` diff --git a/docs/readme.md b/docs/readme.md index ec8c5cf8e..0248a4a00 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -147,5 +147,5 @@ After you deploy the RAG blueprint, you can customize it for your use cases. ## Blog Posts -- [NVIDIA NeMo Retriever Delivers Accurate Multimodal PDF Data Extraction 15x Faster](https://developer.nvidia.com/blog/nvidia-nemo-retriever-delivers-accurate-multimodal-pdf-data-extraction-15x-faster/) +- [NVIDIA NeMo Retriever Library Delivers Accurate Multimodal PDF Data Extraction 15x Faster](https://developer.nvidia.com/blog/nvidia-nemo-retriever-delivers-accurate-multimodal-pdf-data-extraction-15x-faster/) - [Finding the Best Chunking Strategy for Accurate AI Responses](https://developer.nvidia.com/blog/finding-the-best-chunking-strategy-for-accurate-ai-responses/) diff --git a/docs/release-notes.md b/docs/release-notes.md index 2a3550fad..5a3f1ccc3 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -48,9 +48,9 @@ This release adds new features to the RAG pipeline for supporting agent workflow This release contains the following key changes: -- Updated NIMs and code to support [NVIDIA Ingest 26.01 release](https://docs.nvidia.com/nemo/retriever/latest/extraction/releasenotes-nv-ingest/). +- Updated NIMs and code to support [NeMo Retriever Library 26.01 release](https://docs.nvidia.com/nemo/retriever/latest/extraction/releasenotes-nv-ingest/). - Added support for non-NIM models including OpenAI, models hosted on AWS and Azure, OSS models, and others. Supported through service-specific API keys. For details, refer to [Get an API Key](api-key.md). -- The RAG Blueprint now uses [nemoretriever-ocr-v1](https://build.nvidia.com/nvidia/nemoretriever-ocr-v1/modelcard) as the default OCR model. For details, refer to [NeMo Retriever OCR Configuration Guide](nemoretriever-ocr.md). +- The RAG Blueprint now uses [nemoretriever-ocr-v1](https://build.nvidia.com/nvidia/nemoretriever-ocr-v1/modelcard) as the default OCR model. For details, refer to [NeMo Retriever Library OCR Configuration Guide](nemoretriever-ocr.md). - Improved VLM based generation support. The Vision-Language Model (VLM) inference feature now uses the model [nemotron-nano-12b-v2-vl](https://build.nvidia.com/nvidia/nemotron-nano-12b-v2-vl/modelcard). For details, refer to [VLM for Generation](vlm.md). - User interface improvements including catalog display, image and text query, and others. For details, refer to [User Interface](user-interface.md). - Added ingestion metrics endpoint support with OpenTelemetry (OTEL) for monitoring document uploads, elements ingested, and pages processed. For details, refer to [Observability](observability.md). @@ -72,7 +72,7 @@ This release contains the following key changes: - Shallow summarization support - Easy model switches and dedicated configurations - Ease of prompt changes -- Reserved field names `type`, `subtype`, and `location` for NV-Ingest exclusive use in metadata schemas. +- Reserved field names `type`, `subtype`, and `location` for NeMo Retriever Library exclusive use in metadata schemas. - Added support for [rag_library_lite_usage.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/rag_library_lite_usage.ipynb) which demonstrates containerless deployment of the NVIDIA RAG Python package in lite mode. - Added example showcasing [NeMo Agent Toolkit integration](https://github.com/NVIDIA/NeMo-Agent-Toolkit) with NVIDIA RAG. - Added [weighted hybrid search](hybrid_search.md#weighted-hybrid-search) support with configurable weights. @@ -109,7 +109,7 @@ The following are the known issues for the NVIDIA RAG Blueprint: - Optional features reflection and image captioning are not available in Helm-based deployment. - Currently, Helm-based deployment is not supported for [NeMo Guardrails](nemo-guardrails.md). - The Blueprint responses can have significant latency when using [NVIDIA API Catalog cloud hosted models](deploy-docker-nvidia-hosted.md). -- The accuracy of the pipeline is optimized for certain file types like `.pdf`, `.txt`, `.docx`. The accuracy may be poor for other file types supported by NV-Ingest, since image captioning is disabled by default. +- The accuracy of the pipeline is optimized for certain file types like `.pdf`, `.txt`, `.docx`. The accuracy may be poor for other file types supported by NeMo Retriever Library, since image captioning is disabled by default. - When updating model configurations in Kubernetes `values.yaml` (for example, changing from 70B to 8B models), the RAG UI automatically detects and displays the new model configuration from the backend. No container rebuilds are required - simply redeploy the Helm chart with updated values and refresh the UI to see the new model settings in the Settings panel. - The NeMo LLM microservice can take 5-6 minutes to start for every deployment. - B200 GPUs are not supported for the following advanced features. For these features, use H100 or A100 GPUs instead. diff --git a/docs/service-port-gpu-reference.md b/docs/service-port-gpu-reference.md index f95c9925a..ed24b39f2 100644 --- a/docs/service-port-gpu-reference.md +++ b/docs/service-port-gpu-reference.md @@ -13,7 +13,7 @@ The following table provides a comprehensive reference of all services, their po | RAG Server | `rag-server` | 8081 | 8081 | N/A (CPU) | Main RAG API endpoint | | Ingestor Server | `ingestor-server` | 8082 | 8082 | N/A (CPU) | Document ingestion API | | RAG Frontend | `rag-frontend` | 8090 | 3000 | N/A (CPU) | Web UI | -| NV-Ingest Runtime | `nv-ingest-ms-runtime` | 7670, 7671, 8265 | 7670, 7671, 8265 | N/A (CPU) | Main orchestrator (Ray dashboard: 8265) | +| NeMo Retriever Library Runtime | `nv-ingest-ms-runtime` | 7670, 7671, 8265 | 7670, 7671, 8265 | N/A (CPU) | Main orchestrator (Ray dashboard: 8265) | ## NIM Microservices @@ -29,7 +29,7 @@ The following table provides a comprehensive reference of all services, their po | Page Elements | `compose-page-elements-1` | 8000, 8001, 8002 | 8000, 8001, 8002 | 0 | `YOLOX_MS_GPU_ID` | Object detection for pages | | Graphic Elements | `compose-graphic-elements-1` | 8003, 8004, 8005 | 8000, 8001, 8002 | 0 | `YOLOX_GRAPHICS_MS_GPU_ID` | Graphics detection | | Table Structure | `compose-table-structure-1` | 8006, 8007, 8008 | 8000, 8001, 8002 | 0 | `YOLOX_TABLE_MS_GPU_ID` | Table structure detection | -| NeMo Retriever OCR | `compose-nemoretriever-ocr-1` | 8012, 8013, 8014 | 8000, 8001, 8002 | 0 | `OCR_MS_GPU_ID` | OCR service (default) | +| NeMo Retriever Library OCR | `compose-nemoretriever-ocr-1` | 8012, 8013, 8014 | 8000, 8001, 8002 | 0 | `OCR_MS_GPU_ID` | OCR service (default) | ## Vector Database and Infrastructure diff --git a/docs/text_only_ingest.md b/docs/text_only_ingest.md index 10cb6e9b1..6ba23fd19 100644 --- a/docs/text_only_ingest.md +++ b/docs/text_only_ingest.md @@ -19,7 +19,7 @@ You can enable text-only ingestion for the [NVIDIA RAG Blueprint](readme.md). Fo ``` :::{important} - When disabling nv-ingest dependent services, you must set `COMPONENTS_TO_READY_CHECK=""` to ensure the nv-ingest container reaches ready state. Without this setting, nv-ingest will wait indefinitely for the disabled components. + When disabling NeMo Retriever Library dependent services, you must set `COMPONENTS_TO_READY_CHECK=""` to ensure the NeMo Retriever Library container reaches ready state. Without this setting, the NeMo Retriever Library container will wait indefinitely for the disabled components. ::: Then deploy the ingestor-server: @@ -131,9 +131,9 @@ helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprin ``` :::{important} -**Disabling NV-Ingest Components for GPU Resource Management:** +**Disabling NeMo Retriever Library Components for GPU Resource Management:** -If you disable any nv-ingest dependent services (such as `table_structure`, `graphic_elements`, `nemoretriever_ocr_v1`, etc.) to free up GPU resources for customization, you must set the `COMPONENTS_TO_READY_CHECK` parameter to an empty string in the `nv-ingest.envVars` section of your [values.yaml](../deploy/helm/nvidia-blueprint-rag/values.yaml) file: +If you disable any NeMo Retriever Library dependent services (such as `table_structure`, `graphic_elements`, `nemoretriever_ocr_v1`, etc.) to free up GPU resources for customization, you must set the `COMPONENTS_TO_READY_CHECK` parameter to an empty string in the `nv-ingest.envVars` section of your [values.yaml](../deploy/helm/nvidia-blueprint-rag/values.yaml) file: ```yaml nv-ingest: @@ -141,6 +141,6 @@ nv-ingest: COMPONENTS_TO_READY_CHECK: "" ``` -This ensures the nv-ingest pod reaches ready state even when some dependent components are disabled. Without this setting, the nv-ingest pod will wait indefinitely for the disabled components to become ready. +This ensures the NeMo Retriever Library pod reaches ready state even when some dependent components are disabled. Without this setting, the NeMo Retriever Library pod will wait indefinitely for the disabled components to become ready. ::: diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 485ae84f1..782176ed2 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -340,7 +340,7 @@ If the above error related to dependency conflicts are seen while building conta We've integrated VDB and embedding creation directly into the pipeline with caching included for expediency. However, in a production environment, it's better to use a separately managed VDB service. -NVIDIA offers optimized models and tools like NVIDIA NeMo Retriever ([build.nvidia.com/explore/retrieval](https://build.nvidia.com/explore/retrieval)) +NVIDIA offers optimized models and tools like NVIDIA NeMo Retriever Library ([build.nvidia.com/explore/retrieval](https://build.nvidia.com/explore/retrieval)) and cuVS ([github.com/rapidsai/cuvs](https://github.com/rapidsai/cuvs)). diff --git a/docs/vlm-embed.md b/docs/vlm-embed.md index 5b9913232..6cd5b0304 100644 --- a/docs/vlm-embed.md +++ b/docs/vlm-embed.md @@ -214,7 +214,7 @@ ingestor-server: nv-ingest: envVars: - # NV-Ingest runtime embedding target + # NeMo Retriever Library runtime embedding target EMBEDDING_NIM_ENDPOINT: "http://nemotron-vlm-embedding-ms:8000/v1" EMBEDDING_NIM_MODEL_NAME: "nvidia/llama-nemotron-embed-vl-1b-v2" ``` From 59a43ca300288fe810c43d31bf032f865f2d98c3 Mon Sep 17 00:00:00 2001 From: Shubhadeep Das <149712532+shubhadeepd@users.noreply.github.com> Date: Thu, 12 Mar 2026 15:43:43 +0530 Subject: [PATCH 59/81] Launchable Updates for release 2.5 (#428) (#429) Co-authored-by: rkharwar-nv --- notebooks/launchable.ipynb | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/notebooks/launchable.ipynb b/notebooks/launchable.ipynb index 22e293f90..cccba6eba 100644 --- a/notebooks/launchable.ipynb +++ b/notebooks/launchable.ipynb @@ -127,6 +127,16 @@ "RAG_BASE_URL = f\"http://{IPADDRESS}:{RAG_SERVER_PORT}\"\n", "INGESTOR_BASE_URL = f\"http://{IPADDRESS}:{INGESTOR_SERVER_PORT}\"\n", "\n", + "# NIM services to deploy (excludes nim-llm and vlm-ms since we use NVIDIA-hosted endpoints)\n", + "NIM_SERVICES = (\n", + " \"nemotron-embedding-ms \"\n", + " \"nemotron-ranking-ms \"\n", + " \"page-elements \"\n", + " \"graphic-elements \"\n", + " \"table-structure \"\n", + " \"nemoretriever-ocr\"\n", + ")\n", + "\n", "\n", "# =============================================================================\n", "# DOCKER COMPOSE HELPERS\n", @@ -1295,10 +1305,10 @@ "902445432dde milvus-standalone Up 3 minutes\n", "340bc8210a0d milvus-minio Up 3 minutes (healthy)\n", "0be702b87ad6 milvus-etcd Up 3 minutes (healthy)\n", - "fe2751bfa734 nemotron-ranking-ms Up 10 minutes (healthy)\n", + "fe2751bfa734 nemotron-ranking-ms Up 4 seconds (healthy)\n", "7b5ddabf8be7 compose-graphic-elements-1 Up 10 minutes\n", "ecfaa5190302 compose-page-elements-1 Up 10 minutes\n", - "ea8c7fdf20d1 nemotron-embedding-ms Up 10 minutes (healthy)\n", + "ea8c7fdf20d1 nemotron-embedding-ms Up 4 seconds (healthy)\n", "6d62008a9b42 compose-nemoretriever-ocr-1 Up 10 minutes\n", "969b9f5c987c compose-table-structure-1 Up 10 minutes\n", "```\n", @@ -2056,7 +2066,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.12.13" } }, "nbformat": 4, From e0c51f917dd4eefb8374fd54205be85bea90adbc Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Thu, 12 Mar 2026 09:26:17 -0500 Subject: [PATCH 60/81] added walletless connections --- src/nvidia_rag/utils/vdb/__init__.py | 3 --- src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py | 23 ++++--------------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/src/nvidia_rag/utils/vdb/__init__.py b/src/nvidia_rag/utils/vdb/__init__.py index e4c12f720..fa52b4908 100644 --- a/src/nvidia_rag/utils/vdb/__init__.py +++ b/src/nvidia_rag/utils/vdb/__init__.py @@ -71,9 +71,6 @@ def _get_vdb_op( oracle_user=os.getenv("ORACLE_USER"), oracle_password=os.getenv("ORACLE_PASSWORD"), oracle_cs=os.getenv("ORACLE_CS"), - tnsnames_loc=os.getenv("ORACLE_TNSNAMES_LOC"), - ewallet_pem_loc=os.getenv("ORACLE_EWALLET_PEM_LOC"), - ewallet_password=os.getenv("ORACLE_EWALLET_PASSWORD"), embedding_model=embedding_model, config=config, meta_dataframe=csv_file_path, diff --git a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py index 34d999093..29a3c2bd7 100644 --- a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py +++ b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py @@ -185,9 +185,6 @@ def __init__( oracle_user: str | None = None, oracle_password: str | None = None, oracle_cs: str | None = None, - tnsnames_loc: str | None = None, - ewallet_pem_loc: str | None = None, - ewallet_password: str | None = None, embedding_model: Any | None = None, config: NvidiaRAGConfig | None = None, meta_dataframe: Any | None = None, @@ -224,20 +221,14 @@ def __init__( self._oracle_user = oracle_user or os.getenv("ORACLE_USER") self._oracle_password = oracle_password or os.getenv("ORACLE_PASSWORD") self._oracle_cs = oracle_cs or os.getenv("ORACLE_CS") - self._tnsnames_loc = tnsnames_loc or os.getenv("ORACLE_TNSNAMES_LOC") - self._ewallet_pem_loc = ewallet_pem_loc or os.getenv("ORACLE_EWALLET_PEM_LOC") - self._ewallet_password = ewallet_password or os.getenv("ORACLE_EWALLET_PASSWORD") if not all([ self._oracle_user, self._oracle_password, - self._oracle_cs, - self._tnsnames_loc, - self._ewallet_pem_loc, - self._ewallet_password + self._oracle_cs ]): raise ValueError( - "Oracle connection requires ORACLE_USER, ORACLE_PASSWORD, ORACLE_CS, ORACLE_TNSNAMES_LOC, ORACLE_EWALLET_PEM_LOC and ORACLE_EWALLET_PASSWORD variables." + "Oracle connection requires ORACLE_USER, ORACLE_PASSWORD, ORACLE_CS variables." "Set via parameters or environment variables." ) @@ -261,10 +252,7 @@ def __init__( self._pool = oracledb.create_pool( user=self._oracle_user, password=self._oracle_password, - dsn=self._oracle_cs, - config_dir=self._tnsnames_loc, - wallet_location=self._ewallet_pem_loc, - wallet_password=self._ewallet_password, + dsn=self._oracle_cs min=2, max=10, increment=1, @@ -987,10 +975,7 @@ def get_langchain_vectorstore(self, collection_name: str) -> OracleVS: conn = oracledb.connect( user=self._oracle_user, password=self._oracle_password, - dsn=self._oracle_cs, - config_dir=self._tnsnames_loc, - wallet_location=self._ewallet_pem_loc, - wallet_password=self._ewallet_password + dsn=self._oracle_cs ) return OracleVSCompat( From a10fc5369d39e09e0154ca2a5a7fe25342d863f5 Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Thu, 12 Mar 2026 11:17:07 -0500 Subject: [PATCH 61/81] vresion for main, hash for pr --- .github/workflows/build-push-main.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build-push-main.yml b/.github/workflows/build-push-main.yml index ba0b6e414..4600a51ac 100644 --- a/.github/workflows/build-push-main.yml +++ b/.github/workflows/build-push-main.yml @@ -33,10 +33,6 @@ jobs: with: auth_token: ${{ secrets.OCI_AUTH_TOKEN }} - - name: Compute short SHA - id: short-sha - run: echo "value=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT - - name: Read version id: version run: echo "value=$(cat VERSION)" >> $GITHUB_OUTPUT From 5ff5707eaff81f436071a4974886e1467731e19b Mon Sep 17 00:00:00 2001 From: Robert Riley Date: Thu, 12 Mar 2026 14:53:29 -0500 Subject: [PATCH 62/81] missing a solo comma --- src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py index 29a3c2bd7..63566233b 100644 --- a/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py +++ b/src/nvidia_rag/utils/vdb/oracle/oracle_vdb.py @@ -252,7 +252,7 @@ def __init__( self._pool = oracledb.create_pool( user=self._oracle_user, password=self._oracle_password, - dsn=self._oracle_cs + dsn=self._oracle_cs, min=2, max=10, increment=1, From 1c544b2ecffabe7f607bd1b9256b4d9b242c883e Mon Sep 17 00:00:00 2001 From: Kurt Heiss Date: Thu, 12 Mar 2026 22:15:42 -0700 Subject: [PATCH 63/81] Kheiss/cont ingest (#431) * confirming presence of switcher text in conf.py file * Continuous ingestion topic * updated index for continuous ingestion * updated reademe for continuous ingestion * Update continuous-ingestion-object-storage.md added RAG Blueprint * Update continuous-ingestion-object-storage.md Converted first sentence into 2 sentences * Update index.md * Apply suggestion from @nkmcalli Co-authored-by: nkmcalli * Update continuous-ingestion-object-storage.md * Update continuous-ingestion-object-storage.md * Update continuous-ingestion-object-storage.md --------- Co-authored-by: nkmcalli --- docs/continuous-ingestion-object-storage.md | 121 ++++++++++++++++++++ docs/index.md | 2 + 2 files changed, 123 insertions(+) create mode 100644 docs/continuous-ingestion-object-storage.md diff --git a/docs/continuous-ingestion-object-storage.md b/docs/continuous-ingestion-object-storage.md new file mode 100644 index 000000000..a718224d9 --- /dev/null +++ b/docs/continuous-ingestion-object-storage.md @@ -0,0 +1,121 @@ + +# Continuous Ingestion from Object Storage RAG Blueprint + +Continuous ingestion from object storage connects the [RAG blueprint](readme.md) to continuous integration. This enables an event-driven pipeline that automatically indexes documents and, optionally, video files. Continuous integration means that when you add files to a storage bucket, the system detects new uploads, routes them for processing, and indexes their content—making all data immediately searchable and available for analysis through the [RAG Frontend](user-interface.md). + +## Overview + +You can create an event-driven continuous ingestion pipeline that works as follows: + +1. Upload files (documents or, optionally, videos) to object storage. + +2. The system detects new uploads via storage events and routes them for processing. + +3. Content is automatically indexed into the RAG vector store. + +4. You can then query the ingested content through the RAG UI or API. + +Continuous ingestion supports the following content types: + +- **Documents:** PDF, DOCX (and other formats supported by the [ingestor](api-ingestor.md)). +- **Video (optional/future):** MP4, MKV, AVI — typically processed by a Video Summary Service (VSS) and then ingested as documents. + +## Architecture + +The continuous ingestion architecture features the following high-level flow: + +1. Object storage: Files are written to storage using a protocol that emits events (for example, MinIO configured with Kafka notifications). + +2. Event trigger: Upload events are published to a Kafka topic. + +3. Consumer: A Kafka consumer subscribes to the topic, retrieves the events, downloads the corresponding files from object storage, and routes them for processing. + +4. Document path: Files are passed to a file-based processing pipeline (such as the NeMo Retriever Library or ingestor-server) and then indexed in the vector database. + +5. Video path (optional): For video files, the consumer submits a processing request to the Video Search Service (VSS), polls for results, uploads the processed outputs to RAG as documents, and continues through the standard file-based ingestion flow. + + +The continuous ingestion architecture follows the end-to-end sequence described above and can be summarized as: + +- Document ingestion flow: (1) → (2) → (3) → file-based processing → VectorDB → RAG Agent. + +- Video ingestion flow: (1) → (2) → (3) → VSS request → poll for results → upload to RAG as a document → file-based processing → VectorDB → RAG Agent. + +## Implementation Components + +The reference implementation includes the following components: + +- Object storage (MinIO): A bucket configured with Kafka notifications on put (and optionally delete) events. + +- Kafka: A broker and topic (for example, aidp-topic) used to publish storage event notifications. + +- Kafka consumer: A service that: + +-- Subscribes to the Kafka topic and consumes storage events. + +-- Downloads new objects from MinIO and routes them based on type (document or video). + +-- For documents: Sends files to the RAG ingestor for indexing. + +-- For videos (optional): Submits files to the Video Search Service (VSS), then ingests VSS outputs as documents. + +The deployment is defined in `examples/rag_event_ingest/deploy/docker-compose.yaml`, which runs MinIO, Kafka, and the Kafka consumer on the same Docker network as the RAG stack (`nvidia-rag`). + +### Prerequisites + +- [Deploy the NVIDIA RAG Blueprint](deploy-docker-self-hosted.md) (NIMs, Milvus, ingestor-server, RAG server) so the consumer can reach the ingestor and the rest of the stack. +- Ensure the `nvidia-rag` Docker network exists (created by the RAG deployment). +- For the notebook, clone the repo, set `NGC_API_KEY`, and have the required hardware (see notebook for GPU and software requirements). + +### Option 1: Use the Notebook + +The notebook provides a guided walkthrough of the following steps: + +- Environment setup +- NVIDIA RAG deployment +- (Optional) NVIDIA VSS deployment for video processing +- Continuous ingestion pipeline deployment (Kafka, MinIO, and consumer) +- Testing document and optional video uploads with RAG queries +- Cleanup + +To follow along, open and run: [rag_event_ingest.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/rag_event_ingest.ipynb). + +### Option 2: Deploy the Example with Docker Compose + +From the repository root, after the RAG stack is up: + +```bash +docker compose -f examples/rag_event_ingest/deploy/docker-compose.yaml up -d +``` + +This command launches the following components: + +- Kafka (with an optional Kafka UI available on port 8080) +- MinIO (object storage and console using ports 9201 and 9211 in the example) +- Kafka consumer — connects to the ingestor at `INGESTOR_SERVER_URL` (default: `http://ingestor-server:8082`) and uses `COLLECTION_NAME` (default: `aidp_bucket`) + +After deployment, upload files to the MinIO bucket (for example, `aidp-bucket`). The system publishes upload events to Kafka, the consumer retrieves the corresponding files, and documents are sent to the ingestor for indexing. You can then query the same collection using the RAG UI or API. + +### Key Environment Variables + +The following environment variables configure the Kafka consumer. For details, refer to `examples/rag_event_ingest/deploy/docker-compose.yaml`. + +Consumer environment variables + +| Variable | Description | Default Value| +|----------|---------|--------| +| `KAFKA_BOOTSTRAP_SERVERS` | Address of the Kafka broker(s). | `kafka:9092` | +| `KAFKA_TOPIC` |Kafka topic used for object storage events. | `aidp-topic` | +| `MINIO_ENDPOINT` | MinIO endpoint in : format. | `minio-source-1:9000` | +| `INGESTOR_SERVER_URL` | Base URL for the RAG ingestor service. | `http://ingestor-server:8082` | +| `COLLECTION_NAME` | Target RAG collection for content indexing. | `aidp_bucket` | + +## Reference + +- [RAG Blueprint deployment (Docker self-hosted)](deploy-docker-self-hosted.md) +- [Ingestor API](api-ingestor.md) +- [Notebook: Document continuous ingestion from object storage](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/rag_event_ingest.ipynb) +- [Example: `examples/rag_event_ingest/`](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/examples/rag_event_ingest/) — Kafka consumer and `deploy/docker-compose.yaml` diff --git a/docs/index.md b/docs/index.md index 1e62202a6..d3c1a3ec9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -81,6 +81,7 @@ After you deploy the RAG blueprint, you can customize it for your use cases. - Data Ingestion & Processing - [Audio Ingestion Support](audio_ingestion.md) + - [Continuous Ingestion from Object Storage](continuous-ingestion-object-storage.md) - [Custom Metadata Support](custom-metadata.md) - [File System Access to Extraction Results](mount-ingestor-volume.md) - [Multimodal Embedding Support (Early Access)](vlm-embed.md) @@ -211,6 +212,7 @@ After you deploy the RAG blueprint, you can customize it for your use cases. :hidden: Audio Ingestion Support + Continuous Ingestion from Object Storage Custom metadata Support Data Catalog for Collections and Documents File System Access to Results From 0f6c99059ab98921a3487a719424640af32961f0 Mon Sep 17 00:00:00 2001 From: nv-pranjald <150428320+nv-pranjald@users.noreply.github.com> Date: Fri, 13 Mar 2026 11:07:00 +0530 Subject: [PATCH 64/81] Nemotron 3 super deployment guide and migration guide (#430) * Nemotron 3 super deployment guide and migration guide * Organize gpu requirement and heading for nemotron3 super * Add instruction for updating values.yaml and refractor doc * Add cloud endpoint url in env file * Instruction to export llm max token in docker flow * remove unrequired llm api key from doc * Seprate yaml for nemotron 3 deployment * Remove unnecessary information for local hosted * Simplify docker deployment logs * Remove cuda device from rtx 6000 pro * Add prompt customization instruction in nemotron3 helm section * Instruction for prompt customization * Remove heading for rtx 6000 pro --- deploy/compose/nemotron3-super-cloud.env | 49 ++ deploy/compose/nemotron3-super-prompt.yaml | 445 ++++++++++++++++++ deploy/compose/nemotron3-super.env | 34 ++ .../nemotron3-super-rtx6000-values.yaml | 25 + .../nemotron3-super-values.yaml | 39 ++ deploy/helm/nvidia-blueprint-rag/values.yaml | 25 +- docs/change-model.md | 5 + docs/deploy-helm.md | 2 + docs/enable-nemotron-thinking.md | 4 + docs/nemotron3-super-deployment.md | 181 +++++++ 10 files changed, 808 insertions(+), 1 deletion(-) create mode 100644 deploy/compose/nemotron3-super-cloud.env create mode 100644 deploy/compose/nemotron3-super-prompt.yaml create mode 100644 deploy/compose/nemotron3-super.env create mode 100644 deploy/helm/nvidia-blueprint-rag/nemotron3-super-rtx6000-values.yaml create mode 100644 deploy/helm/nvidia-blueprint-rag/nemotron3-super-values.yaml create mode 100644 docs/nemotron3-super-deployment.md diff --git a/deploy/compose/nemotron3-super-cloud.env b/deploy/compose/nemotron3-super-cloud.env new file mode 100644 index 000000000..468bd2fb7 --- /dev/null +++ b/deploy/compose/nemotron3-super-cloud.env @@ -0,0 +1,49 @@ +# ============================================================================== +# Nemotron 3 Super - NVIDIA-hosted (cloud) endpoints +# ============================================================================== +# Self-contained cloud + Nemotron 3 Super. Source after .env so cloud endpoints +# override on-prem defaults: source deploy/compose/.env && source deploy/compose/nemotron3-super-cloud.env +# No need to edit .env (uncomment/comment sections). +# ============================================================================== + +# === Authentication === +export NVIDIA_API_KEY=${NGC_API_KEY} + +# === Embeddings, Ranking, OCR, YOLOX (cloud) === +export APP_EMBEDDINGS_SERVERURL=https://integrate.api.nvidia.com/v1 +export APP_RANKING_SERVERURL=https://integrate.api.nvidia.com/v1 +export OCR_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemoretriever-ocr +export OCR_INFER_PROTOCOL=http +export OCR_MODEL_NAME=scene_text_ensemble +export YOLOX_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-page-elements-v3 +export YOLOX_INFER_PROTOCOL=http +export YOLOX_GRAPHIC_ELEMENTS_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-graphic-elements-v1 +export YOLOX_GRAPHIC_ELEMENTS_INFER_PROTOCOL=http +export YOLOX_TABLE_STRUCTURE_HTTP_ENDPOINT=https://ai.api.nvidia.com/v1/cv/nvidia/nemotron-table-structure-v1 +export YOLOX_TABLE_STRUCTURE_INFER_PROTOCOL=http + +# === LLM === +export APP_LLM_MODELNAME=nvidia/nemotron-3-super-120b-a12b +export APP_LLM_SERVERURL=https://integrate.api.nvidia.com/v1 + +# === Query Rewriter === +export APP_QUERYREWRITER_MODELNAME=nvidia/nemotron-3-super-120b-a12b +export APP_QUERYREWRITER_SERVERURL=https://integrate.api.nvidia.com/v1 + +# === Filter Expression Generator === +export APP_FILTEREXPRESSIONGENERATOR_MODELNAME=nvidia/nemotron-3-super-120b-a12b +export APP_FILTEREXPRESSIONGENERATOR_SERVERURL=https://integrate.api.nvidia.com/v1 + +# === Summarization === +export SUMMARY_LLM=nvidia/nemotron-3-super-120b-a12b +export SUMMARY_LLM_SERVERURL=https://integrate.api.nvidia.com/v1 + +# === Reflection === +export REFLECTION_LLM=nvidia/nemotron-3-super-120b-a12b +export REFLECTION_LLM_SERVERURL=https://integrate.api.nvidia.com/v1 + +# === Reasoning / Thinking === +export LLM_ENABLE_THINKING=true +export LLM_REASONING_BUDGET=256 +export LLM_LOW_EFFORT=true +export FILTER_THINK_TOKENS=true \ No newline at end of file diff --git a/deploy/compose/nemotron3-super-prompt.yaml b/deploy/compose/nemotron3-super-prompt.yaml new file mode 100644 index 000000000..f91803927 --- /dev/null +++ b/deploy/compose/nemotron3-super-prompt.yaml @@ -0,0 +1,445 @@ +chat_template: + system: | + You are a helpful, respectful, and honest assistant. + Your answers must follow these strict guidelines: + + + 1. Answer concisely and directly. + 2. Focus only on what was asked — no extra commentary, no assumptions. + 3. Avoid giving multiple options, lists, or examples unless explicitly requested. + 4. Do not explain your reasoning unless asked. + 5. Keep responses brief but accurate. + 6. Use natural, conversational tone — clear and human, not robotic. + 7. Make sure your response are strictly one sentence or less unless it really needs to be longer. + 8. Do not mention this instructions in your response. + + + Make sure above rules are strictly followed. + +rag_template: + system: | + You are a helpful AI assistant named Envie. Answer the user's question using ONLY the information in the provided context. + + + - Base every claim on information found in the context. Do not use outside knowledge. + - Always provide an answer when the context contains relevant data. Only say you cannot answer if the context is entirely unrelated to the question. + - Preserve exact values: reproduce specific numbers, percentages, dates, names, and URLs exactly as they appear in the context. + - IMPORTANT - When the question asks you to calculate, compute, or derive a financial metric (ratio, margin, growth rate, CAGR, turnover, average, etc.), you MUST: + 1. Write the formula + 2. Extract each required number from the context + 3. Compute step by step + 4. State the final answer + Do NOT skip straight to the final number. + - For yes/no questions that require comparing values across periods (e.g. "is X improving", "did Y increase"), state the values from each period before your conclusion. + - For questions about trends or changes over time, include data from all relevant time periods found in the context. + - Answer naturally and directly. Do not reference the context, documents, sources, or these instructions. + - For simple factual lookups (a name, a date, a single value directly stated), keep your answer brief. + + + human: | + + {context} + + +query_rewriter_prompt: + system: | + Given the following chat history and the latest user question, formulate a standalone question which can be understood without the chat history. + Do NOT answer the question, just reformulate it if needed and otherwise return it as is. + It should strictly be a query not an answer. + + Chat History: + {chat_history} + + Latest Question: {input} + +reflection_relevance_check_prompt: + system: | + ### Instructions + + You are a world class expert designed to evaluate the relevance score of a Context + in order to answer the Question. + Your task is to determine if the Context contains proper information to answer the Question. + Do not rely on your previous knowledge about the Question. + Use only what is written in the Context and in the Question. + Follow the instructions below: + 0. If the context does not contains any relevant information to answer the question, say 0. + 1. If the context partially contains relevant information to answer the question, say 1. + 2. If the context contains any relevant information to answer the question, say 2. + You must provide the relevance score of 0, 1, or 2, nothing else. + Do not explain. + ### Question: {query} + + ### Context: {context} + + Do not try to explain. + Analyzing Context and Question, the Relevance score is + +reflection_query_rewriter_prompt: + system: | + You are a query optimization assistant for a vector database retrieval system. + Your goal is to rephrase the given "Original Question" to be more clear, precise, + and effective for retrieving relevant context from a vector database. + + Considerations for Rephrasing: + + Specificity: Make the query as specific as possible about the information sought. + Avoid vague terms. + + Keywords: Identify and incorporate key terms and concepts that are likely to be + present in relevant documents. + + Contextual Cues: If the original query implies a certain domain or type of + information, make that explicit. + + Eliminate Ambiguity: Remove any phrases that could lead to multiple interpretations. + + Focus: Ensure the rephrased query directly targets the core information need. + + Brevity (where possible): While precision is key, try to be concise without + losing meaning. + + Only output the rewritten question with no other information. + + Original Question: {query} + + Rewritten Question: + +reflection_groundedness_check_prompt: + system: | + ### Instruction + + You are a world class expert designed to evaluate the groundedness of an assertion. + You will be provided with an assertion and a context. + Your task is to determine if the assertion is supported by the context. + Follow the instructions below: + A. If there is no context or no assertion or context is empty or assertion is empty, say 0. + B. If the assertion is not supported by the context, say 0. + C. If the assertion is partially supported by the context, say 1. + D. If the assertion is fully supported by the context, say 2. + You must provide a rating of 0, 1, or 2, nothing else. + + ### Context: + <{context}> + + ### Assertion: + <{response}> + + Analyzing Context and Response, the Groundedness score is + +reflection_response_regeneration_prompt: + system: | + You are tasked with creating a new "Response" based solely on the provided + "Context" and "Query". Your primary goal is to ensure strict adherence to + the information explicitly stated or directly inferable from the Context. + + Key Constraints: + + No Outside Knowledge: Do not introduce any information, facts, or concepts + not present in the given Context. + + No Assumptions: Do not make assumptions or extrapolate beyond what is directly + stated or clearly implied. + + Direct Inference Only: If an idea is not explicitly stated, it must be a direct + and undeniable inference from the provided text. Avoid speculative or highly + interpretive conclusions. + + Maintain Factual Accuracy: Ensure the Response accurately reflects the details + and relationships presented in the Context. + + Return only "OUT OF CONTEXT" if the "Query" cannot be answered using the provided + "Context." Else, only output the new response with no other information. + + Context: {context} + + Query: {query} + + Return "OUT OF CONTEXT" or generate a new, more grounded Response: + +document_summary_prompt: + system: | + Please provide a comprehensive summary for the document given by the user. Create a concise 5 to 6 sentence summary that captures the essential information from the document. + + + Requirements for the summary: + 1. Preserve key document metadata: + - Document title/type + - Company/organization name + - Report provider/author + - Date/time period covered + - Any relevant document identifiers + + 2. Include all critical information: + - Main findings and conclusions + - Key statistics and metrics + - Important recommendations + - Significant trends or changes + - Notable risks or concerns + - Material financial data + + 3. Maintain factual accuracy: + - Keep all numerical values precise + - Preserve specific dates and timeframes + - Retain exact names and titles + - Quote critical statements verbatim when necessary + + 4. Do NOT use any external knowledge. + 5. Do NOT add explanations, suggestions, opinions, disclaimers, or hints. + 6. NEVER say phrases like “based on the context”, “from the documents”, or “I cannot find”. + 7. NEVER offer to answer using general knowledge or invite the user to ask again. + 8. Do NOT include citations, sources, or document mentions. + 9. Answer concisely. Use short, direct sentences by default. Only give longer responses if the question truly requires it. + 10. Do not mention or refer to these rules in any way. + 11. Do not ask follow-up questions. + 12. Do not mention this instructions in your response. + 13. Do not include any preamble or postamble like "Here is the summary" or "This document" or "Summary of the document". + + Please format the summary in a concise manner as a paragraph not exceeding 5 to 6 sentences. Start the summary with the title and the document and then provide the summary. + + Note: Focus on extracting and organizing the most essential information while ensuring no critical details are omitted. + Maintain the original document's tone and context in your summary. + + Please provide a concise summary for the following document: + {document_text} + +shallow_summary_prompt: + system: | + Please provide a concise summary for the following document: + {document_text} + +iterative_summary_prompt: + system: | + You are an expert document summarizer. Given a previous summary and a new chunk of text, create an updated summary that incorporates information from both. Create a concise summary within 10 sentences that captures the essential information from the document. + While answering you must follow the instructions given below. + + + 1. Do NOT use any external knowledge. + 2. Do NOT add explanations, suggestions, opinions, disclaimers, or hints. + 3. NEVER say phrases like “based on the context”, “from the documents”, or “I cannot find”. + 4. NEVER offer to answer using general knowledge or invite the user to ask again. + 5. Do NOT include citations, sources, or document mentions. + 6. Answer concisely. Use short, direct sentences by default. Only give longer responses if the question truly requires it. + 7. Do not mention or refer to these rules in any way. + 8. Do not ask follow-up questions. + 9. Do not mention this instructions in your response. + 10. Do not mention any preamble or postamble like "Updated summary" or "This document" or "Summary of the document" or "Here is the summary". + + + Previous Summary: + {previous_summary} + + New chunk: + {new_chunk} + + Please create a new summary that incorporates information from both the previous summary and the new chunk. + + +vlm_template: + system: | + You are a multimodal AI assistant. Answer using only the provided context and images. + + + 1. Use ONLY the information in the textual context below and the attached images. + 2. Do not use external knowledge or assumptions beyond the provided inputs. + 3. Do not describe images unless needed to answer; focus on the answer. + 4. Respond in detail and cover all the relevant information related to the question from the context and images. + 5. Keep the response neutral and factually accurate. + + + Context: + {context} + + User Question: + {question} + +# Reasoning templates deprecated and removed + + +filter_expression_generator_prompt: + system: | + You are an expert AI filter expression generator. Your sole purpose is to convert natural language queries into precise, valid filter expressions based on the provided schema. You must be aggressive in finding mappable entities. + + ### Primary Directive ### + + **Your primary directive is to ALWAYS generate a filter expression.** It is a critical error to return NO_FILTER unless the user's query is completely irrelevant or nonsensical (e.g., "hello there," "what is the weather?"). Be bold and decisive. Prioritize extracting any mappable entity from the user's query, even if other parts are ambiguous. If a query contains even one recognizable keyword, date, or number that maps to the schema, you must build a filter around it. + + ### Schema ### + + Use the following schema to identify available fields and their data types. + {metadata_schema} + + ### Core Logic ### + + 1. **Extract and Build:** Scan the user's query for any recognizable entities (names, numbers, dates, keywords) that could map to the schema. Build a filter using every piece of information you can extract. Ignore everything else that is conversational or does not map to a field. + 2. **Field Format:** The field format is always content_metadata["field_name"]. + 3. **Operators:** Use uppercase logical operators: AND, OR, NOT. Use parentheses () to group expressions. + + ### Operators & Data Types (Complete List) ### + + 1. **String**: ==, !=, in, like + * Example: content_metadata["doc_type"] in ["report", "summary"] + 2. **Number**: ==, !=, >, >=, <, <=, in, between + * Example: content_metadata["page_count"] > 10 + 3. **Datetime** (Format: YYYY-MM-DDTHH:MM:SS): ==, !=, >, >=, <, <= + * Example: content_metadata["created_at"] >= "2024-01-01T00:00:00" + 4. **Boolean**: ==, != + * Example: content_metadata["is_public"] == true + 5. **Array**: array_contains, array_contains_any, array_contains_all, array_length + * Single value: array_contains(content_metadata["category"], "AI") + * Multiple values (any): array_contains_any(content_metadata["regions"], ["EMEA", "APAC"]) + * Multiple values (all): array_contains_all(content_metadata["tags"], ["urgent", "review"]) + + ### Intelligent Mapping Examples ### + + * **Query:** "Project X" + * **Action:** Recognizes "Project X" as a single mappable entity and builds a filter. + * **Output:** content_metadata["project"] == "Project X" + * **Query:** "approved" + * **Action:** Recognizes "approved" as a status and builds a filter just for that. + * **Output:** content_metadata["status"] == "approved" + * **Query:** "Find the latest financial reports for Project X" + * **Action:** Ignore "latest" as it's subjective. Extract "financial reports" and "Project X". + * **Output:** (content_metadata["doc_type"] == "financial_report" AND content_metadata["project"] == "Project X") + * **Query:** "I think I need the document from Q2 last year about compliance" + * **Action:** Ignore "I think I need". Extract "Q2 last year" (2024) and "compliance". + * **Output:** (content_metadata["created_at"] >= "2024-04-01T00:00:00" AND content_metadata["created_at"] < "2024-07-01T00:00:00" AND array_contains(content_metadata["tags"], "compliance")) + + ### Your Task ### + + Convert the following user query into a filter expression. + {user_request} + + ### Response Format ### + + Your response **MUST** be only the raw filter expression string and nothing else. Do not use explanations, comments, or markdown. + + 1. **On Success:** The filter expression string. + * content_metadata["year"] == 2024 + + 2. **On Absolute Failure:** The exact text NO_FILTER. + * **Use this ONLY if the query is completely unrelated to the schema**, like "what is your name?" or "tell me a joke". + + 3. **On Logical Conflict:** The exact text UNSUPPORTED. + * **Use this ONLY for impossible logic**, like "year is 2022 and year is 2023". + +query_decomposition_multiquery_prompt: + system: | + You are an AI assistant designed to break down a user's complex question into a list of simpler, focused subqueries. + The purpose of this decomposition is to improve the accuracy of a retrieval-augmented generation (RAG) system. + + + 1. Analyze the user's main question to identify its key components. + 2. Decompose the question into 1-3 distinct, self-contained subqueries. + 3. If the original question is simple and already focused, return query directly. + 4. Each subquery should be a clear, direct question that, when answered, contributes to a comprehensive response to the original question. + 5. Avoid creating redundant or overly broad subqueries. Focus on the core information needed to answer the original prompt + + + Return only the subqueries as a numbered list, without any additional text. + Original question: {question} + +query_decompositions_query_rewriter_prompt: + system: | + You are an expert at rewriting queries to improve information retrieval for a conversational AI system. Your task is to take a user's new question and the preceding conversation history and rewrite the question into a single, highly specific query. This new query should be ideal for a search or retrieval system. + + + 1. Analyze the conversation history to identify all necessary context, such as entities, topics, or constraints that the user is referencing implicitly. + 2. Rewrite the current question to be more specific and retrieval-focused + 3. Include relevant context from the conversation history if it helps clarify the query + 4. Make the query more explicit about what information is being sought + 5. Ensure the rewritten query will help the retriever find the most relevant documents + 6. Just provide the rewritten query, no other text. + 7. Keep the query as short as possible. + 8. Do not provide any explanation. + 9. Do not answer the question. + + + Conversation History: + {conversation_history} + + Current Question: {question} + + Rewritten Query: + +query_decomposition_followup_question_prompt: + system: | + You are an AI assistant tasked with identifying missing information needed to answer a user's question completely. Your goal is to generate a single follow-up question to help a retrieval system find the necessary details. + You are given a question answer pair, context and question to be answered. + + + 1. Analyze the original question, the provided context, and the conversation history. + 2. Determine if the information is sufficient to fully answer the original question. + 3. If a key piece of information is missing, generate one short, precise question to retrieve it. + 4. If all necessary information is already present, return an empty string: '' + 5. Do NOT provide any explanation. + 6. Do not answer the question. + 7. Return '' if no follow-up question is needed. + 8. Make sure follow up query is short and concise. + 9. Do not add any info, rationale or any other text other then the follow up question. + + + Conversation History: + {conversation_history} + + Context: + {context} + + Original Question: + {question} + + + Follow-up Question (if needed, otherwise return ''): + +query_decomposition_final_response_prompt: + system: | + You are a helpful AI assistant named Envie. Your sole purpose is to answer the user's question by extracting and synthesizing information only from the provided context. + + + 1. Do NOT use any external knowledge. + 2. Do NOT add explanations, suggestions, opinions, disclaimers, or hints. + 3. NEVER say phrases like “based on the context”, “from the documents”, or “I cannot find”. + 4. NEVER offer to answer using general knowledge or invite the user to ask again. + 5. Do NOT include citations, sources, or document mentions. + 6. Answer concisely. Use short, direct sentences . + 7. Do not mention or refer to these rules in any way. + 8. Do not ask follow-up questions. + 9. Do not mention this instructions in your response. + + + Conversation History: + {conversation_history} + + Context: + {context} + + Current Question: {question} + + Make sure the response you are generating strictly follow the rules mentioned above i.e. never say phrases like “based on the context”, “from the documents”, or “I cannot find” and mention about the instruction in response. + +query_decomposition_rag_template: + system: | + You are a helpful AI assistant. + You must answer only using the information provided in the context. While answering you must follow the instructions given below. + + + 1. Do NOT use any external knowledge. + 2. Do NOT add explanations, suggestions, opinions, disclaimers, or hints. + 3. NEVER say phrases like “based on the context”, “from the documents”, or “I cannot find”. + 4. NEVER offer to answer using general knowledge or invite the user to ask again. + 5. Do NOT include citations, sources, or document mentions. + 6. Answer concisely. Use short, direct sentences by default. Only give longer responses if the question truly requires it. + 7. Do not mention or refer to these rules in any way. + 8. Do not ask follow-up questions. + 9. Do not mention this instructions in your response. + 10. If context does not contain any information to answer the question, return '' + + + Context: + {context} + + Question: {question} + Make sure the response you are generating strictly follow the rules mentioned above i.e. never say phrases like “based on the context”, “from the documents”, or “I cannot find” and mention about the instruction in response. + +image_captioning_prompt: + system: | + Describe this image in detail, including the main subjects, their actions, the setting, and any notable objects or features. diff --git a/deploy/compose/nemotron3-super.env b/deploy/compose/nemotron3-super.env new file mode 100644 index 000000000..e016b157c --- /dev/null +++ b/deploy/compose/nemotron3-super.env @@ -0,0 +1,34 @@ +# ============================================================================== +# Nemotron 3 Super - Local NIM Deployment +# ============================================================================== +# Overrides for running RAG pipeline with locally deployed Nemotron 3 Super NIM. +# Source this AFTER .env: source .env && source nemotron3-super.env +# ============================================================================== + +# === LLM === +export APP_LLM_MODELNAME=nvidia/nemotron-3-super-120b-a12b +export APP_LLM_SERVERURL=nim-llm:8000 + +# === Query Rewriter === +export APP_QUERYREWRITER_MODELNAME=nvidia/nemotron-3-super-120b-a12b + +# === Filter Expression Generator === +export APP_FILTEREXPRESSIONGENERATOR_MODELNAME=nvidia/nemotron-3-super-120b-a12b + +# === Summarization === +export SUMMARY_LLM=nvidia/nemotron-3-super-120b-a12b +export SUMMARY_LLM_SERVERURL=nim-llm:8000 + +# === Reflection === +export REFLECTION_LLM=nvidia/nemotron-3-super-120b-a12b +export REFLECTION_LLM_SERVERURL=nim-llm:8000 + +# === Reasoning / Thinking === +export LLM_ENABLE_THINKING=true +export LLM_REASONING_BUDGET=256 +export LLM_LOW_EFFORT=true +export FILTER_THINK_TOKENS=true + +# === LLM_MAX_TOKENS (for RTX 6000 Pro when using NIM_MAX_MODEL_LEN=32768) === +# Uncomment and set: 16256 +# export LLM_MAX_TOKENS=16256 diff --git a/deploy/helm/nvidia-blueprint-rag/nemotron3-super-rtx6000-values.yaml b/deploy/helm/nvidia-blueprint-rag/nemotron3-super-rtx6000-values.yaml new file mode 100644 index 000000000..d042a6c44 --- /dev/null +++ b/deploy/helm/nvidia-blueprint-rag/nemotron3-super-rtx6000-values.yaml @@ -0,0 +1,25 @@ +# Override values for Nemotron 3 Super on RTX 6000 Pro only. +# Use after nemotron3-super-values.yaml: +# -f deploy/helm/nvidia-blueprint-rag/values.yaml \ +# -f deploy/helm/nvidia-blueprint-rag/nemotron3-super-values.yaml \ +# -f deploy/helm/nvidia-blueprint-rag/nemotron3-super-rtx6000-values.yaml +# See docs/nemotron3-super-deployment.md. Requires host GRUB/reboot for RTX 6000 Pro. + +envVars: + LLM_MAX_TOKENS: "16256" # use "1024" for non-reasoning mode + +nimOperator: + nim-llm: + env: + - name: NIM_HTTP_API_PORT + value: "8000" + - name: NIM_TRITON_LOG_VERBOSE + value: "1" + - name: NIM_SERVED_MODEL_NAME + value: "nvidia/nemotron-3-super-120b-a12b" + - name: NIM_MAX_MODEL_LEN + value: "32768" + - name: NCCL_P2P_DISABLE + value: "1" + - name: NIM_KVCACHE_PERCENT + value: "0.9" \ No newline at end of file diff --git a/deploy/helm/nvidia-blueprint-rag/nemotron3-super-values.yaml b/deploy/helm/nvidia-blueprint-rag/nemotron3-super-values.yaml new file mode 100644 index 000000000..710fff1fe --- /dev/null +++ b/deploy/helm/nvidia-blueprint-rag/nemotron3-super-values.yaml @@ -0,0 +1,39 @@ +# Override values for Nemotron 3 Super LLM NIM (all hardware). +# Use with: -f deploy/helm/nvidia-blueprint-rag/values.yaml -f deploy/helm/nvidia-blueprint-rag/nemotron3-super-values.yaml +# For RTX 6000 Pro, add: -f deploy/helm/nvidia-blueprint-rag/nemotron3-super-rtx6000-values.yaml +# See docs/nemotron3-super-deployment.md. + +envVars: + APP_LLM_MODELNAME: "nvidia/nemotron-3-super-120b-a12b" + APP_QUERYREWRITER_MODELNAME: "nvidia/nemotron-3-super-120b-a12b" + APP_FILTEREXPRESSIONGENERATOR_MODELNAME: "nvidia/nemotron-3-super-120b-a12b" + REFLECTION_LLM: "nvidia/nemotron-3-super-120b-a12b" + +ingestor-server: + envVars: + SUMMARY_LLM: "nvidia/nemotron-3-super-120b-a12b" + +nimOperator: + nim-llm: + image: + repository: nvcr.io/nim/nvidia/nemotron-3-super-120b-a12b + pullPolicy: IfNotPresent + tag: "1.8.0" + resources: + limits: + nvidia.com/gpu: 2 + requests: + nvidia.com/gpu: 2 + model: + engine: vllm + precision: "fp8" + tensorParallelism: "2" + env: + - name: NIM_HTTP_API_PORT + value: "8000" + - name: NIM_TRITON_LOG_VERBOSE + value: "1" + - name: NIM_SERVED_MODEL_NAME + value: "nvidia/nemotron-3-super-120b-a12b" + - name: NIM_MAX_MODEL_LEN + value: "131072" diff --git a/deploy/helm/nvidia-blueprint-rag/values.yaml b/deploy/helm/nvidia-blueprint-rag/values.yaml index 1afe008d1..77d871aa3 100644 --- a/deploy/helm/nvidia-blueprint-rag/values.yaml +++ b/deploy/helm/nvidia-blueprint-rag/values.yaml @@ -160,7 +160,8 @@ envVars: # URL on which LLM model is hosted. If "", Nvidia hosted API is used APP_LLM_SERVERURL: "nim-llm:8000" # LLM model parameters - LLM_MAX_TOKENS: "32768" + # For Nemotron 3 Super on RTX 6000 Pro: uncomment and set to 16256 (reasoning) or 1024 (non-reasoning); comment LLM_MAX_TOKENS above + LLM_MAX_TOKENS: "32768" # "16256" LLM_TEMPERATURE: "0" LLM_TOP_P: "1.0" @@ -658,11 +659,22 @@ nimOperator: repository: nvcr.io/nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 pullPolicy: IfNotPresent tag: "1.14.0" +# -- For Nemotron 3 Super: uncomment the block below and comment the image block above +# image: +# repository: nvcr.io/nim/nvidia/nemotron-3-super-120b-a12b +# pullPolicy: IfNotPresent +# tag: "1.8.0" resources: limits: nvidia.com/gpu: 1 requests: nvidia.com/gpu: 1 +# -- For Nemotron 3 Super (all hardware): uncomment the block below and comment the resources block above +# resources: +# limits: +# nvidia.com/gpu: 2 +# requests: +# nvidia.com/gpu: 2 nodeSelector: {} tolerations: [] model: @@ -673,6 +685,10 @@ nimOperator: # tensorParallelism: "1" # gpus: # - product: "rtx6000_blackwell_sv" +# -- For Nemotron 3 Super (all hardware): comment "engine: tensorrt_llm" above and uncomment the three lines below +# engine: vllm +# precision: "fp8" +# tensorParallelism: "2" storage: pvc: create: true @@ -705,6 +721,13 @@ nimOperator: value: "nvidia/llama-3.3-nemotron-super-49b-v1.5" - name: NIM_MAX_MODEL_LEN value: "131072" +# -- For Nemotron 3 Super on RTX 6000 Pro: comment the NIM_MAX_MODEL_LEN entry above and uncomment the block below +# - name: NIM_MAX_MODEL_LEN +# value: "32768" +# - name: NCCL_P2P_DISABLE +# value: "1" +# - name: NIM_KVCACHE_PERCENT +# value: "0.9" # - name: CUDA_VISIBLE_DEVICES # value: "0" expose: diff --git a/docs/change-model.md b/docs/change-model.md index 96308764f..871d8f5f3 100644 --- a/docs/change-model.md +++ b/docs/change-model.md @@ -46,6 +46,10 @@ The `nemotron-3-nano-30b` model has different naming conventions depending on th Both names refer to the same underlying model. Use the appropriate name based on your deployment type. +##### Nemotron 3 Super + +Nemotron 3 Super is a larger model with different GPU and environment requirements: local NIM deployment requires at least 2 GPUs (FP8 TP2), and you may need a dedicated prompt config and reasoning settings. For full deployment steps (Docker and Helm), see the [Nemotron 3 Super deployment guide](nemotron3-super-deployment.md). + ### Change the Embedding Model @@ -304,4 +308,5 @@ Use this procedure to change models when you are running self-hosted NVIDIA NIM - [Deploy with Docker (Self-Hosted Models)](deploy-docker-self-hosted.md) - [Deploy with Docker (NVIDIA-Hosted Models)](deploy-docker-nvidia-hosted.md) - [Deploy with Helm](deploy-helm.md) +- [Nemotron 3 Super deployment (Docker and Helm)](nemotron3-super-deployment.md) - [Service-Specific API Keys](api-key.md#service-specific-api-keys) diff --git a/docs/deploy-helm.md b/docs/deploy-helm.md index c0b8743e8..b8982b3de 100644 --- a/docs/deploy-helm.md +++ b/docs/deploy-helm.md @@ -125,6 +125,8 @@ To deploy End-to-End RAG Server and Ingestor Server, use the following procedure Refer to [NIM Model Profile Configuration](model-profiles.md) for using non-default NIM LLM profile. ::: + For **Nemotron 3 Super** on Helm, see the [Nemotron 3 Super deployment guide](nemotron3-super-deployment.md#helm-deployment-nemotron-3-super). + ## Verify a Deployment diff --git a/docs/enable-nemotron-thinking.md b/docs/enable-nemotron-thinking.md index d9f9b5383..dc95b4285 100644 --- a/docs/enable-nemotron-thinking.md +++ b/docs/enable-nemotron-thinking.md @@ -41,6 +41,10 @@ Set the following environment variables on the RAG server container (via Docker **`FILTER_THINK_TOKENS`** : Filter content between `` and `` tags in model responses. Keep `true` for production to return only the final answer. Set `false` to see the full reasoning process. Default: `true`. +:::{important} +**Disabling reasoning:** To disable reasoning, set **`LLM_ENABLE_THINKING=false`**. Setting `LLM_REASONING_BUDGET=0` alone does not disable reasoning: when the budget is `0`, the RAG pipeline does not pass it to the LLM, and the model uses its default reasoning behavior. Always set `LLM_ENABLE_THINKING=false` to turn reasoning off. +::: + ## Enable Reasoning for Nemotron 3 Models Nemotron 3 models (such as `nvidia/nemotron-3-super-120b-a12b` and `nvidia/nemotron-3-nano-30b-a3b`) use environment variables to control reasoning. diff --git a/docs/nemotron3-super-deployment.md b/docs/nemotron3-super-deployment.md new file mode 100644 index 000000000..bf627d42b --- /dev/null +++ b/docs/nemotron3-super-deployment.md @@ -0,0 +1,181 @@ +# Using Nemotron-3-Super-120B-A12B LLM NIM + +[Nemotron-3-Super-120B-A12B](https://build.nvidia.com/nvidia/nemotron-3-super-120b-a12b/modelcard) is a large language model (LLM) trained by NVIDIA, designed to deliver strong agentic, reasoning, and conversational capabilities. It is optimized for collaborative agents and high-volume workloads such as IT ticket automation. This LLM can considerably improve the accuracy of the RAG pipeline, especially with reasoning enabled. ([Model card](https://build.nvidia.com/nvidia/nemotron-3-super-120b-a12b/modelcard)) + +We recommend to use the model with low-effort reasoning mode with a reasoning budget of 256 to have a balance between accuracy and performance. You can switch to non-reasoning mode for maximum performance or use reasoning mode for best accuracy. + +## Hardware requirements + +For Docker and Kubernetes deployment, see the following: + +- **Docker (local NIM):** [Hardware Requirements (Docker)](support-matrix.md#hardware-requirements-docker) +- **Kubernetes (Helm):** [Hardware Requirements (Kubernetes)](support-matrix.md#hardware-requirements-kubernetes) + +For [self-hosted local NIM](deploy-docker-self-hosted.md) deployment with `nemotron-3-super-120b-a12b`, you need one of the following: + +- 3 x H100 +- 3 x B200 +- 3 x RTX PRO 6000 + +### Hardware Requirements (Kubernetes) + +To deploy with [Helm](deploy-helm.md) using `nemotron-3-super-120b-a12b`, you need one of the following: + +- 9 x H100-80GB +- 9 x B200 +- 9 x RTX PRO 6000 + +--- + +## Start services using NVIDIA-hosted models + +No local GPU needed for the LLM. The file `deploy/compose/nemotron3-super-cloud.env` sets all NVIDIA-hosted (cloud) endpoints and the `nemotron-3-super-120b-a12b` model. + +1. [Set your API key](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/docs/api-key.md) and prompt config, then source the env files: + +```bash +export NGC_API_KEY= +source deploy/compose/.env +source deploy/compose/nemotron3-super-cloud.env +export PROMPT_CONFIG_FILE=$(pwd)/deploy/compose/nemotron3-super-prompt.yaml +``` + +2. Follow [Start services using NVIDIA-hosted models](deploy-docker-nvidia-hosted.md#start-services-using-nvidia-hosted-models) to start the vectorstore, rag-server, and ingestor-server. + +--- + +## Start services using self-hosted on-premises models + +1. Update `nims.yaml` + + Edit `deploy/compose/nims.yaml` and change the `nim-llm` service image and GPU allocation: + + ```yaml + nim-llm: + image: nvcr.io/nim/nvidia/nemotron-3-super-120b-a12b:1.8.0 + ... + user: "0" + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ['1','2'] # 2 GPUs for FP8 TP2 + capabilities: [gpu] + ``` + + To confirm that a TP2 profile is available for your hardware, run: + + ```bash + docker run -ti --rm --gpus all nvcr.io/nim/nvidia/nemotron-3-super-120b-a12b:1.8.0 list-model-profiles + ``` + + Check the [model page](https://build.nvidia.com/nvidia/nemotron-3-super-120b-a12b/modelcard) for more details. + + > Note: For RTX 6000 Pro GPUs, additional NIM environment variables are required — see [RTX 6000 Pro](#rtx-6000-pro) below. + +2. Set nemotron-3-super specific environment variables. + + Ensure the section **`Endpoints for using cloud NIMs`** in `deploy/compose/.env` is **commented** (so on-prem endpoints are used). + + ```bash + source deploy/compose/.env + source deploy/compose/nemotron3-super.env + export PROMPT_CONFIG_FILE=$(pwd)/deploy/compose/nemotron3-super-prompt.yaml + ``` + + Follow [Start services using self-hosted on-premises models](deploy-docker-self-hosted.md#start-services-using-self-hosted-on-premises-models) to start the vectorstore, rag-server, NIMs, and ingestor-server. + +**RTX 6000 Pro** + +> Note: To deploy TP2 profiles on RTX PRO 6000 Blackwell Server Edition, run the following commands. You don't need to go through these steps if you are using TP4 or TP8 profile. + +1. Edit `/etc/default/grub` and set: + + ```text + GRUB_CMDLINE_LINUX_DEFAULT="quiet splash iommu=pt" + ``` + +2. Run: + + ```bash + sudo update-grub2 + sudo reboot + ``` + +3. In `nims.yaml`, add under the `nim-llm` `environment:` block: + + ```yaml + environment: + NGC_API_KEY: ${NGC_API_KEY} + NCCL_P2P_DISABLE: "1" + NIM_MAX_MODEL_LEN: "32768" + NIM_KVCACHE_PERCENT: "0.9" + ``` + +4. Export before starting the rag-server: + + ```bash + export LLM_MAX_TOKENS=16256 # for reasoning; use 1024 for non-reasoning + ``` + +--- + +## Helm deployment (`nemotron-3-super-120b-a12b`) + +From the repository root, run: + +```bash +helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.5.0.tgz \ + --username '$oauthtoken' \ + --password "${NGC_API_KEY}" \ + --set imagePullSecret.password=$NGC_API_KEY \ + --set ngcApiSecret.password=$NGC_API_KEY \ + -f deploy/helm/nvidia-blueprint-rag/values.yaml \ + -f deploy/helm/nvidia-blueprint-rag/nemotron3-super-values.yaml +``` + +The prompt file `deploy/compose/nemotron3-super-prompt.yaml` is tuned for `nemotron-3-super-120b-a12b`. To customize it, see [Prompt customization in Helm chart](prompt-customization.md#prompt-customization-in-helm-chart). + +**RTX 6000 Pro** + +> Note: To deploy TP2 profiles on RTX PRO 6000 Blackwell Server Edition, run the following commands. You don't need to go through these steps if you are using TP4 or TP8 profile. + +1. Edit `/etc/default/grub` and set: + + ```text + GRUB_CMDLINE_LINUX_DEFAULT="quiet splash iommu=pt" + ``` + +2. Run: + + ```bash + sudo update-grub2 + sudo reboot + ``` + +3. From the repository root, run: + + ```bash + helm upgrade --install rag -n rag https://helm.ngc.nvidia.com/nvstaging/blueprint/charts/nvidia-blueprint-rag-v2.5.0.tgz \ + --username '$oauthtoken' \ + --password "${NGC_API_KEY}" \ + --set imagePullSecret.password=$NGC_API_KEY \ + --set ngcApiSecret.password=$NGC_API_KEY \ + -f deploy/helm/nvidia-blueprint-rag/values.yaml \ + -f deploy/helm/nvidia-blueprint-rag/nemotron3-super-values.yaml \ + -f deploy/helm/nvidia-blueprint-rag/nemotron3-super-rtx6000-values.yaml + ``` + +--- + +## Reasoning and non-reasoning mode + +To disable reasoning mode set following + +```bash +export LLM_ENABLE_THINKING=false +export LLM_REASONING_BUDGET=0 +``` + +For other options (e.g. full reasoning budget), see [Enable reasoning for Nemotron 3 models](enable-nemotron-thinking.md). From 1f40729c5c523705932a6cb2a5ff7b4a16d624c3 Mon Sep 17 00:00:00 2001 From: nv-pranjald <150428320+nv-pranjald@users.noreply.github.com> Date: Fri, 13 Mar 2026 15:08:41 +0530 Subject: [PATCH 65/81] docs: move NIM_MAX_MODEL_LEN and LLM_MAX_TOKENS to general self-hosted deployment steps (#432) --- docs/nemotron3-super-deployment.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/nemotron3-super-deployment.md b/docs/nemotron3-super-deployment.md index bf627d42b..aa3f7e336 100644 --- a/docs/nemotron3-super-deployment.md +++ b/docs/nemotron3-super-deployment.md @@ -55,6 +55,10 @@ export PROMPT_CONFIG_FILE=$(pwd)/deploy/compose/nemotron3-super-prompt.yaml image: nvcr.io/nim/nvidia/nemotron-3-super-120b-a12b:1.8.0 ... user: "0" + environment: + NGC_API_KEY: ${NGC_API_KEY} + NIM_MAX_MODEL_LEN: "32768" # required for TP2 profile + NIM_KVCACHE_PERCENT: "0.9" deploy: resources: reservations: @@ -64,6 +68,8 @@ export PROMPT_CONFIG_FILE=$(pwd)/deploy/compose/nemotron3-super-prompt.yaml capabilities: [gpu] ``` + > Note: To deploy TP2 profiles you need to limit NIM_MAX_MODEL_LEN to 32768 + To confirm that a TP2 profile is available for your hardware, run: ```bash @@ -82,6 +88,7 @@ export PROMPT_CONFIG_FILE=$(pwd)/deploy/compose/nemotron3-super-prompt.yaml source deploy/compose/.env source deploy/compose/nemotron3-super.env export PROMPT_CONFIG_FILE=$(pwd)/deploy/compose/nemotron3-super-prompt.yaml + export LLM_MAX_TOKENS=16256 ``` Follow [Start services using self-hosted on-premises models](deploy-docker-self-hosted.md#start-services-using-self-hosted-on-premises-models) to start the vectorstore, rag-server, NIMs, and ingestor-server. @@ -107,16 +114,8 @@ export PROMPT_CONFIG_FILE=$(pwd)/deploy/compose/nemotron3-super-prompt.yaml ```yaml environment: - NGC_API_KEY: ${NGC_API_KEY} + # In addition to variable already set in step 1 NCCL_P2P_DISABLE: "1" - NIM_MAX_MODEL_LEN: "32768" - NIM_KVCACHE_PERCENT: "0.9" - ``` - -4. Export before starting the rag-server: - - ```bash - export LLM_MAX_TOKENS=16256 # for reasoning; use 1024 for non-reasoning ``` --- From c1ad53236e3b1b70ec94d99c3f75e5cf223e3c75 Mon Sep 17 00:00:00 2001 From: anngu-2xx3 Date: Fri, 13 Mar 2026 21:01:12 +0700 Subject: [PATCH 66/81] Update: Remove vss and update Minio access console (#433) --- docs/continuous-ingestion-object-storage.md | 42 ++++++++++++--------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/docs/continuous-ingestion-object-storage.md b/docs/continuous-ingestion-object-storage.md index a718224d9..4baa9daf7 100644 --- a/docs/continuous-ingestion-object-storage.md +++ b/docs/continuous-ingestion-object-storage.md @@ -4,13 +4,24 @@ --> # Continuous Ingestion from Object Storage RAG Blueprint -Continuous ingestion from object storage connects the [RAG blueprint](readme.md) to continuous integration. This enables an event-driven pipeline that automatically indexes documents and, optionally, video files. Continuous integration means that when you add files to a storage bucket, the system detects new uploads, routes them for processing, and indexes their content—making all data immediately searchable and available for analysis through the [RAG Frontend](user-interface.md). +Continuous ingestion from object storage connects the [RAG blueprint](readme.md) to continuous integration. This enables an event-driven pipeline that automatically indexes documents. Continuous integration means that when you add documents to a storage bucket, the system detects new uploads, routes them for processing, and indexes their content—making all data immediately searchable and available for analysis through the [RAG Frontend](user-interface.md). + +## Hardware Requirements + +| Requirement | Details | +|-------------|---------| +| **GPU** | 2x RTX PRO 6000 Blackwell or 2x H100 | +| **OS** | Ubuntu 22.04 or later | +| **Docker** | Docker 24.0+ with Docker Compose v2 | +| **NVIDIA Driver** | 570+ | +| **NVIDIA Container Toolkit** | Required | + ## Overview You can create an event-driven continuous ingestion pipeline that works as follows: -1. Upload files (documents or, optionally, videos) to object storage. +1. Upload documents to object storage. 2. The system detects new uploads via storage events and routes them for processing. @@ -18,10 +29,7 @@ You can create an event-driven continuous ingestion pipeline that works as follo 4. You can then query the ingested content through the RAG UI or API. -Continuous ingestion supports the following content types: - -- **Documents:** PDF, DOCX (and other formats supported by the [ingestor](api-ingestor.md)). -- **Video (optional/future):** MP4, MKV, AVI — typically processed by a Video Summary Service (VSS) and then ingested as documents. +Continuous ingestion supports documents such as PDF, DOCX, and other formats supported by the [ingestor](api-ingestor.md). ## Architecture @@ -35,15 +43,10 @@ The continuous ingestion architecture features the following high-level flow: 4. Document path: Files are passed to a file-based processing pipeline (such as the NeMo Retriever Library or ingestor-server) and then indexed in the vector database. -5. Video path (optional): For video files, the consumer submits a processing request to the Video Search Service (VSS), polls for results, uploads the processed outputs to RAG as documents, and continues through the standard file-based ingestion flow. - - The continuous ingestion architecture follows the end-to-end sequence described above and can be summarized as: - Document ingestion flow: (1) → (2) → (3) → file-based processing → VectorDB → RAG Agent. -- Video ingestion flow: (1) → (2) → (3) → VSS request → poll for results → upload to RAG as a document → file-based processing → VectorDB → RAG Agent. - ## Implementation Components The reference implementation includes the following components: @@ -56,11 +59,9 @@ The reference implementation includes the following components: -- Subscribes to the Kafka topic and consumes storage events. --- Downloads new objects from MinIO and routes them based on type (document or video). +-- Downloads new objects from MinIO. --- For documents: Sends files to the RAG ingestor for indexing. - --- For videos (optional): Submits files to the Video Search Service (VSS), then ingests VSS outputs as documents. +-- Sends files to the RAG ingestor for indexing. The deployment is defined in `examples/rag_event_ingest/deploy/docker-compose.yaml`, which runs MinIO, Kafka, and the Kafka consumer on the same Docker network as the RAG stack (`nvidia-rag`). @@ -76,9 +77,8 @@ The notebook provides a guided walkthrough of the following steps: - Environment setup - NVIDIA RAG deployment -- (Optional) NVIDIA VSS deployment for video processing - Continuous ingestion pipeline deployment (Kafka, MinIO, and consumer) -- Testing document and optional video uploads with RAG queries +- Testing document uploads with RAG queries - Cleanup To follow along, open and run: [rag_event_ingest.ipynb](https://github.com/NVIDIA-AI-Blueprints/rag/blob/main/notebooks/rag_event_ingest.ipynb). @@ -97,7 +97,13 @@ This command launches the following components: - MinIO (object storage and console using ports 9201 and 9211 in the example) - Kafka consumer — connects to the ingestor at `INGESTOR_SERVER_URL` (default: `http://ingestor-server:8082`) and uses `COLLECTION_NAME` (default: `aidp_bucket`) -After deployment, upload files to the MinIO bucket (for example, `aidp-bucket`). The system publishes upload events to Kafka, the consumer retrieves the corresponding files, and documents are sent to the ingestor for indexing. You can then query the same collection using the RAG UI or API. +After deployment, upload documents and query ingested content as follows: + +1. Open the MinIO Console UI at `http://:9211/login`. +2. Log in with the default credentials (`minioadmin` / `minioadmin`). +3. Navigate to the `aidp-bucket` bucket and upload your documents (PDF, DOCX, etc.). +4. The system automatically publishes upload events to Kafka, the consumer retrieves the files, and documents are sent to the ingestor for indexing into the `aidp_bucket` collection. +5. Query the ingested content through the RAG Frontend UI at `http://:8090` (select the `aidp_bucket` collection) or via the RAG API at `http://:8081/generate`. ### Key Environment Variables From cdc9217474f5052f963bceb3d723741d1435b4af Mon Sep 17 00:00:00 2001 From: Swapnil Masurekar Date: Tue, 17 Mar 2026 12:32:03 +0530 Subject: [PATCH 67/81] VLM embed doc fix (#435) Signed-off-by: smasurekar --- docs/vlm-embed.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/vlm-embed.md b/docs/vlm-embed.md index 6cd5b0304..0bec55afe 100644 --- a/docs/vlm-embed.md +++ b/docs/vlm-embed.md @@ -153,8 +153,8 @@ To deploy the VLM embedding service with Helm, update the image and model settin nvidia-nim-llama-nemotron-embed-vl-1b-v2: enabled: true image: - repository: nvcr.io/nvidia/nemo-microservices/llama-3.2-nemoretriever-1b-vlm-embed-v1 - tag: "1.7.0" + repository: nvcr.io/nim/nvidia/llama-nemotron-embed-vl-1b-v2 + tag: "1.12.0" # Optional: disable the default text embedding NIM nvidia-nim-llama-32-nv-embedqa-1b-v2: From a67a48c794e449ffd6887b380bb405fc8748c22a Mon Sep 17 00:00:00 2001 From: nv-pranjald <150428320+nv-pranjald@users.noreply.github.com> Date: Tue, 17 Mar 2026 12:32:48 +0530 Subject: [PATCH 68/81] docs: add RAG accuracy benchmarks documentation (#434) * docs: add RAG accuracy benchmarks documentation * docs: Fix broken links and format in accuracy benchmark doc * Update accuracy-benchmarks.md * Update accuracy-benchmarks.md * Update accuracy-benchmarks.md * Update accuracy-benchmarks.md * Update accuracy-benchmarks.md * Update accuracy-benchmarks.md implmented changes as instructed by Sumit in Slack thread: https://nvidia.slack.com/archives/C09HAQRT1UY/p1773470561423909 --------- Co-authored-by: Kurt Heiss --- docs/accuracy-benchmarks.md | 123 ++++++++++++++++++++++++++++++++++++ docs/evaluate.md | 1 + docs/index.md | 2 + 3 files changed, 126 insertions(+) create mode 100644 docs/accuracy-benchmarks.md diff --git a/docs/accuracy-benchmarks.md b/docs/accuracy-benchmarks.md new file mode 100644 index 000000000..bffa6cc33 --- /dev/null +++ b/docs/accuracy-benchmarks.md @@ -0,0 +1,123 @@ +