Refactored the ingestion admin and docling servers

Author: vaaale

.env.example

@@ -1,62 +1,62 @@
-# ============================================================================
-# OpenAPI HTML Server Environment Configuration
-# ============================================================================
-# Copy this file to .env and update with your values
-
-# ============================================================================
-# OpenAI / LLM API Configuration
-# ============================================================================
-# For OpenAI
-OPENAI_API_KEY=sk-123
-OPENAI_BASE_URL=http://wingman.akhbar.home/v1
+# ------------------------------------------#LLM_MODEL=gpt-4-turbo
 
-# For Ollama (local)
-# OPENAI_API_KEY=sk-321
-# OPENAI_BASE_URL=http://localhost:11434/v1
+#   General
+# ------------------------------------------
+DOCUMENTS_PATH=./docs
 
-# For other OpenAI-compatible APIs
-# OPENAI_API_KEY=your-key
-# OPENAI_BASE_URL=http://your-api-endpoint/v1
+# Callback base URL for async parsing
+# This should be the externally accessible URL of this admin server
+CALLBACK_BASE_URL=http://localhost:8005
 
-# ============================================================================
-# LLM Model Configuration
-# ============================================================================
-LLM_MODEL=gpt-4-turbo
-# Alternatives: gpt-3.5-turbo, gpt-4, llama2, mistral, etc.
+DOCLING_SERVER=http://docling-server:8001
+MONGO_URI=mongodb://mongodb:27017/
+POSTGRES_URL=postgresql://postgres:password@pgvector:5432/ingestion
+# Async parsing configuration
+USE_ASYNC_PARSING=true
 
-# ============================================================================
-# Embeddings Configuration
-# ============================================================================
-EMBEDDING_MODEL=bge-m3
-# Alternatives: text-embedding-ada-002, bge-large, etc.
+# Worker configuration
+# Number of concurrent workers for processing documents
+POOL_MIN_SIZE=10
+POOL_MAX_SIZE=20
+NUM_WORKERS=12
 
-# ============================================================================
-# Qdrant Vector Database Configuration
-# ============================================================================
-QDRANT_HOST=wingman.akhbar.home
-QDRANT_PORT=6333
 COLLECTION_NAME=fetchcraft_chatbot
 
-# ============================================================================
-# Documents Configuration
-# ============================================================================
-DOCUMENTS_PATH=/app/Documents
-
-# ============================================================================
-# Chunking Configuration
-# ============================================================================
-CHUNK_SIZE=8192
-CHUNK_OVERLAP=200
+# ------------------------------------------
+#   Embeddings
+# ------------------------------------------
+EMBEDDING_MODEL=bge-m3
+EMBEDDING_API_KEY=sk-123
+EMBEDDING_BASE_URL=http://localhost:8000/v1
 
-# ============================================================================
-# Hybrid Search Configuration
-# ============================================================================
+# ------------------------------------------
+#   LLM - OpenAI
+# ------------------------------------------
+OPENAI_API_KEY=sk-123
+OPENAI_BASE_URL=http://localhost:8000/v1
+LLM_MODEL=gpt-5
+
+# ------------------------------------------
+# PostgreSQL
+# ------------------------------------------
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=password
+POSTGRES_DB=fetchcraft
+
+# ------------------------------------------
+#   Qdrant Vector Store
+# ------------------------------------------
+QDRANT_HOST=localhost
+QDRANT_PORT=6333
 ENABLE_HYBRID=true
 FUSION_METHOD=rrf
-# Alternatives: dbsf (distribution-based score fusion)
 
-# ============================================================================
-# Server Configuration
-# ============================================================================
-HOST=0.0.0.0
-PORT=8001
+# ------------------------------------------
+#   Vector Index
+# ------------------------------------------
+INDEX_ID=docs-index
+CHUNK_SIZE=16384
+CHILD_CHUNKS=4096,1024
+CHUNK_OVERLAP=200
+
+HF_HOME=$HOME/.cache/huggingface

Dockerfile

@@ -1,7 +1,7 @@
 # Use Python 3.12 slim image
 FROM python:3.12-slim
 
-RUN apt-get update && apt-get install ffmpeg libsm6 libxext6  -y
+RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 htop vim wget  -y
 
 # Set working directory
 WORKDIR /app

Dockerfile-gpu

@@ -1,7 +1,7 @@
-FROM pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel
+FROM pytorch/pytorch:2.6.0-cuda12.6-cudnn9-devel
 #FROM pytorch/pytorch:2.6.0-cuda12.4-cudnn9-runtime
 
-#RUN apt-get update && apt install python3 install ffmpeg libsm6 libxext6 nvtop -y
+RUN apt-get update && apt install ffmpeg libsm6 libxext6 nvtop wget htop vim -y
 
 # Set working directory
 WORKDIR /app