update

Author: Tom Softreck

Makefile

@@ -1,6 +1,12 @@
 # DialogChain - Flexible Dialog Processing Framework
 # Makefile for development and deployment
 
+# Logging configuration
+LOG_LEVEL ?= INFO
+LOG_FILE ?= logs/dialogchain.log
+DB_LOG_FILE ?= logs/dialogchain.db
+LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
 .PHONY: help install dev test clean build docker run-example lint docs \
         test-unit test-integration test-e2e coverage typecheck format check-codestyle \
         check-all pre-commit-install setup-dev-env docs-serve docs-clean \
@@ -26,6 +32,10 @@ help:
 	@echo "  logs             - View recent logs (use: make logs LINES=50)"
 	@echo "  view-logs        - View logs for running example"
 	@echo "  stop-example     - Stop a running example"
+	@echo "  log-level        - Set log level (use: make log-level LEVEL=DEBUG)"
+	@echo "  log-db           - View database logs (use: make log-db LIMIT=50)"
+	@echo "  log-tail         - Follow log file in real-time"
+	@echo "  log-clear        - Clear log files"
 	@echo "  docs             - Generate documentation"
 	@echo "  setup-env        - Create example .env file"
 	@echo ""
@@ -69,10 +79,18 @@ install-deps:
 # Development
 test: venv test-unit test-integration test-e2e
 
-# Run unit tests
+# Run unit tests with logging
 test-unit:
-	@pytest tests/unit/ -v --cov=src/dialogchain --cov-report=term-missing
-	@echo "✅ Unit tests completed"
+	@mkdir -p logs
+	@echo "🔍 Running unit tests with log level: $(LOG_LEVEL)"
+	@PYTHONPATH=./src pytest tests/unit/ -v \
+		--log-cli-level=$(LOG_LEVEL) \
+		--log-file=$(LOG_FILE) \
+		--log-file-level=$(LOG_LEVEL) \
+		--log-file-format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" \
+		--cov=src/dialogchain \
+		--cov-report=term-missing
+	@echo "✅ Unit tests completed - Logs saved to $(LOG_FILE)"
 
 # Run integration tests
 test-integration:
@@ -97,6 +115,32 @@ typecheck:
 	@mypy src/dialogchain/
 	@echo "✅ Type checking completed"
 
+# Logging commands
+log-level:
+	@if [ -z "$(LEVEL)" ]; then \
+		echo "Current log level: $(LOG_LEVEL)"; \
+		echo "Usage: make log-level LEVEL=DEBUG|INFO|WARNING|ERROR|CRITICAL"; \
+	else \
+		sed -i.bak 's/^LOG_LEVEL = .*/LOG_LEVEL = $(LEVEL)/' Makefile && \
+		rm -f Makefile.bak && \
+		echo "✅ Log level set to $(LEVEL)"; \
+	fi
+
+log-db:
+	@mkdir -p logs
+	@echo "📋 Displaying last $(or $(LIMIT), 50) database log entries:"
+	@python -c "from dialogchain.utils.logger import display_recent_logs; display_recent_logs(limit=$(or $(LIMIT), 50))"
+
+log-tail:
+	@echo "📜 Tailing log file: $(LOG_FILE)"
+	@tail -f $(LOG_FILE)
+
+log-clear:
+	@rm -f $(LOG_FILE) $(DB_LOG_FILE)
+	@mkdir -p logs
+	@touch $(LOG_FILE) $(DB_LOG_FILE)
+	@echo "🧹 Log files cleared"
+
 # Run all linters
 lint:
 	@echo "🔍 Running flake8..."

src/dialogchain/utils/logger.py

@@ -1,25 +1,76 @@
 """
-Standardized logging with SQLite storage support.
+Standardized logging with SQLite storage support for DialogChain.
+
+This module provides a robust logging solution with the following features:
+- SQLite-based log storage for persistence and querying
+- Configurable log levels and formats
+- Thread-safe logging
+- Structured log data with timestamps and metadata
+- Support for both file and database logging
+
+Example usage:
+    >>> from dialogchain.utils.logger import setup_logger, get_logs
+    >>> logger = setup_logger(__name__, log_level='DEBUG')
+    >>> logger.info('This is an info message', extra={'key': 'value'})
+    >>> logs = get_logs(limit=10)
 """
 import logging
 import sqlite3
 import json
 import os
+import sys
+import threading
 from pathlib import Path
 from datetime import datetime
-from typing import Dict, Any, Optional, List
+from typing import Dict, Any, Optional, List, Union
+from logging.handlers import RotatingFileHandler
+
+# Default configuration
+DEFAULT_LOG_LEVEL = logging.INFO
+DEFAULT_DB_PATH = 'logs/dialogchain.db'
+DEFAULT_LOG_FILE = 'logs/dialogchain.log'
+MAX_LOG_SIZE = 10 * 1024 * 1024  # 10MB
+BACKUP_COUNT = 5
+
+# Thread lock for SQLite operations
+db_lock = threading.Lock()
 
 # Create a module-level logger that doesn't trigger setup
 _logger = logging.getLogger(__name__)
-_logger.setLevel(logging.INFO)
+_logger.setLevel(DEFAULT_LOG_LEVEL)
 
-# Add console handler if no handlers are configured
+# Prevent duplicate handlers
 if not _logger.handlers:
-    console_handler = logging.StreamHandler()
-    console_handler.setFormatter(
-        logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    # Create logs directory if it doesn't exist
+    os.makedirs(os.path.dirname(DEFAULT_DB_PATH) or '.', exist_ok=True)
+    os.makedirs(os.path.dirname(DEFAULT_LOG_FILE) or '.', exist_ok=True)
+    
+    # Console handler
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_formatter = logging.Formatter(
+        '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S'
     )
+    console_handler.setFormatter(console_formatter)
     _logger.addHandler(console_handler)
+    
+    # File handler with rotation
+    try:
+        file_handler = RotatingFileHandler(
+            DEFAULT_LOG_FILE,
+            maxBytes=MAX_LOG_SIZE,
+            backupCount=BACKUP_COUNT,
+            encoding='utf-8'
+        )
+        file_formatter = logging.Formatter(
+            '{"timestamp": "%(asctime)s", "level": "%(levelname)s", '
+            '"module": "%(name)s", "message": "%(message)s", "data": %(extra)s}',
+            datefmt='%Y-%m-%dT%H:%M:%S%z'
+        )
+        file_handler.setFormatter(file_formatter)
+        _logger.addHandler(file_handler)
+    except Exception as e:
+        _logger.error(f"Failed to initialize file handler: {e}", exc_info=True)
 
 
 class DatabaseLogHandler(logging.Handler):

tests/unit/test_connectors_destinations.py

@@ -85,17 +85,20 @@ async def test_send_http_request(self, mock_session_class, http_dest, capsys):
         # Setup mock response
         mock_response = AsyncMock()
         mock_response.status = 200
-        mock_response.text.return_value = "OK"
+        mock_response.text = AsyncMock(return_value="OK")
 
-        # Setup mock post method
+        # Setup mock post method with async context manager
         mock_post = AsyncMock()
         mock_post.__aenter__.return_value = mock_response
 
-        # Setup mock session
+        # Setup mock session with async context manager
         mock_session = AsyncMock()
         mock_session.post.return_value = mock_post
-        mock_session.__aenter__.return_value = mock_session
-        mock_session_class.return_value = mock_session
+        
+        # Setup the session context manager
+        mock_session_ctx = AsyncMock()
+        mock_session_ctx.__aenter__.return_value = mock_session
+        mock_session_class.return_value = mock_session_ctx
 
         # Test with dict message
         await http_dest.send({"key": "value"})
@@ -124,7 +127,7 @@ async def test_send_http_request(self, mock_session_class, http_dest, capsys):
         # Reset mocks for error test
         mock_session.post.reset_mock()
         mock_response.status = 400
-        mock_response.text.return_value = "Bad Request"
+        mock_response.text = AsyncMock(return_value="Bad Request")
 
         # Test error case
         await http_dest.send({"key": "value"})