Improve documentation and strengthen path validation logic

Co-authored-by: Mte90 <403283+Mte90@users.noreply.github.com>

Author: Copilot

ai/analyzer.py

@@ -261,6 +261,11 @@ def _get_chunk_text(database_path: str, file_id: int, chunk_index: int) -> Optio
     
     Security: Validates that the resolved file path is within the project directory
     to prevent path traversal attacks using os.path.commonpath for cross-platform safety.
+    
+    Note: Uses errors='replace' when reading files to handle invalid UTF-8 sequences
+    by replacing them with the Unicode replacement character (U+FFFD) rather than
+    dropping them silently. This preserves the file structure and makes encoding
+    issues visible.
     """
     conn = _connect_db(database_path)
     try:
@@ -281,25 +286,34 @@ def _get_chunk_text(database_path: str, file_id: int, chunk_index: int) -> Optio
         project_path = get_project_metadata(database_path, "project_path")
         if not project_path:
             logger.error("Project path not found in metadata, cannot read file from filesystem")
-            raise RuntimeError("Project path metadata is missing - indexing may not have completed properly")
+            raise RuntimeError("Project path metadata is missing - ensure the indexing process has stored project metadata properly")
 
         # Construct full file path and resolve to absolute path
         full_path = os.path.abspath(os.path.join(project_path, file_path))
         normalized_project_path = os.path.abspath(project_path)
 
         # Security check: ensure the resolved path is within the project directory
-        # Using os.path.commonpath for cross-platform safety (handles Windows/Unix path separators)
+        # Using os.path.commonpath to find the longest common path.
+        # If the common path equals the project path, and the full path starts with
+        # the project path (or is the project path itself), then it's safe.
         try:
             common = os.path.commonpath([full_path, normalized_project_path])
+            # Verify the common path is the project path (file is within project)
             if common != normalized_project_path:
                 logger.error(f"Path traversal attempt detected: {file_path} resolves outside project directory")
                 return None
+            # Additional check: ensure full_path actually starts with project path
+            # (covers edge cases where paths might be equal)
+            if full_path != normalized_project_path and not full_path.startswith(normalized_project_path + os.sep):
+                logger.error(f"Path traversal attempt detected: {file_path} does not start with project directory")
+                return None
         except ValueError:
-            # Different drives on Windows
-            logger.error(f"Path traversal attempt detected: {file_path} is on a different drive")
+            # Different drives on Windows or incompatible paths
+            logger.error(f"Path traversal attempt detected: {file_path} is on a different drive or incompatible path")
             return None
 
         # Read file content from filesystem
+        # Using errors='replace' to preserve file structure and make encoding issues visible
         try:
             with open(full_path, "r", encoding="utf-8", errors="replace") as fh:
                 content = fh.read()