fix: delete url embeddings from index when deleting from Admin website (#977)

Author: komalg1

code/backend/batch/utilities/search/search_handler_base.py

@@ -24,6 +24,8 @@ def get_unique_files(self, results, facet_key: str):
 
     def delete_from_index(self, blob_url) -> None:
         documents = self.search_by_blob_url(blob_url)
+        if documents is None or documents.get_count() == 0:
+            return
         files_to_delete = self.output_results(documents)
         self.delete_files(files_to_delete)
 

code/backend/pages/03_Delete_Data.py

@@ -62,15 +62,15 @@
                 st.info("No files selected")
                 st.stop()
             else:
+                files_to_delete = search_handler.delete_files(
+                    selected_files,
+                )
                 blob_client = AzureBlobStorageClient()
                 blob_client.delete_files(
                     selected_files, env_helper.AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION
                 )
-                if len(selected_files) > 0:
-                    st.success(
-                        "Deleted files from storage. Deleting from the index is an asynchronous process and may take a few minutes to complete."
-                        + ", ".join([name for name, ids in selected_files.items()])
-                    )
+                if len(files_to_delete) > 0:
+                    st.success("Deleted files: " + str(files_to_delete))
 
 except Exception:
     logger.error(traceback.format_exc())

code/tests/search_utilities/test_azure_search_handler.py

@@ -3,7 +3,7 @@
 from backend.batch.utilities.search.azure_search_handler import AzureSearchHandler
 import json
 from azure.search.documents.models import VectorizedQuery
-
+from azure.search.documents import SearchItemPaged
 from backend.batch.utilities.common.source_document import SourceDocument
 
 
@@ -369,14 +369,23 @@ def test_delete_from_index(handler, mock_search_client):
     # given
     blob_url = "https://example.com/blob"
     filter_value = f"source eq '{blob_url}_SAS_TOKEN_PLACEHOLDER_'"
-    documents = [
-        {"id": 1, "title": "file1"},
-        {"id": 2, "title": "file2"},
-        {"id": 3, "title": "file1"},
-        {"id": 4, "title": "file3"},
-    ]
+    documents = Mock(
+        SearchItemPaged(
+            [
+                {"id": 1, "title": "file1"},
+                {"id": 2, "title": "file2"},
+                {"id": 3, "title": "file1"},
+                {"id": 4, "title": "file3"},
+            ]
+        )
+    )
+
     handler.search_client.search.return_value = documents
+    documents.get_count.return_value = 4
     ids_to_delete = [{"id": 1}, {"id": 3}, {"id": 2}, {"id": 4}]
+    handler.output_results = MagicMock(
+        return_value={"file1": [1, 3], "file2": [2], "file3": [4]}
+    )
 
     # when
     handler.delete_from_index(blob_url)

code/tests/search_utilities/test_integrated_vectorization_search_handler.py

@@ -1,9 +1,10 @@
 import pytest
-from unittest.mock import Mock, patch
+from unittest.mock import MagicMock, Mock, patch
 from backend.batch.utilities.search.integrated_vectorization_search_handler import (
     IntegratedVectorizationSearchHandler,
 )
 from azure.search.documents.models import VectorizableTextQuery
+from azure.search.documents import SearchItemPaged
 
 from backend.batch.utilities.common.source_document import SourceDocument
 
@@ -261,12 +262,20 @@ def test_delete_from_index(env_helper_mock, handler, search_client_mock):
     env_helper_mock.AZURE_BLOB_CONTAINER_NAME = "documents"
     blob_url = "https://example.com/documents/file1.txt"
     title = "file1.txt"
-    documents = [
-        {"chunk_id": "123_chunk", "title": title},
-        {"chunk_id": "789_chunk", "title": title},
-    ]
+    documents = Mock(
+        SearchItemPaged(
+            [
+                {"chunk_id": "123_chunk", "title": title},
+                {"chunk_id": "789_chunk", "title": title},
+            ]
+        )
+    )
     search_client_mock.search.return_value = documents
+    documents.get_count.return_value = 2
     ids_to_delete = [{"chunk_id": "123_chunk"}, {"chunk_id": "789_chunk"}]
+    handler.output_results = MagicMock(
+        return_value={"file1.txt": ["123_chunk", "789_chunk"]}
+    )
 
     # when
     handler.delete_from_index(blob_url)