fix: improve NLTK resource handling and logging in GCS document retri… (#1395)

* fix: improve NLTK resource handling and logging in GCS document retrieval

* fix: streamline NLTK resource downloading and path management in GCS document retrieval

Author: praveshkumar1988

backend/src/document_sources/gcs_bucket.py

@@ -47,57 +47,59 @@ def gcs_loader_func(file_path):
 
 def get_documents_from_gcs(gcs_project_id, gcs_bucket_name, gcs_bucket_folder, gcs_blob_filename, access_token=None):
 
-  nltk.data.path.append("/usr/local/nltk_data")
-  nltk.data.path.append(os.path.expanduser("~/.nltk_data"))
-  try:
-      nltk.data.find("tokenizers/punkt")
-  except LookupError:
-    for resource in ["punkt", "averaged_perceptron_tagger"]:
+  nltk_data_dirs = ["/usr/local/nltk_data", os.path.expanduser("~/.nltk_data")]
+  for d in nltk_data_dirs:
+      if d not in nltk.data.path:
+          nltk.data.path.append(d)
+
+  resources = [
+      ("punkt", "tokenizers"),
+      ("averaged_perceptron_tagger", "taggers"),
+  ]
+  for res, res_type in resources:
       try:
-          nltk.data.find(f"tokenizers/{resource}" if resource == "punkt" else f"taggers/{resource}")
+          nltk.data.find(f"{res_type}/{res}")
       except LookupError:
-          logging.info(f"Downloading NLTK resource: {resource}")
-          nltk.download(resource, download_dir=os.path.expanduser("~/.nltk_data"))
-          
-    logging.info("NLTK resources downloaded successfully.")
-    if gcs_bucket_folder is not None and gcs_bucket_folder.strip()!="":
+          logging.info(f"NLTK resource '{res}' not found; downloading to /usr/local/nltk_data")
+          nltk.download(res, download_dir="/usr/local/nltk_data")
+
+
+  if gcs_bucket_folder is not None and gcs_bucket_folder.strip() != "":
       if gcs_bucket_folder.endswith('/'):
-        blob_name = gcs_bucket_folder+gcs_blob_filename
+          blob_name = gcs_bucket_folder + gcs_blob_filename
       else:
-        blob_name = gcs_bucket_folder+'/'+gcs_blob_filename 
-    else:
-        blob_name = gcs_blob_filename  
-    
-    logging.info(f"GCS project_id : {gcs_project_id}")  
-  
-    if access_token is None:
+          blob_name = gcs_bucket_folder + '/' + gcs_blob_filename
+  else:
+      blob_name = gcs_blob_filename
+
+  logging.info(f"GCS project_id : {gcs_project_id}")
+
+  if access_token is None:
       storage_client = storage.Client(project=gcs_project_id)
       bucket = storage_client.bucket(gcs_bucket_name)
-      blob = bucket.blob(blob_name) 
-      
+      blob = bucket.blob(blob_name)
       if blob.exists():
           loader = GCSFileLoader(project_name=gcs_project_id, bucket=gcs_bucket_name, blob=blob_name, loader_func=gcs_loader_func)
-          pages = loader.load() 
-      else :
-        raise LLMGraphBuilderException('File does not exist, Please re-upload the file and try again.')
-    else:
-      creds= Credentials(access_token)
+          pages = loader.load()
+      else:
+          raise LLMGraphBuilderException('File does not exist, Please re-upload the file and try again.')
+  else:
+      creds = Credentials(access_token)
       storage_client = storage.Client(project=gcs_project_id, credentials=creds)
-    
       bucket = storage_client.bucket(gcs_bucket_name)
-      blob = bucket.blob(blob_name) 
+      blob = bucket.blob(blob_name)
       if blob.exists():
-        content = blob.download_as_bytes()
-        pdf_file = io.BytesIO(content)
-        pdf_reader = PdfReader(pdf_file)
-        # Extract text from all pages
-        text = ""
-        for page in pdf_reader.pages:
-              text += page.extract_text()
-        pages = [Document(page_content = text)]
+          content = blob.download_as_bytes()
+          pdf_file = io.BytesIO(content)
+          pdf_reader = PdfReader(pdf_file)
+          # Extract text from all pages
+          text = ""
+          for page in pdf_reader.pages:
+              text += page.extract_text() or ""
+          pages = [Document(page_content=text)]
       else:
-        raise LLMGraphBuilderException(f'File Not Found in GCS bucket - {gcs_bucket_name}')
-    return gcs_blob_filename, pages
+          raise LLMGraphBuilderException(f'File Not Found in GCS bucket - {gcs_bucket_name}')
+  return gcs_blob_filename, pages
 
 def upload_file_to_gcs(file_chunk, chunk_number, original_file_name, bucket_name, folder_name_sha1_hashed):
   try: