fixed probe uuencoding

Author: john-friedman

datamule/datamule/datamule/tar_downloader.py

@@ -393,6 +393,7 @@ def close_all(self):
     async def download_and_process(self, session, url, semaphore, extraction_pool, tar_manager, output_dir, pbar, keep_document_types, keep_filtered_metadata):
         async with semaphore:
             filename = url.split('/')[-1]
+
             accession_num = filename.replace('.tar', '').split('/')[-1]
 
             api_key = self.api_key
@@ -438,6 +439,9 @@ async def download_and_process(self, session, url, semaphore, extraction_pool, t
                         extraction_pool,
                         partial(self._extract_documents_from_probe, probe_bytes, metadata_with_positions, keep_document_types)
                     )
+
+                    for doc in documents:
+                        doc['content'] = should_decode_file_from_content(doc['content'])
 
                 elif keep_document_types == ['metadata']:
                     # Only metadata requested
@@ -462,7 +466,7 @@ async def download_and_process(self, session, url, semaphore, extraction_pool, t
                         for doc in probe_documents:
                             doc['content'] = should_decode_file_from_content(doc['content'])
                         documents.extend(probe_documents)
-                    
+
                     # Download each document beyond probe individually
                     if docs_beyond_probe:
                         for doc in docs_beyond_probe:
@@ -492,8 +496,9 @@ async def download_and_process(self, session, url, semaphore, extraction_pool, t
                                     extraction_pool,
                                     partial(self._decompress_zstd, doc_content)
                                 )
-
+                            
                                 decompressed = should_decode_file_from_content(decompressed)
+
 
                                 documents.append({
                                     'name': doc_name,
@@ -584,7 +589,6 @@ def download(self, accession_numbers, output_dir="downloads",
         for accession in accession_numbers:
             url = f"{self.BASE_URL}{format_accession(accession,'no-dash').zfill(18)}.tar"
             urls.append(url)
-        
         # Deduplicate URLs
         urls = list(set(urls))
 

datamule/datamule/utils/pdf.py

@@ -1,25 +1,30 @@
-def has_extractable_text(pdf_bytes, search_range=50000):
+import pypdfium2 as pdfium
+
+def has_extractable_text(pdf_bytes):
     """
-    Check if PDF contains extractable text within first N bytes
-    Returns True if found in range, False otherwise
-    
-    Args:
-        pdf_bytes: PDF content as bytes
-        search_range: Number of bytes to search from start (default 50KB)
+    Lightweight check if PDF has extractable text.
+    Only checks first page.
     """
-    # Text indicators to search for
-    indicators = [
-        b'BT',  # Begin text - most common
-        b'Tj',  # Show text
-        b'TJ',  # Show text with positioning
-        b'Tf',  # Set font
-    ]
-    
-    # Search only within the specified range
-    search_data = pdf_bytes[:search_range]
-    
-    for indicator in indicators:
-        if indicator in search_data:
-            return True
-    
-    return False
+    try:
+        # Load PDF from bytes
+        pdf = pdfium.PdfDocument(pdf_bytes)
+        
+        # Only check first page
+        if len(pdf) == 0:
+            return False
+        
+        page = pdf[0]
+        textpage = page.get_textpage()
+        text = textpage.get_text_range()
+        
+        # Close resources
+        textpage.close()
+        page.close()
+        pdf.close()
+        
+        # Check if we got any text (after stripping whitespace)
+        return len(text.strip()) > 0
+        
+    except Exception as e:
+        # If can't open/parse, assume no text
+        return False

datamule/setup.py

@@ -32,7 +32,7 @@
 setup(
     name="datamule",
     author="John Friedman",
-    version="3.2.9",
+    version="3.3.0",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",