secsgml2 integrated

Author: john-friedman

datamule/datamule/datamule/downloader.py

@@ -165,7 +165,7 @@ def decompress_and_parse_and_write(self, compressed_chunks, filename, keep_docum
             content = decompressed_content.getvalue()
 
             metadata, documents = parse_sgml_content_into_memory(
-                bytes_content=content,
+                data=content,
                 filter_document_types=keep_document_types
             )
 
@@ -187,7 +187,7 @@ def parse_and_write_regular_file(self, chunks, filename, keep_document_types, ta
             content = b''.join(chunks)
 
             metadata, documents = parse_sgml_content_into_memory(
-                bytes_content=content,
+                data=content,
                 filter_document_types=keep_document_types
             )
 

datamule/datamule/document/document.py

@@ -253,8 +253,17 @@ def __init__(self, type, content, extension,accession,filing_date,path=None):
 
         if path is not None:
             # need to think through document parsing w/ and w/o path... e.g. from url, metadata should fill it
-            self.path = Path(path)
-            self.filename = self.path.stem
+            path_str = str(path)
+            if isinstance(path, Path) or "::" not in path_str:
+                self.path = Path(path)
+            else:
+                # Preserve virtual "tar::inner/path" references
+                self.path = path
+
+            filename_source = path_str
+            if "::" in filename_source:
+                filename_source = filename_source.split("::", 1)[1]
+            self.filename = Path(filename_source).stem
 
 
 
@@ -573,4 +582,4 @@ def get_tables(self, description_regex=None, description_fields=['preamble', 'po
             name=name, 
             contains_regex=contains_regex,
             title_regex=title_regex
-        )
+        )

datamule/datamule/portfolio/portfolio_compression_utils_legacy.py

@@ -159,14 +159,16 @@ def decompress_portfolio(self, portfolio, max_workers=None):
                         output_dir.mkdir(exist_ok=True)
 
                         # Get all files for this accession
-                        accession_files = [m for m in tar.getmembers() 
-                                            if m.name.startswith(f'{accession_dir}/') and m.isfile()]
+                        accession_files = [
+                            m.name for m in tar.getmembers()
+                            if m.name.startswith(f'{accession_dir}/') and m.isfile()
+                        ]
 
                         # Parallel file extraction
                         with ThreadPoolExecutor(max_workers=max_workers) as executor:
                             file_futures = [
-                                executor.submit(self._extract_file, member, tar, accession_dir, output_dir)
-                                for member in accession_files
+                                executor.submit(self._extract_file, member_name, batch_tar, accession_dir, output_dir)
+                                for member_name in accession_files
                             ]
 
                             # Wait for all files to be processed
@@ -209,12 +211,17 @@ def _process_document(self, doc, compression, threshold, compression_level):
 
         return content, compression_type
 
-    def _extract_file(self, member, tar, accession_dir, output_dir):
+    def _extract_file(self, member_name, batch_tar_path, accession_dir, output_dir):
         """Extract and decompress a single file from tar."""
-        relative_path = member.name[len(accession_dir)+1:]  # Remove accession prefix
+        relative_path = member_name[len(accession_dir)+1:]  # Remove accession prefix
         output_path = output_dir / relative_path
 
-        content = tar.extractfile(member).read()
+        # Open the tar per thread to avoid concurrent reads on a shared TarFile handle.
+        with tarfile.open(batch_tar_path, 'r') as tar:
+            extracted = tar.extractfile(member_name)
+            if extracted is None:
+                return
+            content = extracted.read()
 
         # Handle decompression based on filename
         if relative_path.endswith('.gz'):
@@ -285,4 +292,4 @@ def _write_submission_to_tar(self, tar_handle, submission, documents, compressio
 
             tarinfo = tarfile.TarInfo(name=f'{accession_prefix}/{filename}')
             tarinfo.size = len(content)
-            tar_handle.addfile(tarinfo, io.BytesIO(content))
+            tar_handle.addfile(tarinfo, io.BytesIO(content))

datamule/datamule/sec/submissions/downloader.py

@@ -4,7 +4,7 @@
 from ...helper import _process_cik_and_metadata_filters
 import tarfile
 import io
-from secsgml2.parse_sgml import parse_sgml_content_into_memory
+from secsgml2 import parse_sgml_content_into_memory
 from secsgml2.utils import calculate_documents_locations_in_tar
 
 
@@ -50,10 +50,10 @@ def write_sgml_file_to_tar(output_path, bytes_content=None, input_path=None,filt
 
         with open(input_path, 'rb') as f:
             bytes_content = f.read()
-            metadata, documents = parse_sgml_content_into_memory(bytes_content=bytes_content,filter_document_types=filter_document_types)
+            metadata, documents = parse_sgml_content_into_memory(data=bytes_content,filter_document_types=filter_document_types)
     else:
         # Use content directly
-        metadata, documents = parse_sgml_content_into_memory(bytes_content=bytes_content, filter_document_types=filter_document_types)
+        metadata, documents = parse_sgml_content_into_memory(data=bytes_content, filter_document_types=filter_document_types)
 
     write_submission_to_tar(output_path,metadata,documents)
 

datamule/datamule/submission/submission.py

@@ -117,12 +117,13 @@ def __init__(self, path=None, sgml_content=None, keep_document_types=None,
                     raise ValueError(f"URL: {url}, Error: {response.getcode()}")
 
             self.path = None
-            metadata, raw_documents = parse_sgml_content_into_memory(sgml_content)
+            metadata, raw_documents = parse_sgml_content_into_memory(data=sgml_content)
 
 
 
             self.metadata = Document(type='submission_metadata', content=metadata, extension='.json',filing_date=None,accession=None,path=None)
-            self.filing_date= f"{self.metadata.content['filing-date'][:4]}-{self.metadata.content['filing-date'][4:6]}-{self.metadata.content['filing-date'][6:8]}"
+            fd = self.metadata.content.get('filing-date')
+            self.filing_date = f"{fd[:4]}-{fd[4:6]}-{fd[6:8]}" if fd else None
 
             self.documents_obj_list = []
             filtered_metadata_documents = []
@@ -158,7 +159,8 @@ def __init__(self, path=None, sgml_content=None, keep_document_types=None,
 
             self.metadata = Document(type='submission_metadata', content=metadata, extension='.json',filing_date=None,accession=None,path=metadata_path)
 
-            self.filing_date= f"{self.metadata.content['filing-date'][:4]}-{self.metadata.content['filing-date'][4:6]}-{self.metadata.content['filing-date'][6:8]}"
+            fd = self.metadata.content.get('filing-date')
+            self.filing_date = f"{fd[:4]}-{fd[4:6]}-{fd[6:8]}" if fd else None
 
         elif path is not None:
             self.path = Path(path)  
@@ -177,7 +179,8 @@ def __init__(self, path=None, sgml_content=None, keep_document_types=None,
                     metadata = json.load(f) 
 
             self.metadata = Document(type='submission_metadata', content=metadata, extension='.json',filing_date=None,accession=None,path=metadata_path)
-            self.filing_date= f"{self.metadata.content['filing-date'][:4]}-{self.metadata.content['filing-date'][4:6]}-{self.metadata.content['filing-date'][6:8]}"
+            fd = self.metadata.content.get('filing-date')
+            self.filing_date = f"{fd[:4]}-{fd[4:6]}-{fd[6:8]}" if fd else None
 
 
         # booleans

datamule/setup.py

@@ -32,7 +32,7 @@
 setup(
     name="datamule",
     author="John Friedman",
-    version="3.3.0",
+    version="3.4.0",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",