Fix/parsing logic error for sax

src/scribe_data/wikipedia/extract_wiki.py

@@ -463,3 +463,64 @@ def __init__(self):
         self._values = {}
         self._current_tag = None
         self.target_articles = []
+
+    def startElement(self, name, attrs):
+        """
+        Handle the start of an XML element.
+
+        Parameters
+        ----------
+        name : str
+            The name of the XML element being opened.
+        attrs : xml.sax.xmlreader.AttributesImpl
+            The attributes associated with the element.
+        """
+        if name in ("title", "text", "timestamp"):
+            self._current_tag = name
+            self._buffer = []
+
+    def endElement(self, name):
+        """
+        Handle the end of an XML element.
+
+        Parameters
+        ----------
+        name : str
+            The name of the XML element being closed.
+        """
+        if name == self._current_tag:
+            self._values[name] = "".join(self._buffer)
+
+        if name == "page":
+            # Process the complete page
+            title = self._values.get("title", "")
+            text = self._values.get("text", "")
+
+            # Filter out redirect pages and special pages
+            if (
+                text
+                and not text.strip().startswith("#REDIRECT")
+                and not text.strip().startswith("#redirect")
+                and ":" not in title
+            ):  # Skip namespace pages
+                processed_title, processed_text = _process_article(title, text)
+
+                if processed_text and len(processed_text) > 100:  # Minimum text length
+                    self.target_articles.append([processed_title, processed_text])
+
+            # Reset values for next page, clear up for next page
+            self._values = {}
+            self._buffer = None
+            self._current_tag = None
+
+    def characters(self, content):
+        """
+        Handle character data within an XML element.
+
+        Parameters
+        ----------
+        content : str
+            The character data content from the XML element.
+        """
+        if self._current_tag:
+            self._buffer.append(content)

src/scribe_data/wikipedia/generate_autosuggestions.py

@@ -73,7 +73,7 @@ def generate_autosuggestions(language, dump_id, force_download):
         verbose=True,
     )
 
-    with open(output_path, "r") as fin:
+    with open(output_path, "r", encoding="utf-8") as fin:
         article_texts = [
             json.loads(lang)[1]
             for lang in tqdm(fin, desc="Articles added", unit="articles")
@@ -101,3 +101,13 @@ def generate_autosuggestions(language, dump_id, force_download):
         update_local_data=True,
         verbose=True,
     )
+
+
+# Uncomment to test
+"""if __name__ == "__main__":
+    generate_autosuggestions(
+        language="english",
+        dump_id="20250520",
+        force_download=False,
+        file_limit=1  # limiting test with just 1 file
+    )"""