Merge pull request #24 from TogetherCrew/fix/mediawiki-activities-wrong-arg

feat: trying to fix the no doc_ref_id error on loading documents!

Author: amindadgar

hivemind_etl/mediawiki/etl.py

@@ -42,9 +42,11 @@ def transform(self) -> list[Document]:
         documents: list[Document] = []
         for page in pages:
             try:
+                # Generate a ref_doc_id if needed for newer llama-index versions
+                doc_id = page.page_id
                 documents.append(
                     Document(
-                        doc_id=page.page_id,
+                        doc_id=doc_id,
                         text=page.revision.text,
                         metadata={
                             "title": page.title,
@@ -57,6 +59,7 @@ def transform(self) -> list[Document]:
                             "contributor_user_id": page.revision.contributor.user_id,
                             "sha1": page.revision.sha1,
                             "model": page.revision.model,
+                            "ref_doc_id": doc_id,  # Add ref_doc_id to metadata
                         },
                         excluded_embed_metadata_keys=[
                             "namespace",

requirements.txt

@@ -1,5 +1,5 @@
 python-dotenv>=1.0.0, <2.0.0
-tc-hivemind-backend==1.4.0
+tc-hivemind-backend==1.4.2.post2
 llama-index-storage-docstore-redis==0.1.2
 llama-index-storage-docstore-mongodb==0.1.3
 crawlee[playwright]==0.3.8