Merge pull request #23 from TogetherCrew/fix/mediawiki-activities-wrong-arg

feat: mediaWiki change dump dir!

Author: amindadgar

hivemind_etl/mediawiki/etl.py

@@ -25,7 +25,7 @@ def __init__(
             community_id, namespaces=namespaces, proxy_url=self.proxy_url
         )
 
-        self.dump_dir = f"dump_{self.community_id}"
+        self.dump_dir = f"dumps/{self.community_id}"
         self.delete_dump_after_load = delete_dump_after_load
 
     def extract(self, api_url: str, dump_dir: str | None = None) -> None:

tests/unit/test_mediawiki_etl.py

@@ -15,20 +15,21 @@ def setUp(self):
         self.namespaces = [0, 1]  # Main and Talk namespaces
 
         # Create a temporary dumps directory
-        os.makedirs(f"dump_{self.community_id}", exist_ok=True)
+        os.makedirs("dumps", exist_ok=True)
+        os.makedirs(f"dumps/{self.community_id}", exist_ok=True)
 
     def tearDown(self):
         # Clean up any created files
-        if os.path.exists(f"dump_{self.community_id}"):
-            shutil.rmtree(f"dump_{self.community_id}")
+        if os.path.exists(f"dumps/{self.community_id}"):
+            shutil.rmtree(f"dumps/{self.community_id}")
         if os.path.exists(self.custom_path):
             shutil.rmtree(self.custom_path)
 
     def test_mediawiki_etl_initialization(self):
         etl = MediawikiETL(community_id=self.community_id, namespaces=self.namespaces)
         self.assertEqual(etl.community_id, self.community_id)
         self.assertTrue(etl.delete_dump_after_load)
-        self.assertEqual(etl.dump_dir, f"dump_{self.community_id}")
+        self.assertEqual(etl.dump_dir, f"dumps/{self.community_id}")
 
         etl = MediawikiETL(
             community_id=self.community_id,
@@ -45,7 +46,7 @@ def test_extract_with_default_path(self):
         etl.extract(self.api_url)
 
         etl.wikiteam_crawler.crawl.assert_called_once_with(
-            self.api_url, f"dump_{self.community_id}"
+            self.api_url, f"dumps/{self.community_id}"
         )
 
     def test_extract_with_custom_path(self):