Skip to content

Commit 61cc52a

Browse files
amindadgaramindadgar
authored
Merge pull request #28 from TogetherCrew/feat/27-mediawiki-module-update
feat: update mediaWiki platform & modules schema!
2 parents d6e9aa5 + b5341fc commit 61cc52a

File tree

4 files changed

+110
-8
lines changed

4 files changed

+110
-8
lines changed

‎hivemind_etl/mediawiki/module.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,16 +62,26 @@ def get_learning_platforms(
6262
platform_id=platform_id,
6363
metadata_name="path",
6464
)
65+
namespaces = self.get_platform_metadata(
66+
platform_id=platform_id,
67+
metadata_name="namespaces",
68+
)
6569

6670
if not isinstance(path, str) and not isinstance(base_url, str):
6771
raise ValueError("Wrong format for `path` and `base_url`!")
6872

6973
modules_options = platform["metadata"]
70-
namespaces = modules_options.get("namespaces", [])
74+
activated = modules_options["activated"]
75+
76+
if not activated:
77+
logging.warning(
78+
f"Platform: {platform_id} is not activated! Skipping it..."
79+
)
80+
continue
7181

7282
if not namespaces:
7383
logging.warning(
74-
f"No namespaces found for platform: {platform_id}"
84+
f"No namespaces found for platform: {platform_id}. Skipping it..."
7585
)
7686
continue
7787

‎requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
python-dotenv>=1.0.0, <2.0.0
2-
tc-hivemind-backend==1.4.2.post2
2+
tc-hivemind-backend==1.4.3
33
llama-index-storage-docstore-redis==0.1.2
44
llama-index-storage-docstore-mongodb==0.1.3
55
crawlee[playwright]==0.3.8

‎tests/integration/test_mediawiki_modules.py‎

Lines changed: 93 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,12 @@ def test_get_single_data(self):
4848
"platform": platform_id,
4949
"name": "mediaWiki",
5050
"metadata": {
51-
"namespaces": [0, 1, 2],
51+
"activated": True,
5252
},
5353
}
5454
]
5555
},
56+
"activated": True,
5657
}
5758
)
5859

@@ -85,18 +86,19 @@ def test_get_mediawiki_communities_data_multiple_platforms(self):
8586
"platform": platform_id1,
8687
"name": "mediaWiki",
8788
"metadata": {
88-
"namespaces": [0, 1, 2],
89+
"activated": True,
8990
},
9091
},
9192
{
9293
"platform": platform_id2,
9394
"name": "mediaWiki",
9495
"metadata": {
95-
"namespaces": [3, 4, 5],
96+
"activated": True,
9697
},
9798
},
9899
]
99100
},
101+
"activated": True,
100102
}
101103
)
102104

@@ -107,6 +109,7 @@ def test_get_mediawiki_communities_data_multiple_platforms(self):
107109
"metadata": {
108110
"baseURL": "http://example1.com",
109111
"path": "/api",
112+
"namespaces": [0, 1, 2],
110113
},
111114
"community": community_id,
112115
"disconnectedAt": None,
@@ -123,6 +126,7 @@ def test_get_mediawiki_communities_data_multiple_platforms(self):
123126
"metadata": {
124127
"baseURL": "http://example2.com",
125128
"path": "/api",
129+
"namespaces": [3, 4, 5],
126130
},
127131
"community": community_id,
128132
"disconnectedAt": None,
@@ -171,18 +175,19 @@ def test_get_mediawiki_communities_data_filtered_platforms(self):
171175
"platform": platform_id1,
172176
"name": "mediaWiki",
173177
"metadata": {
174-
"namespaces": [0, 1, 2],
178+
"activated": True,
175179
},
176180
},
177181
{
178182
"platform": platform_id2,
179183
"name": "mediaWiki",
180184
"metadata": {
181-
"namespaces": [3, 4, 5],
185+
"activated": True,
182186
},
183187
},
184188
]
185189
},
190+
"activated": True,
186191
}
187192
)
188193

@@ -193,6 +198,7 @@ def test_get_mediawiki_communities_data_filtered_platforms(self):
193198
"metadata": {
194199
"baseURL": "http://example1.com",
195200
"path": "/api",
201+
"namespaces": [0, 1, 2],
196202
},
197203
"community": community_id,
198204
"disconnectedAt": None,
@@ -209,6 +215,7 @@ def test_get_mediawiki_communities_data_filtered_platforms(self):
209215
"metadata": {
210216
"baseURL": "http://example2.com",
211217
"path": "/api",
218+
"namespaces": [3, 4, 5],
212219
},
213220
"community": community_id,
214221
"disconnectedAt": None,
@@ -232,3 +239,84 @@ def test_get_mediawiki_communities_data_filtered_platforms(self):
232239
"base_url": "http://example1.com/api",
233240
},
234241
)
242+
243+
def test_get_mediawiki_communities_data_filtered_platforms_not_activated(self):
244+
"""
245+
Two mediawiki platforms for one community
246+
"""
247+
platform_id1 = ObjectId("6579c364f1120850414e0dc6")
248+
platform_id2 = ObjectId("6579c364f1120850414e0dc7")
249+
community_id = ObjectId("1009c364f1120850414e0dc5")
250+
251+
self.client["Core"]["modules"].insert_one(
252+
{
253+
"name": "hivemind",
254+
"community": community_id,
255+
"options": {
256+
"platforms": [
257+
{
258+
"platform": platform_id1,
259+
"name": "mediaWiki",
260+
"metadata": {
261+
"activated": False,
262+
},
263+
},
264+
{
265+
"platform": platform_id2,
266+
"name": "mediaWiki",
267+
"metadata": {
268+
"activated": True,
269+
},
270+
},
271+
]
272+
},
273+
"activated": True,
274+
}
275+
)
276+
277+
self.client["Core"]["platforms"].insert_one(
278+
{
279+
"_id": platform_id1,
280+
"name": "mediaWiki",
281+
"metadata": {
282+
"baseURL": "http://example1.com",
283+
"path": "/api",
284+
"namespaces": [0, 1, 2],
285+
},
286+
"community": community_id,
287+
"disconnectedAt": None,
288+
"connectedAt": datetime.now(),
289+
"createdAt": datetime.now(),
290+
"updatedAt": datetime.now(),
291+
}
292+
)
293+
294+
self.client["Core"]["platforms"].insert_one(
295+
{
296+
"_id": platform_id2,
297+
"name": "mediaWiki",
298+
"metadata": {
299+
"baseURL": "http://example2.com",
300+
"path": "/api",
301+
"namespaces": [3, 4, 5],
302+
},
303+
"community": community_id,
304+
"disconnectedAt": None,
305+
"connectedAt": datetime.now(),
306+
"createdAt": datetime.now(),
307+
"updatedAt": datetime.now(),
308+
}
309+
)
310+
311+
result = self.modules_mediawiki.get_learning_platforms()
312+
313+
self.assertIsInstance(result, list)
314+
self.assertEqual(len(result), 1)
315+
self.assertEqual(
316+
result[0],
317+
{
318+
"community_id": str(community_id),
319+
"namespaces": [3, 4, 5],
320+
"base_url": "http://example2.com/api",
321+
},
322+
)

‎tests/integration/test_module_website.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ def test_get_website_communities_data_single_modules(self):
5454
}
5555
]
5656
},
57+
"activated": True,
5758
}
5859
)
5960

@@ -115,6 +116,7 @@ def test_get_website_communities_data_module_multiple_platforms(self):
115116
},
116117
]
117118
},
119+
"activated": True,
118120
}
119121
)
120122

@@ -191,6 +193,7 @@ def test_get_website_communities_data_module_multiple_platforms_filtered_one(sel
191193
},
192194
]
193195
},
196+
"activated": True,
194197
}
195198
)
196199

@@ -269,6 +272,7 @@ def test_get_website_communities_data_module_multiple_platforms_one_disabled(sel
269272
},
270273
]
271274
},
275+
"activated": True,
272276
}
273277
)
274278

0 commit comments

Comments
 (0)