Change batch size to 200 and enable multiprocessing

TG1999 · TG1999 · commit e36500bbd547 · 2025-02-26T20:20:17.000+05:30
Signed-off-by: Tushar Goel &lt;tushar.goel.dav@gmail.com&gt;
diff --git a/vulnerabilities/pipelines/recompute_content_ids.py b/vulnerabilities/pipelines/recompute_content_ids.py
@@ -94,8 +94,7 @@ def process_advisories(
     )
 
     log(f"Running function: {advisory_func.__name__}", level=logging.INFO)
-    # if max_workers <= 0:
-    if True:
+    if max_workers <= 0:
         log(f"Running function in single process", level=logging.INFO)
         for advisory_ids in progress.iter(advisory_batches):
             progress.log_progress()
diff --git a/vulnerabilities/pipelines/remove_duplicate_advisories.py b/vulnerabilities/pipelines/remove_duplicate_advisories.py
@@ -73,7 +73,7 @@ class RemoveDuplicateAdvisoriesPipeline(VulnerableCodePipeline):
     """Pipeline to remove duplicate advisories based on their content."""
 
     pipeline_id = "remove_duplicate_advisories"
-    BATCH_SIZE = 1000
+    BATCH_SIZE = 200
 
     @classmethod
     def steps(cls):

Original file line number	Diff line number	Diff line change
`@@ -94,8 +94,7 @@ def process_advisories(`
`94`	`94`	`)`
`95`	`95`
`96`	`96`	`log(f"Running function: {advisory_func.__name__}", level=logging.INFO)`
`97`		`- # if max_workers <= 0:`
`98`		`- if True:`
	`97`	`+ if max_workers <= 0:`
`99`	`98`	`log(f"Running function in single process", level=logging.INFO)`
`100`	`99`	`for advisory_ids in progress.iter(advisory_batches):`
`101`	`100`	`progress.log_progress()`