Made the changes required Signed-off-by: Shrish Mishra shrish409@gmail.com

Signed-off-by: Shrish0098 <shrish409@gmail.com>

Author: Dedsec0098

vulnerabilities/models.py

@@ -74,20 +74,6 @@ def get_or_none(self, *args, **kwargs):
         with suppress(self.model.DoesNotExist, ValidationError):
             return self.get(*args, **kwargs)
 
-    def paginated(self, per_page=5000):
-        """
-        Iterate over a (large) QuerySet by chunks of ``per_page`` items.
-        This technique is essential for preventing memory issues when iterating
-        See these links for inspiration:
-        https://nextlinklabs.com/resources/insights/django-big-data-iteration
-        https://stackoverflow.com/questions/4222176/why-is-iterating-through-a-large-django-queryset-consuming-massive-amounts-of-me/
-        """
-        paginator = Paginator(self, per_page=per_page)
-        for page_number in paginator.page_range:
-            page = paginator.page(page_number)
-            for obj in page.object_list:
-                yield obj
-
 
 class VulnerabilityQuerySet(BaseQuerySet):
     def affecting_vulnerabilities(self):

vulnerabilities/pipelines/add_cvss31_to_CVEs.py

@@ -44,10 +44,9 @@ def process_cve_advisory_mapping(self):
             progress_step=5,
         )
 
-        batch_size = 1000
         results = []
 
-        for severity in progress.iter(nvd_severities.iterator()):
+        for severity in progress.iter(nvd_severities.iterator(chunk_size=2000)):
             print(severity.url)
             cve_pattern = re.compile(r"(CVE-\d{4}-\d{4,7})").search
             cve_match = cve_pattern(severity.url)

vulnerabilities/pipelines/collect_commits.py

@@ -51,7 +51,9 @@ def collect_and_store_fix_commits(self):
             total_iterations=affected_by_package_related_vulnerabilities.count(), logger=self.log
         )
 
-        for apv in progress.iter(affected_by_package_related_vulnerabilities.iterator()):
+        for apv in progress.iter(
+            affected_by_package_related_vulnerabilities.iterator(chunk_size=2000)
+        ):
             vulnerability = apv.vulnerability
             for reference in vulnerability.references.all():
                 if not "/commit/" in reference.url:

vulnerabilities/pipelines/compute_package_risk.py

@@ -54,7 +54,7 @@ def compute_and_store_vulnerability_risk_score(self):
         updated_vulnerability_count = 0
         batch_size = 5000
 
-        for vulnerability in progress.iter(affected_vulnerabilities.iterator()):
+        for vulnerability in progress.iter(affected_vulnerabilities.iterator(chunk_size=2000)):
             severities = vulnerability.severities.all()
             references = vulnerability.references.all()
             exploits = vulnerability.exploits.all()
@@ -110,7 +110,7 @@ def compute_and_store_package_risk_score(self):
         updated_package_count = 0
         batch_size = 10000
 
-        for package in progress.iter(affected_packages.iterator()):
+        for package in progress.iter(affected_packages.iterator(chunk_size=2000)):
             risk_score = compute_package_risk(package)
 
             if not risk_score: