From 3d413f252052c00eee311e2dc3edfc9079958203 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 14 Apr 2026 17:40:59 +0530 Subject: [PATCH 1/3] Optimize V3 API Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 253 ++++++++++++++++++--------- vulnerabilities/models.py | 42 +++-- vulnerabilities/tests/test_api_v3.py | 2 +- 3 files changed, 202 insertions(+), 95 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index c17202f25..a0cb24c91 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from collections import defaultdict from typing import List from urllib.parse import urlencode @@ -21,6 +22,7 @@ from rest_framework.reverse import reverse from rest_framework.throttling import AnonRateThrottle +from vulnerabilities.models import AdvisoryAlias from vulnerabilities.models import AdvisoryReference from vulnerabilities.models import AdvisorySet from vulnerabilities.models import AdvisorySetMember @@ -216,6 +218,26 @@ def get_fixing_vulnerabilities_url(self, obj): def get_affected_by_vulnerabilities(self, package): """Return a dictionary with advisory as keys and their details, including fixed_by_packages.""" + advisories = self.context["advisory_map"].get(package.id, []) + impact_map = self.context["impact_map"].get(package.id, {}) + + if advisories: + result = [] + + for adv in advisories: + fixed = impact_map.get(adv["avid"]) + if not fixed: + continue + + result.append( + { + **adv, + "fixed_by_packages": fixed, + } + ) + + return result + advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url) advisories = [] @@ -250,56 +272,35 @@ def get_affected_by_vulnerabilities(self, package): "advisory_id": advisory.advisory_id.split("/")[-1], "aliases": [alias.alias for alias in advisory.aliases.all()], "summary": advisory.summary, - "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], "severity": advisory.weighted_severity, "exploitability": advisory.exploitability, "risk_score": advisory.risk_score, + "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], } ) return result - is_grouped = AdvisorySet.objects.filter(package=package, relation_type="affecting").exists() - - if is_grouped: - affected_by_advisories_qs = ( - AdvisorySet.objects.filter(package=package, relation_type="affecting") - .select_related("primary_advisory") - .prefetch_related( - Prefetch( - "members", - queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( - "advisory" - ), - to_attr="secondary_members", - ) + if not advisories: + if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + advisories_qs = advisories_qs.prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", ) - ) - - affected_groups = [ - Group( - aliases=list(adv.aliases.all()), - primary=adv.primary_advisory, - secondaries=[member.advisory for member in adv.secondary_members], + advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( + package, advisories_qs, "affecting" ) - for adv in affected_by_advisories_qs - ] + return self.return_advisories_data(package, advisories_qs, advisories) - advisories: List[GroupedAdvisory] = get_advisories_from_groups(affected_groups) - return self.return_advisories_data(package, advisories_qs, advisories) + def get_fixing_vulnerabilities(self, package): + fixing_advisories = AdvisorySet.objects.filter( + package=package, relation_type="fixing" + ).values_list("primary_advisory__advisory_id", flat=True) - if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: - advisories_qs = advisories_qs.prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) - advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( - package, advisories_qs, "affecting" - ) - return self.return_advisories_data(package, advisories_qs, advisories) + if fixing_advisories: + return [{"advisory_id": adv_id.split("/")[-1]} for adv_id in fixing_advisories] - def get_fixing_vulnerabilities(self, package): advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) if not package.type in TYPES_WITH_MULTIPLE_IMPORTERS: @@ -319,37 +320,6 @@ def get_fixing_vulnerabilities(self, package): ) return results - advisories = [] - - is_grouped = AdvisorySet.objects.filter(package=package, relation_type="fixing").exists() - - if is_grouped: - fixing_advisories_qs = ( - AdvisorySet.objects.filter(package=package, relation_type="fixing") - .select_related("primary_advisory") - .prefetch_related( - Prefetch( - "members", - queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( - "advisory" - ), - to_attr="secondary_members", - ) - ) - ) - - fixing_groups = [ - Group( - aliases=list(adv.aliases.all()), - primary=adv.primary_advisory, - secondaries=[member.advisory for member in adv.secondary_members], - ) - for adv in fixing_advisories_qs - ] - - advisories: List[GroupedAdvisory] = get_advisories_from_groups(fixing_groups) - return self.return_fixing_advisories_data(advisories) - if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: advisories_qs = advisories_qs.prefetch_related( "aliases", @@ -409,11 +379,11 @@ def return_advisories_data(self, package, advisories_qs, advisories): return result def get_next_non_vulnerable_version(self, package): - if next_non_vulnerable := package.get_non_vulnerable_versions()[0]: + if next_non_vulnerable := package.next_non_vulnerable_version: return next_non_vulnerable.version def get_latest_non_vulnerable_version(self, package): - if latest_non_vulnerable := package.get_non_vulnerable_versions()[-1]: + if latest_non_vulnerable := package.latest_non_vulnerable_version: return latest_non_vulnerable.version @@ -464,13 +434,11 @@ def create(self, request, *args, **kwargs): query = ( PackageV2.objects.filter(plain_package_url__in=plain_purls) .values_list("plain_package_url", flat=True) - .distinct() .order_by("plain_package_url") ) else: query = ( PackageV2.objects.filter(package_url__in=purls) - .distinct() .order_by("package_url") .values_list("package_url", flat=True) ) @@ -479,20 +447,20 @@ def create(self, request, *args, **kwargs): return self.get_paginated_response(page) if ignore_qualifiers_subpath: - query = ( - PackageV2.objects.filter(plain_package_url__in=plain_purls) - .order_by("plain_package_url") - .distinct("plain_package_url") + query = PackageV2.objects.filter(plain_package_url__in=plain_purls).order_by( + "plain_package_url" ) else: - query = ( - PackageV2.objects.filter(package_url__in=purls) - .order_by("package_url") - .distinct("package_url") - ) + query = PackageV2.objects.filter(package_url__in=purls).order_by("package_url") page = self.paginate_queryset(query) - serializer = self.get_serializer(page, many=True, context={"request": request}) + advisory_map = get_grouped_advisories_bulk(page) + impact_map = get_impacts_bulk(page) + serializer = self.get_serializer( + page, + many=True, + context={"request": request, "advisory_map": advisory_map, "impact_map": impact_map}, + ) return self.get_paginated_response(serializer.data) @@ -592,3 +560,124 @@ class FixingAdvisoriesViewSet(PackageAdvisoriesViewSet): class AffectedByAdvisoriesViewSet(PackageAdvisoriesViewSet): relation = "impacted_packages__affecting_packages__package_url" serializer_class = AffectedByAdvisoryV3Serializer + + +def get_grouped_advisories_bulk(packages): + package_ids = [p.id for p in packages] + + advisory_sets = list( + AdvisorySet.objects.filter( + package_id__in=package_ids, + relation_type="affecting", + ) + .select_related("primary_advisory", "package") + .prefetch_related( + Prefetch("aliases", queryset=AdvisoryAlias.objects.only("alias")), + Prefetch( + "members", + queryset=AdvisorySetMember.objects.filter(is_primary=False) + .select_related("advisory") + .only( + "advisory__avid", + "advisory__weighted_severity", + "advisory__exploitability", + ), + to_attr="secondary_members", + ), + ) + .only( + "id", + "package_id", + "primary_advisory__avid", + "primary_advisory__summary", + "primary_advisory__weighted_severity", + "primary_advisory__exploitability", + "primary_advisory__advisory_id", + ) + ) + + package_map = defaultdict(list) + for adv in advisory_sets: + adv._aliases_cache = [a.alias for a in adv.aliases.all()] + package_map[adv.package_id].append(adv) + + result = {} + + for package in packages: + groups = package_map.get(package.id, []) + grouped = [] + + for adv in groups: + primary = adv.primary_advisory + secondaries = [m.advisory for m in adv.secondary_members] + + max_sev = primary.weighted_severity or 0.0 + max_exp = primary.exploitability or 0.0 + + for sec in secondaries: + if sec.weighted_severity: + max_sev = max(max_sev, sec.weighted_severity) + if sec.exploitability: + max_exp = max(max_exp, sec.exploitability) + + weighted_severity = round(max_sev, 1) if max_sev else None + exploitability = max_exp or None + + risk_score = None + if exploitability and weighted_severity: + risk_score = round(min(exploitability * weighted_severity, 10.0), 1) + + identifier = primary.advisory_id.split("/")[-1] + + aliases = [a for a in adv._aliases_cache if a != identifier] + + grouped.append( + { + "avid": primary.avid, + "advisory_id": identifier, + "aliases": aliases, + "weighted_severity": weighted_severity, + "exploitability": exploitability, + "risk_score": risk_score, + "summary": primary.summary, + } + ) + + result[package.id] = grouped + + return result + + +def get_impacts_bulk(packages): + package_ids = [p.id for p in packages] + + impacts = ( + ImpactedPackageAffecting.objects.filter(package_id__in=package_ids) + .select_related("impacted_package__advisory") + .prefetch_related( + Prefetch( + "impacted_package__fixed_by_packages", + queryset=PackageV2.objects.only("package_url"), + ) + ) + .only( + "package_id", + "impacted_package_id", + "impacted_package__advisory_id", + "impacted_package__advisory__avid", + ) + ) + + impact_map = defaultdict(dict) + fixed_cache = {} + + for impact in impacts: + ip = impact.impacted_package + avid = ip.advisory.avid + + if ip.id not in fixed_cache: + fixed_cache[ip.id] = list({pkg.purl for pkg in ip.fixed_by_packages.all()}) + + impact_map[impact.package_id][avid] = fixed_cache[ip.id] + + return impact_map diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 896da7c76..f7b6f75ee 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2907,6 +2907,13 @@ def latest_affecting_advisories_for_purls(self, purls): ) return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() + def latest_affecting_advisories_for_packages(self, purls): + adv_ids = ImpactedPackageAffecting.objects.filter(package__in=purls).values_list( + "impacted_package__advisory_id", + flat=True, + ) + return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() + def latest_fixed_by_advisories_for_purl(self, purl): adv_ids = ImpactedPackageFixedBy.objects.filter(package__package_url=purl).values_list( "impacted_package__advisory_id", @@ -3577,25 +3584,36 @@ def calculate_version_rank(self): PackageV2.objects.bulk_update(sorted_packages, fields=["version_rank"]) return self.version_rank - def get_non_vulnerable_versions(self): + @cached_property + def _non_vulnerable_versions(self): """ - Return a tuple of the next and latest non-vulnerable versions as Package instance. - Return a tuple of (None, None) if there is no non-vulnerable version. + Cached computation to avoid duplicate queries. + Returns (next, latest) """ if self.version_rank == 0: self.calculate_version_rank - non_vulnerable_versions = PackageV2.objects.get_fixed_by_package_versions( - self, fix=False - ).only_non_vulnerable() - later_non_vulnerable = non_vulnerable_versions.filter( - version_rank__gte=self.version_rank - ).order_by("version_rank") + qs = ( + PackageV2.objects.get_fixed_by_package_versions(self, fix=False) + .only_non_vulnerable() + .filter(version_rank__gt=self.version_rank) + .order_by("version_rank") + ) - if later_non_vulnerable.exists(): - return later_non_vulnerable.first(), later_non_vulnerable.last() + next_non_vulnerable = qs.first() + latest_non_vulnerable = qs.last() - return None, None + return next_non_vulnerable, latest_non_vulnerable + + @property + def next_non_vulnerable_version(self): + next_nv, _ = self._non_vulnerable_versions + return next_nv if next_nv else None + + @property + def latest_non_vulnerable_version(self): + _, latest_nv = self._non_vulnerable_versions + return latest_nv if latest_nv else None @cached_property def version_class(self): diff --git a/vulnerabilities/tests/test_api_v3.py b/vulnerabilities/tests/test_api_v3.py index 137692abf..84e1cf94c 100644 --- a/vulnerabilities/tests/test_api_v3.py +++ b/vulnerabilities/tests/test_api_v3.py @@ -66,7 +66,7 @@ def test_packages_post_without_details(self): def test_packages_post_with_details(self): url = reverse("package-v3-list") - with self.assertNumQueries(33): + with self.assertNumQueries(34): response = self.client.post( url, data={ From c9552b433ce28afd4868470a4f360b7f2114f4d8 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 14 Apr 2026 18:31:47 +0530 Subject: [PATCH 2/3] Optimize get non vulnerable versions Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 97 ++++++++++++++++++++++++--------------- vulnerabilities/models.py | 6 +-- vulnerabilities/views.py | 2 +- 3 files changed, 63 insertions(+), 42 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index a0cb24c91..ee62ea1d5 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -12,6 +12,7 @@ from urllib.parse import urlencode from django.db.models import Exists +from django.db.models import Max from django.db.models import OuterRef from django.db.models import Prefetch from django_filters import rest_framework as filters @@ -226,8 +227,7 @@ def get_affected_by_vulnerabilities(self, package): for adv in advisories: fixed = impact_map.get(adv["avid"]) - if not fixed: - continue + adv.pop("avid", None) result.append( { @@ -294,12 +294,9 @@ def get_affected_by_vulnerabilities(self, package): return self.return_advisories_data(package, advisories_qs, advisories) def get_fixing_vulnerabilities(self, package): - fixing_advisories = AdvisorySet.objects.filter( - package=package, relation_type="fixing" - ).values_list("primary_advisory__advisory_id", flat=True) - - if fixing_advisories: - return [{"advisory_id": adv_id.split("/")[-1]} for adv_id in fixing_advisories] + advisories = self.context["fixing_advisory_map"].get(package.id, []) + if advisories: + return advisories advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) @@ -326,6 +323,8 @@ def get_fixing_vulnerabilities(self, package): "impacted_packages__affecting_packages", "impacted_packages__fixed_by_packages", ) + if not advisories_qs.exists(): + return [] advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( package, advisories_qs, "fixing" ) @@ -454,12 +453,13 @@ def create(self, request, *args, **kwargs): query = PackageV2.objects.filter(package_url__in=purls).order_by("package_url") page = self.paginate_queryset(query) - advisory_map = get_grouped_advisories_bulk(page) + affected_advisory_map = get_affected_advisories_bulk(page) + fixing_advisory_map = get_fixing_advisories_bulk(page) impact_map = get_impacts_bulk(page) serializer = self.get_serializer( page, many=True, - context={"request": request, "advisory_map": advisory_map, "impact_map": impact_map}, + context={"request": request, "advisory_map": affected_advisory_map, "impact_map": impact_map, "fixing_advisory_map": fixing_advisory_map}, ) return self.get_paginated_response(serializer.data) @@ -562,7 +562,7 @@ class AffectedByAdvisoriesViewSet(PackageAdvisoriesViewSet): serializer_class = AffectedByAdvisoryV3Serializer -def get_grouped_advisories_bulk(packages): +def get_affected_advisories_bulk(packages): package_ids = [p.id for p in packages] advisory_sets = list( @@ -570,19 +570,14 @@ def get_grouped_advisories_bulk(packages): package_id__in=package_ids, relation_type="affecting", ) - .select_related("primary_advisory", "package") - .prefetch_related( - Prefetch("aliases", queryset=AdvisoryAlias.objects.only("alias")), - Prefetch( - "members", - queryset=AdvisorySetMember.objects.filter(is_primary=False) - .select_related("advisory") - .only( - "advisory__avid", - "advisory__weighted_severity", - "advisory__exploitability", - ), - to_attr="secondary_members", + .select_related("primary_advisory") + .prefetch_related(Prefetch("aliases", queryset=AdvisoryAlias.objects.only("alias"))) + .annotate( + max_severity=Max( + "members__advisory__weighted_severity", + ), + max_exploitability=Max( + "members__advisory__exploitability", ), ) .only( @@ -590,13 +585,12 @@ def get_grouped_advisories_bulk(packages): "package_id", "primary_advisory__avid", "primary_advisory__summary", - "primary_advisory__weighted_severity", - "primary_advisory__exploitability", "primary_advisory__advisory_id", ) ) package_map = defaultdict(list) + for adv in advisory_sets: adv._aliases_cache = [a.alias for a in adv.aliases.all()] package_map[adv.package_id].append(adv) @@ -609,23 +603,14 @@ def get_grouped_advisories_bulk(packages): for adv in groups: primary = adv.primary_advisory - secondaries = [m.advisory for m in adv.secondary_members] - max_sev = primary.weighted_severity or 0.0 - max_exp = primary.exploitability or 0.0 - - for sec in secondaries: - if sec.weighted_severity: - max_sev = max(max_sev, sec.weighted_severity) - if sec.exploitability: - max_exp = max(max_exp, sec.exploitability) + max_sev = adv.max_severity or 0.0 + max_exp = adv.max_exploitability or 0.0 weighted_severity = round(max_sev, 1) if max_sev else None exploitability = max_exp or None - risk_score = None - if exploitability and weighted_severity: - risk_score = round(min(exploitability * weighted_severity, 10.0), 1) + risk_score = round(min(max_exp * max_sev, 10.0), 1) if max_exp and max_sev else None identifier = primary.advisory_id.split("/")[-1] @@ -681,3 +666,39 @@ def get_impacts_bulk(packages): impact_map[impact.package_id][avid] = fixed_cache[ip.id] return impact_map + + +def get_fixing_advisories_bulk(packages): + package_ids = [p.id for p in packages] + + advisory_sets = list( + AdvisorySet.objects.filter( + package_id__in=package_ids, + relation_type="fixing", + ) + .only( + "id", + "package_id", + "primary_advisory__advisory_id", + ) + ) + + package_map = defaultdict(list) + + for adv in advisory_sets: + package_map[adv.package_id].append(adv.primary_advisory.advisory_id) + + result = {} + + for package in packages: + groups = package_map.get(package.id, []) + grouped = [] + + for adv_id in groups: + grouped.append( + {"advisory_id": adv_id.split("/")[-1]} + ) + + result[package.id] = grouped + + return result diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index f7b6f75ee..4b6c17627 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -3585,7 +3585,7 @@ def calculate_version_rank(self): return self.version_rank @cached_property - def _non_vulnerable_versions(self): + def get_non_vulnerable_versions(self): """ Cached computation to avoid duplicate queries. Returns (next, latest) @@ -3607,12 +3607,12 @@ def _non_vulnerable_versions(self): @property def next_non_vulnerable_version(self): - next_nv, _ = self._non_vulnerable_versions + next_nv, _ = self.get_non_vulnerable_versions return next_nv if next_nv else None @property def latest_non_vulnerable_version(self): - _, latest_nv = self._non_vulnerable_versions + _, latest_nv = self.get_non_vulnerable_versions return latest_nv if latest_nv else None @cached_property diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 5b9406f87..371dcd217 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -257,7 +257,7 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) package = self.object - next_non_vulnerable, latest_non_vulnerable = package.get_non_vulnerable_versions() + next_non_vulnerable, latest_non_vulnerable = package.get_non_vulnerable_versions context["package"] = package context["next_non_vulnerable"] = next_non_vulnerable From d83fce384e6de4103d7e45970007ac306099a447 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 14 Apr 2026 18:33:05 +0530 Subject: [PATCH 3/3] Fix tests Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 14 ++++++++------ vulnerabilities/tests/test_api_v3.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index ee62ea1d5..12f10ed1c 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -459,7 +459,12 @@ def create(self, request, *args, **kwargs): serializer = self.get_serializer( page, many=True, - context={"request": request, "advisory_map": affected_advisory_map, "impact_map": impact_map, "fixing_advisory_map": fixing_advisory_map}, + context={ + "request": request, + "advisory_map": affected_advisory_map, + "impact_map": impact_map, + "fixing_advisory_map": fixing_advisory_map, + }, ) return self.get_paginated_response(serializer.data) @@ -675,8 +680,7 @@ def get_fixing_advisories_bulk(packages): AdvisorySet.objects.filter( package_id__in=package_ids, relation_type="fixing", - ) - .only( + ).only( "id", "package_id", "primary_advisory__advisory_id", @@ -695,9 +699,7 @@ def get_fixing_advisories_bulk(packages): grouped = [] for adv_id in groups: - grouped.append( - {"advisory_id": adv_id.split("/")[-1]} - ) + grouped.append({"advisory_id": adv_id.split("/")[-1]}) result[package.id] = grouped diff --git a/vulnerabilities/tests/test_api_v3.py b/vulnerabilities/tests/test_api_v3.py index 84e1cf94c..be4b1d923 100644 --- a/vulnerabilities/tests/test_api_v3.py +++ b/vulnerabilities/tests/test_api_v3.py @@ -66,7 +66,7 @@ def test_packages_post_without_details(self): def test_packages_post_with_details(self): url = reverse("package-v3-list") - with self.assertNumQueries(34): + with self.assertNumQueries(31): response = self.client.post( url, data={