diff --git a/vulnerabilities/api_v2.py b/vulnerabilities/api_v2.py
index c45dbfebe..ba41d0906 100644
--- a/vulnerabilities/api_v2.py
+++ b/vulnerabilities/api_v2.py
@@ -31,6 +31,7 @@
from vulnerabilities.models import AdvisoryWeakness
from vulnerabilities.models import CodeFix
from vulnerabilities.models import CodeFixV2
+from vulnerabilities.models import ImpactedPackage
from vulnerabilities.models import Package
from vulnerabilities.models import PackageV2
from vulnerabilities.models import PipelineRun
@@ -42,6 +43,10 @@
from vulnerabilities.throttling import PermissionBasedUserRateThrottle
+class CharInFilter(filters.BaseInFilter, filters.CharFilter):
+ pass
+
+
class WeaknessV2Serializer(serializers.ModelSerializer):
cwe_id = serializers.CharField()
name = serializers.CharField()
@@ -306,8 +311,8 @@ class AdvisoryPackageV2Serializer(serializers.ModelSerializer):
risk_score = serializers.FloatField(read_only=True)
affected_by_vulnerabilities = serializers.SerializerMethodField()
fixing_vulnerabilities = serializers.SerializerMethodField()
- next_non_vulnerable_version = serializers.CharField(read_only=True)
- latest_non_vulnerable_version = serializers.CharField(read_only=True)
+ next_non_vulnerable_version = serializers.SerializerMethodField()
+ latest_non_vulnerable_version = serializers.SerializerMethodField()
class Meta:
model = Package
@@ -320,36 +325,36 @@ class Meta:
"risk_score",
]
- def get_affected_by_vulnerabilities(self, obj):
- """
- Return a dictionary with vulnerabilities as keys and their details, including fixed_by_packages.
- """
+ def get_affected_by_vulnerabilities(self, package):
+ """Return a dictionary with advisory as keys and their details, including fixed_by_packages."""
result = {}
request = self.context.get("request")
- for adv in getattr(obj, "prefetched_affected_advisories", []):
- fixed_by_package = adv.fixed_by_packages.first()
- purl = None
- if fixed_by_package:
- purl = fixed_by_package.package_url
- # Get code fixed for a vulnerability
- code_fixes = CodeFixV2.objects.filter(advisory=adv).distinct()
+ for impact in package.affected_in_impacts.all():
+ advisory = impact.advisory
+ fixed_by_packages = [pkg.purl for pkg in impact.fixed_by_packages.all()]
+ code_fixes = CodeFixV2.objects.filter(advisory=advisory).distinct()
code_fix_urls = [
reverse("advisory-codefix-detail", args=[code_fix.id], request=request)
for code_fix in code_fixes
]
-
- result[adv.avid] = {
- "advisory_id": adv.avid,
- "fixed_by_packages": purl,
+ result[advisory.avid] = {
+ "advisory_id": advisory.avid,
+ "fixed_by_packages": fixed_by_packages,
"code_fixes": code_fix_urls,
}
+
return result
- def get_fixing_vulnerabilities(self, obj):
- # Ghost package should not fix any vulnerability.
- if obj.is_ghost:
- return []
- return [adv.avid for adv in obj.fixing_advisories.all()]
+ def get_fixing_vulnerabilities(self, package):
+ return [impact.advisory.avid for impact in package.fixed_in_impacts.all()]
+
+ def get_next_non_vulnerable_version(self, package):
+ if next_non_vulnerable := package.get_non_vulnerable_versions()[0]:
+ return next_non_vulnerable.version
+
+ def get_latest_non_vulnerable_version(self, package):
+ if latest_non_vulnerable := package.get_non_vulnerable_versions()[-1]:
+ return latest_non_vulnerable.version
class PackageurlListSerializer(serializers.Serializer):
@@ -381,9 +386,24 @@ class PackageV2FilterSet(filters.FilterSet):
class AdvisoryPackageV2FilterSet(filters.FilterSet):
- affected_by_vulnerability = filters.CharFilter(field_name="affected_by_advisory__advisory_id")
- fixing_vulnerability = filters.CharFilter(field_name="fixing_advisories__advisory_id")
- purl = filters.CharFilter(field_name="package_url")
+ affected_by_advisory = filters.CharFilter(
+ field_name="affected_in_impacts__advisory__avid",
+ label="Affected By Advisory ID",
+ help_text="Filter packages affected by a specific Advisory ID.",
+ )
+
+ fixing_advisory = filters.CharFilter(
+ field_name="fixed_in_impacts__advisory__avid",
+ label="Fixed By Advisory ID",
+ help_text="Filter packages fixed by a specific Advisory ID.",
+ )
+
+ purls = CharInFilter(
+ field_name="package_url",
+ lookup_expr="in",
+ label="Package URL",
+ help_text="Filter by one or more Package URLs. Multi-value supported (comma-separated).",
+ )
class PackageV2ViewSet(viewsets.ReadOnlyModelViewSet):
@@ -968,40 +988,39 @@ def get_view_name(self):
class AdvisoriesPackageV2ViewSet(viewsets.ReadOnlyModelViewSet):
- queryset = PackageV2.objects.all().prefetch_related(
- Prefetch(
- "affected_by_advisories",
- queryset=AdvisoryV2.objects.prefetch_related("fixed_by_packages"),
- to_attr="prefetched_affected_advisories",
- )
- )
+ queryset = PackageV2.objects.all()
serializer_class = AdvisoryPackageV2Serializer
- filter_backends = (filters.DjangoFilterBackend,)
+ filter_backends = [filters.DjangoFilterBackend]
filterset_class = AdvisoryPackageV2FilterSet
def get_queryset(self):
- queryset = super().get_queryset()
- package_purls = self.request.query_params.getlist("purl")
- affected_by_advisory = self.request.query_params.get("affected_by_advisory")
- fixing_advisory = self.request.query_params.get("fixing_advisory")
- if package_purls:
- queryset = queryset.filter(package_url__in=package_purls)
- if affected_by_advisory:
- queryset = queryset.filter(affected_by_advisories__advisory_id=affected_by_advisory)
- if fixing_advisory:
- queryset = queryset.filter(fixing_advisories__advisory=fixing_advisory)
- return queryset.with_is_vulnerable()
+ return (
+ super()
+ .get_queryset()
+ .prefetch_related(
+ Prefetch(
+ "affected_in_impacts",
+ queryset=ImpactedPackage.objects.select_related("advisory").prefetch_related(
+ "fixed_by_packages",
+ ),
+ ),
+ Prefetch(
+ "fixed_in_impacts",
+ queryset=ImpactedPackage.objects.select_related("advisory"),
+ ),
+ )
+ .with_is_vulnerable()
+ )
def list(self, request, *args, **kwargs):
- queryset = self.get_queryset()
- # Apply pagination
- page = self.paginate_queryset(queryset)
+ filtered_queryset = self.filter_queryset(self.get_queryset())
+ page = self.paginate_queryset(filtered_queryset)
+
+ advisories = set()
if page is not None:
- # Collect only vulnerabilities for packages in the current page
- advisories = set()
for package in page:
- advisories.update(package.affected_by_advisories.all())
- advisories.update(package.fixing_advisories.all())
+ advisories.update({impact.advisory for impact in package.affected_in_impacts.all()})
+ advisories.update({impact.advisory for impact in package.fixed_in_impacts.all()})
# Serialize the vulnerabilities with advisory_id and advisory label as keys
advisory_data = {f"{adv.avid}": AdvisoryV2Serializer(adv).data for adv in advisories}
@@ -1009,15 +1028,14 @@ def list(self, request, *args, **kwargs):
# Serialize the current page of packages
serializer = self.get_serializer(page, many=True)
data = serializer.data
- print(data)
+
# Use 'self.get_paginated_response' to include pagination data
return self.get_paginated_response({"advisories": advisory_data, "packages": data})
# If pagination is not applied, collect vulnerabilities for all packages
- advisories = set()
for package in queryset:
- advisories.update(package.affected_by_vulnerabilities.all())
- advisories.update(package.fixing_vulnerabilities.all())
+ advisories.update({impact.advisory for impact in package.affected_in_impacts.all()})
+ advisories.update({impact.advisory for impact in package.fixed_in_impacts.all()})
advisory_data = {f"{adv.avid}": AdvisoryV2Serializer(adv).data for adv in advisories}
@@ -1053,13 +1071,28 @@ def bulk_lookup(self, request):
purls = validated_data.get("purls")
# Fetch packages matching the provided purls
- packages = PackageV2.objects.for_purls(purls).with_is_vulnerable()
+ packages = (
+ PackageV2.objects.for_purls(purls)
+ .prefetch_related(
+ Prefetch(
+ "affected_in_impacts",
+ queryset=ImpactedPackage.objects.select_related("advisory").prefetch_related(
+ "fixed_by_packages",
+ ),
+ ),
+ Prefetch(
+ "fixed_in_impacts",
+ queryset=ImpactedPackage.objects.select_related("advisory"),
+ ),
+ )
+ .with_is_vulnerable()
+ )
# Collect vulnerabilities associated with these packages
advisories = set()
for package in packages:
- advisories.update(package.affected_by_advisories.all())
- advisories.update(package.fixing_advisories.all())
+ advisories.update({impact.advisory for impact in package.affected_in_impacts.all()})
+ advisories.update({impact.advisory for impact in package.fixed_in_impacts.all()})
# Serialize vulnerabilities with vulnerability_id as keys
advisory_data = {adv.avid: AdvisoryV2Serializer(adv).data for adv in advisories}
@@ -1124,6 +1157,20 @@ def bulk_search(self, request):
PackageV2.objects.filter(plain_package_url__in=plain_purls)
.order_by("plain_package_url")
.distinct("plain_package_url")
+ .prefetch_related(
+ Prefetch(
+ "affected_in_impacts",
+ queryset=ImpactedPackage.objects.select_related(
+ "advisory"
+ ).prefetch_related(
+ "fixed_by_packages",
+ ),
+ ),
+ Prefetch(
+ "fixed_in_impacts",
+ queryset=ImpactedPackage.objects.select_related("advisory"),
+ ),
+ )
.with_is_vulnerable()
)
@@ -1132,14 +1179,16 @@ def bulk_search(self, request):
# Collect vulnerabilities associated with these packages
advisories = set()
for package in packages:
- advisories.update(package.affected_by_advisories.all())
- advisories.update(package.fixing_advisories.all())
+ advisories.update({impact.advisory for impact in package.affected_in_impacts.all()})
+ advisories.update({impact.advisory for impact in package.fixed_in_impacts.all()})
advisory_data = {adv.avid: AdvisoryV2Serializer(adv).data for adv in advisories}
if not purl_only:
package_data = AdvisoryPackageV2Serializer(
- packages, many=True, context={"request": request}
+ packages,
+ many=True,
+ context={"request": request},
).data
return Response(
{
@@ -1154,20 +1203,39 @@ def bulk_search(self, request):
vulnerable_purls = [str(package.plain_package_url) for package in vulnerable_purls]
return Response(data=vulnerable_purls)
- query = PackageV2.objects.filter(package_url__in=purls).distinct().with_is_vulnerable()
+ query = (
+ PackageV2.objects.filter(package_url__in=purls)
+ .order_by("plain_package_url")
+ .distinct("plain_package_url")
+ .prefetch_related(
+ Prefetch(
+ "affected_in_impacts",
+ queryset=ImpactedPackage.objects.select_related("advisory").prefetch_related(
+ "fixed_by_packages",
+ ),
+ ),
+ Prefetch(
+ "fixed_in_impacts",
+ queryset=ImpactedPackage.objects.select_related("advisory"),
+ ),
+ )
+ .with_is_vulnerable()
+ )
packages = query
# Collect vulnerabilities associated with these packages
advisories = set()
for package in packages:
- advisories.update(package.affected_by_advisories.all())
- advisories.update(package.fixing_advisories.all())
+ advisories.update({impact.advisory for impact in package.affected_in_impacts.all()})
+ advisories.update({impact.advisory for impact in package.fixed_in_impacts.all()})
advisory_data = {adv.advisory_id: AdvisoryV2Serializer(adv).data for adv in advisories}
if not purl_only:
package_data = AdvisoryPackageV2Serializer(
- packages, many=True, context={"request": request}
+ packages,
+ many=True,
+ context={"request": request},
).data
return Response(
{
diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py
index 52f9d62dd..da7f743da 100644
--- a/vulnerabilities/importer.py
+++ b/vulnerabilities/importer.py
@@ -11,11 +11,8 @@
import datetime
import functools
import logging
-import os
-import shutil
import traceback
import xml.etree.ElementTree as ET
-from pathlib import Path
from typing import Iterable
from typing import List
from typing import Mapping
@@ -339,6 +336,88 @@ def from_dict(cls, affected_pkg: dict):
)
+@functools.total_ordering
+@dataclasses.dataclass(eq=True)
+class AffectedPackageV2:
+ """
+ Relate a Package URL with a range of affected versions and fixed versions.
+ The Package URL must *not* have a version.
+ AffectedPackage must contain either ``affected_version_range`` or ``fixed_version_range``.
+ """
+
+ package: PackageURL
+ affected_version_range: Optional[VersionRange] = None
+ fixed_version_range: Optional[VersionRange] = None
+
+ def __post_init__(self):
+ if self.package.version:
+ raise ValueError(f"Affected Package URL {self.package!r} cannot have a version.")
+
+ if not (self.affected_version_range or self.fixed_version_range):
+ raise ValueError(
+ f"Affected Package {self.package!r} should have either fixed version range or an "
+ "affected version range."
+ )
+
+ def __lt__(self, other):
+ if not isinstance(other, AffectedPackageV2):
+ return NotImplemented
+ return self._cmp_key() < other._cmp_key()
+
+ # TODO: Add cache
+ def _cmp_key(self):
+ return (
+ str(self.package),
+ str(self.affected_version_range or ""),
+ str(self.fixed_version_range or ""),
+ )
+
+ def to_dict(self):
+ """Return a serializable dict that can be converted back using self.from_dict"""
+
+ affected_version_range = (
+ str(self.affected_version_range) if self.affected_version_range else None
+ )
+ fixed_version_range = str(self.fixed_version_range) if self.fixed_version_range else None
+ return {
+ "package": purl_to_dict(self.package),
+ "affected_version_range": affected_version_range,
+ "fixed_version_range": fixed_version_range,
+ }
+
+ @classmethod
+ def from_dict(cls, affected_pkg: dict):
+ """Return an AffectedPackage object from dict generated by self.to_dict"""
+
+ package = PackageURL(**affected_pkg["package"])
+ affected_version_range = None
+ fixed_version_range = None
+ affected_range = affected_pkg["affected_version_range"]
+ fixed_range = affected_pkg["fixed_version_range"]
+
+ try:
+ affected_version_range = VersionRange.from_string(affected_range)
+ fixed_version_range = VersionRange.from_string(fixed_range)
+ except:
+ tb = traceback.format_exc()
+ logger.error(
+ f"Cannot create AffectedPackage with invalid or unknown range: {affected_pkg!r} with error: {tb!r}"
+ )
+ return
+
+ if not fixed_version_range and not affected_version_range:
+ logger.error(
+ f"Cannot create AffectedPackage without fixed or affected range: {affected_pkg!r}"
+ )
+ return
+
+ return cls(
+ package=package,
+ affected_version_range=affected_version_range,
+ fixed_version_range=fixed_version_range,
+ )
+
+
@dataclasses.dataclass(order=True)
class AdvisoryData:
"""
@@ -355,7 +434,9 @@ class AdvisoryData:
advisory_id: str = ""
aliases: List[str] = dataclasses.field(default_factory=list)
summary: Optional[str] = ""
- affected_packages: List[AffectedPackage] = dataclasses.field(default_factory=list)
+ affected_packages: Union[List[AffectedPackage], List[AffectedPackageV2]] = dataclasses.field(
+ default_factory=list
+ )
references: List[Reference] = dataclasses.field(default_factory=list)
references_v2: List[ReferenceV2] = dataclasses.field(default_factory=list)
date_published: Optional[datetime.datetime] = None
@@ -365,8 +446,6 @@ class AdvisoryData:
original_advisory_text: Optional[str] = None
def __post_init__(self):
- if self.date_published and not self.date_published.tzinfo:
- logger.warning(f"AdvisoryData with no tzinfo: {self!r}")
if self.summary:
self.summary = self.clean_summary(self.summary)
@@ -392,13 +471,19 @@ def to_dict(self):
@classmethod
def from_dict(cls, advisory_data):
date_published = advisory_data["date_published"]
+ affected_packages = advisory_data["affected_packages"]
+ affected_package_cls = AffectedPackage
+ if affected_packages:
+ affected_package_cls = (
+ AffectedPackageV2
+ if "fixed_version_range" in affected_packages[0]
+ else AffectedPackage
+ )
transformed = {
"aliases": advisory_data["aliases"],
"summary": advisory_data["summary"],
"affected_packages": [
- AffectedPackage.from_dict(pkg)
- for pkg in advisory_data["affected_packages"]
- if pkg is not None
+ affected_package_cls.from_dict(pkg) for pkg in affected_packages if pkg is not None
],
"references": [Reference.from_dict(ref) for ref in advisory_data["references"]],
"date_published": datetime.datetime.fromisoformat(date_published)
diff --git a/vulnerabilities/importers/osv.py b/vulnerabilities/importers/osv.py
index be27492d7..8bd4ad274 100644
--- a/vulnerabilities/importers/osv.py
+++ b/vulnerabilities/importers/osv.py
@@ -23,6 +23,7 @@
from vulnerabilities.importer import AdvisoryData
from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import AffectedPackageV2
from vulnerabilities.importer import Reference
from vulnerabilities.importer import ReferenceV2
from vulnerabilities.importer import VulnerabilitySeverity
@@ -144,19 +145,27 @@ def parse_advisory_data_v2(
supported_ecosystem=purl.type,
)
+ fixed_versions = []
+ fixed_version_range = None
for fixed_range in affected_pkg.get("ranges") or []:
fixed_version = get_fixed_versions(
fixed_range=fixed_range, raw_id=advisory_id, supported_ecosystem=purl.type
)
+ fixed_versions.extend([v.string for v in fixed_version])
- for version in fixed_version:
- affected_packages.append(
- AffectedPackage(
- package=purl,
- affected_version_range=affected_version_range,
- fixed_version=version,
- )
+ fixed_version_range = (
+ get_fixed_version_range(fixed_versions, purl.type) if fixed_versions else None
+ )
+
+ if fixed_version_range or affected_version_range:
+ affected_packages.append(
+ AffectedPackageV2(
+ package=purl,
+ affected_version_range=affected_version_range,
+ fixed_version_range=fixed_version_range,
)
+ )
+
database_specific = raw_data.get("database_specific") or {}
cwe_ids = database_specific.get("cwe_ids") or []
weaknesses = list(map(get_cwe_id, cwe_ids))
@@ -334,6 +343,13 @@ def get_affected_version_range(affected_pkg, raw_id, supported_ecosystem):
)
+def get_fixed_version_range(versions, ecosystem):
+ try:
+ return RANGE_CLASS_BY_SCHEMES[ecosystem].from_versions(versions)
+ except Exception as e:
+ logger.error(f"Failed to create VersionRange from: {versions}: error:{e!r}")
+
+
def get_fixed_versions(fixed_range, raw_id, supported_ecosystem) -> List[Version]:
"""
Return a list of unique fixed univers Versions given a ``fixed_range``
diff --git a/vulnerabilities/migrations/0100_remove_advisoryv2_affecting_packages_and_more.py b/vulnerabilities/migrations/0100_remove_advisoryv2_affecting_packages_and_more.py
new file mode 100644
index 000000000..9a00e4731
--- /dev/null
+++ b/vulnerabilities/migrations/0100_remove_advisoryv2_affecting_packages_and_more.py
@@ -0,0 +1,87 @@
+# Generated by Django 4.2.22 on 2025-07-28 18:30
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("vulnerabilities", "0099_advisoryv2_original_advisory_text"),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name="advisoryv2",
+ name="affecting_packages",
+ ),
+ migrations.RemoveField(
+ model_name="advisoryv2",
+ name="fixed_by_packages",
+ ),
+ migrations.CreateModel(
+ name="ImpactedPackage",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+ ),
+ ),
+ (
+ "base_purl",
+ models.CharField(
+ blank=True,
+ help_text="Version less PURL related to impacted range.",
+ max_length=500,
+ ),
+ ),
+ (
+ "affecting_vers",
+ models.TextField(
+ blank=True,
+ help_text="VersionRange expression for package vulnerable to this impact.",
+ ),
+ ),
+ (
+ "fixed_vers",
+ models.TextField(
+ blank=True,
+ help_text="VersionRange expression for packages fixing the vulnerable package in this impact.",
+ ),
+ ),
+ (
+ "created_at",
+ models.DateTimeField(
+ auto_now_add=True,
+ db_index=True,
+ help_text="Timestamp indicating when this impact was added.",
+ ),
+ ),
+ (
+ "advisory",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="impacted_packages",
+ to="vulnerabilities.advisoryv2",
+ ),
+ ),
+ (
+ "affecting_packages",
+ models.ManyToManyField(
+ help_text="Packages vulnerable to this impact.",
+ related_name="affected_in_impacts",
+ to="vulnerabilities.packagev2",
+ ),
+ ),
+ (
+ "fixed_by_packages",
+ models.ManyToManyField(
+ help_text="Packages vulnerable to this impact.",
+ related_name="fixed_in_impacts",
+ to="vulnerabilities.packagev2",
+ ),
+ ),
+ ],
+ ),
+ ]
diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py
index c4a302536..d51b2fce6 100644
--- a/vulnerabilities/models.py
+++ b/vulnerabilities/models.py
@@ -2532,6 +2532,20 @@ class Meta:
verbose_name_plural = "Advisory severities"
ordering = ["url", "scoring_system", "value"]
+ def to_dict(self):
+ return {
+ "system": self.scoring_system,
+ "value": self.value,
+ "scoring_elements": self.scoring_elements,
+ "published_at": self.published_at,
+ "url": self.url,
+ }
+
+ def to_vulnerability_severity_data(self):
+ from vulnerabilities.importer import VulnerabilitySeverity
+
+ return VulnerabilitySeverity.from_dict(self.to_dict())
+
class AdvisoryWeakness(models.Model):
"""
@@ -2625,6 +2639,18 @@ def is_cpe(self):
"""
return self.reference_id.startswith("cpe")
+ def to_dict(self):
+ return {
+ "reference_id": self.reference_id,
+ "reference_type": self.reference_type,
+ "url": self.url,
+ }
+
+ def to_reference_v2_data(self):
+ from vulnerabilities.importer import ReferenceV2
+
+ return ReferenceV2.from_dict(self.to_dict())
+
class AdvisoryAlias(models.Model):
alias = models.CharField(
@@ -2750,18 +2776,6 @@ class AdvisoryV2(models.Model):
help_text="Raw advisory data as collected from the upstream datasource.",
)
- affecting_packages = models.ManyToManyField(
- "PackageV2",
- related_name="affected_by_advisories",
- help_text="A list of packages that are affected by this advisory.",
- )
-
- fixed_by_packages = models.ManyToManyField(
- "PackageV2",
- related_name="fixing_advisories",
- help_text="A list of packages that are reported by this advisory.",
- )
-
status = models.IntegerField(
choices=AdvisoryStatusType.choices, default=AdvisoryStatusType.PUBLISHED
)
@@ -2816,21 +2830,19 @@ def get_absolute_url(self):
"""
return reverse("advisory_details", args=[self.avid])
- def to_advisory_data(self) -> "AdvisoryDataV2":
- from vulnerabilities.importer import AdvisoryDataV2
- from vulnerabilities.importer import AffectedPackage
- from vulnerabilities.importer import ReferenceV2
+ def to_advisory_data(self) -> "AdvisoryData":
+ from vulnerabilities.importer import AdvisoryData
- return AdvisoryDataV2(
+ return AdvisoryData(
aliases=[item.alias for item in self.aliases.all()],
summary=self.summary,
affected_packages=[
- AffectedPackage.from_dict(pkg) for pkg in self.affected_packages if pkg
+ impacted.to_affected_package_data() for impacted in self.impacted_packages.all()
],
- references=[ReferenceV2.from_dict(ref) for ref in self.references],
+ references_v2=[ref.to_reference_v2_data() for ref in self.references.all()],
date_published=self.date_published,
- weaknesses=self.weaknesses,
- severities=self.severities,
+ weaknesses=[weak.cwe_id for weak in self.weaknesses.all()],
+ severities=[sev.to_vulnerability_severity_data() for sev in self.severities.all()],
url=self.url,
)
@@ -2841,66 +2853,69 @@ def get_aliases(self):
"""
return self.aliases.all()
- def aggregate_fixed_and_affected_packages(self):
- from vulnerabilities.utils import get_purl_version_class
+ alias = get_aliases
- sorted_fixed_by_packages = self.fixed_by_packages.filter(is_ghost=False).order_by(
- "type", "namespace", "name", "qualifiers", "subpath"
- )
- if sorted_fixed_by_packages:
- sorted_fixed_by_packages.first().calculate_version_rank
+class ImpactedPackage(models.Model):
+ """
+ Represents a single impact for an advisory, including affected range and fixed version and
+ associated package relations.
+ """
- sorted_affected_packages = self.affecting_packages.all()
+ advisory = models.ForeignKey(
+ AdvisoryV2,
+ related_name="impacted_packages",
+ on_delete=models.CASCADE,
+ )
- if sorted_affected_packages:
- sorted_affected_packages.first().calculate_version_rank
+ base_purl = models.CharField(
+ max_length=500,
+ blank=True,
+ help_text="Version less PURL related to impacted range.",
+ )
- grouped_fixed_by_packages = {
- key: list(group)
- for key, group in groupby(
- sorted_fixed_by_packages,
- key=attrgetter("type", "namespace", "name", "qualifiers", "subpath"),
- )
- }
+ affecting_vers = models.TextField(
+ blank=True,
+ help_text="VersionRange expression for package vulnerable to this impact.",
+ )
- all_affected_fixed_by_matches = []
+ fixed_vers = models.TextField(
+ blank=True,
+ help_text="VersionRange expression for packages fixing the vulnerable package in this impact.",
+ )
- for sorted_affected_package in sorted_affected_packages:
- affected_fixed_by_matches = {
- "affected_package": sorted_affected_package,
- "matched_fixed_by_packages": [],
- }
+ affecting_packages = models.ManyToManyField(
+ "PackageV2",
+ related_name="affected_in_impacts",
+ help_text="Packages vulnerable to this impact.",
+ )
- # Build the key to find matching group
- key = (
- sorted_affected_package.type,
- sorted_affected_package.namespace,
- sorted_affected_package.name,
- sorted_affected_package.qualifiers,
- sorted_affected_package.subpath,
- )
+ fixed_by_packages = models.ManyToManyField(
+ "PackageV2",
+ related_name="fixed_in_impacts",
+ help_text="Packages vulnerable to this impact.",
+ )
- # Get matching group from pre-grouped fixed_by_packages
- matching_fixed_packages = grouped_fixed_by_packages.get(key, [])
+ created_at = models.DateTimeField(
+ auto_now_add=True,
+ db_index=True,
+ help_text="Timestamp indicating when this impact was added.",
+ )
- # Get version classes for comparison
- affected_version_class = get_purl_version_class(sorted_affected_package)
- affected_version = affected_version_class(sorted_affected_package.version)
+ def to_dict(self):
+ from vulnerabilities.utils import purl_to_dict
- # Compare versions and filter valid matches
- matched_fixed_by_packages = [
- fixed_by_package.purl
- for fixed_by_package in matching_fixed_packages
- if get_purl_version_class(fixed_by_package)(fixed_by_package.version)
- > affected_version
- ]
+ return {
+ "package": purl_to_dict(self.base_purl),
+ "affected_version_range": self.affecting_vers,
+ "fixed_version_range": self.fixed_vers,
+ }
- affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages
- all_affected_fixed_by_matches.append(affected_fixed_by_matches)
- return sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches
+ def to_affected_package_data(self):
+ """Return `AffectedPackageV2` data from the impact."""
+ from vulnerabilities.importer import AffectedPackageV2
- alias = get_aliases
+ return AffectedPackageV2.from_dict(self.to_dict())
class ToDoRelatedAdvisory(models.Model):
@@ -2942,10 +2957,12 @@ def search(self, query: str = None):
def with_vulnerability_counts(self):
return self.annotate(
vulnerability_count=Count(
- "affected_by_advisories",
+ "affected_in_impacts__advisory",
+ distinct=True,
),
patched_vulnerability_count=Count(
- "fixing_advisories",
+ "fixed_in_impacts__advisory",
+ distinct=True,
),
)
@@ -2963,7 +2980,7 @@ def get_fixed_by_package_versions(self, purl: PackageURL, fix=True):
}
if fix:
- filter_dict["fixing_advisories__isnull"] = False
+ filter_dict["fixed_in_impacts__isnull"] = False
# TODO: why do we need distinct
return PackageV2.objects.filter(**filter_dict).distinct()
@@ -2996,7 +3013,7 @@ def for_purls(self, purls=()):
def _vulnerable(self, vulnerable=True):
"""
- Filter to select only vulnerable or non-vulnearble packages.
+ Filter to select only vulnerable or non-vulnerable packages.
"""
return self.with_is_vulnerable().filter(is_vulnerable=vulnerable)
@@ -3004,14 +3021,16 @@ def vulnerable(self):
"""
Return only packages that are vulnerable.
"""
- return self.filter(affected_by_advisories__isnull=False)
+ return self.filter(affected_in_impacts__isnull=False)
def with_is_vulnerable(self):
"""
Annotate Package with ``is_vulnerable`` boolean attribute.
"""
return self.annotate(
- is_vulnerable=Exists(AdvisoryV2.objects.filter(affecting_packages__pk=OuterRef("pk")))
+ is_vulnerable=Exists(
+ ImpactedPackage.objects.filter(affecting_packages__pk=OuterRef("pk"))
+ )
)
def from_purl(self, purl: Union[PackageURL, str]):
@@ -3118,23 +3137,6 @@ def calculate_version_rank(self):
PackageV2.objects.bulk_update(sorted_packages, fields=["version_rank"])
return self.version_rank
- @property
- def fixed_package_details(self):
- """
- Return a mapping of vulnerabilities that affect this package and the next and
- latest non-vulnerable versions.
- """
- package_details = {}
- package_details["purl"] = PackageURL.from_string(self.purl)
-
- next_non_vulnerable, latest_non_vulnerable = self.get_non_vulnerable_versions()
- package_details["next_non_vulnerable"] = next_non_vulnerable
- package_details["latest_non_vulnerable"] = latest_non_vulnerable
-
- package_details["advisories"] = self.get_affecting_vulnerabilities()
-
- return package_details
-
def get_non_vulnerable_versions(self):
"""
Return a tuple of the next and latest non-vulnerable versions as Package instance.
@@ -3146,17 +3148,12 @@ def get_non_vulnerable_versions(self):
self, fix=False
).only_non_vulnerable()
- later_non_vulnerable_versions = non_vulnerable_versions.filter(
- version_rank__gt=self.version_rank
- )
-
- later_non_vulnerable_versions = list(later_non_vulnerable_versions)
+ later_non_vulnerable = non_vulnerable_versions.filter(
+ version_rank__gte=self.version_rank
+ ).order_by("version_rank")
- if later_non_vulnerable_versions:
- sorted_versions = later_non_vulnerable_versions
- next_non_vulnerable = sorted_versions[0]
- latest_non_vulnerable = sorted_versions[-1]
- return next_non_vulnerable, latest_non_vulnerable
+ if later_non_vulnerable.exists():
+ return later_non_vulnerable.first(), later_non_vulnerable.last()
return None, None
@@ -3175,65 +3172,6 @@ def get_absolute_url(self):
def current_version(self):
return self.version_class(self.version)
- def get_affecting_vulnerabilities(self):
- """
- Return a list of vulnerabilities that affect this package together with information regarding
- the versions that fix the vulnerabilities.
- """
- if self.version_rank == 0:
- self.calculate_version_rank
- package_details_advs = []
-
- fixed_by_packages = PackageV2.objects.get_fixed_by_package_versions(self, fix=True)
-
- package_advisories = self.affected_by_advisories.prefetch_related(
- Prefetch(
- "fixed_by_packages",
- queryset=fixed_by_packages,
- to_attr="fixed_packages",
- )
- )
-
- for adv in package_advisories:
- package_details_advs.append({"advisory": adv})
- later_fixed_packages = []
-
- for fixed_pkg in adv.fixed_by_packages.all():
- if fixed_pkg not in fixed_by_packages:
- continue
- fixed_version = self.version_class(fixed_pkg.version)
- if fixed_version > self.current_version:
- later_fixed_packages.append(fixed_pkg)
-
- next_fixed_package_vulns = []
-
- sort_fixed_by_packages_by_version = []
- if later_fixed_packages:
- sort_fixed_by_packages_by_version = sorted(
- later_fixed_packages, key=lambda p: p.version_rank
- )
-
- fixed_by_pkgs = []
-
- for vuln_details in package_details_advs:
- if vuln_details["advisory"] != adv:
- continue
- vuln_details["fixed_by_purl"] = []
- vuln_details["fixed_by_purl_advisories"] = []
-
- for fixed_by_pkg in sort_fixed_by_packages_by_version:
- fixed_by_package_details = {}
- fixed_by_purl = PackageURL.from_string(fixed_by_pkg.purl)
- next_fixed_package_vulns = list(fixed_by_pkg.affected_by_advisories.all())
-
- fixed_by_package_details["fixed_by_purl"] = fixed_by_purl
- fixed_by_package_details["fixed_by_purl_advisories"] = next_fixed_package_vulns
- fixed_by_pkgs.append(fixed_by_package_details)
-
- vuln_details["fixed_by_package_details"] = fixed_by_pkgs
-
- return package_details_advs
-
class AdvisoryExploit(models.Model):
"""
diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py
index 6232a294d..c642748b0 100644
--- a/vulnerabilities/pipelines/__init__.py
+++ b/vulnerabilities/pipelines/__init__.py
@@ -15,29 +15,18 @@
from traceback import format_exc as traceback_format_exc
from typing import Iterable
from typing import List
-from typing import Optional
from aboutcode.pipeline import LoopProgress
from aboutcode.pipeline import PipelineDefinition
from aboutcode.pipeline import humanize_time
-from fetchcode import package_versions
-from packageurl import PackageURL
from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import AffectedPackage
-from vulnerabilities.importer import UnMergeablePackageError
from vulnerabilities.improver import MAX_CONFIDENCE
from vulnerabilities.models import Advisory
-from vulnerabilities.models import PackageV2
from vulnerabilities.models import PipelineRun
from vulnerabilities.pipes.advisory import import_advisory
from vulnerabilities.pipes.advisory import insert_advisory
from vulnerabilities.pipes.advisory import insert_advisory_v2
-from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage
-from vulnerabilities.utils import classproperty
-from vulnerabilities.utils import get_affected_packages_by_patched_package
-from vulnerabilities.utils import nearest_patched_package
-from vulnerabilities.utils import resolve_version_range
module_logger = logging.getLogger(__name__)
@@ -271,13 +260,15 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline):
license_url = None
spdx_license_expression = None
repo_url = None
- advisory_confidence = MAX_CONFIDENCE
ignorable_versions = []
- unfurl_version_ranges = False
@classmethod
def steps(cls):
- return (cls.collect_and_store_advisories,)
+ return (
+ # Add step for downloading/cloning resource as required.
+ cls.collect_and_store_advisories,
+ # Add step for removing downloaded/cloned resource as required.
+ )
def collect_advisories(self) -> Iterable[AdvisoryData]:
"""
@@ -311,7 +302,6 @@ def collect_and_store_advisories(self):
if _obj := insert_advisory_v2(
advisory=advisory,
pipeline_id=self.pipeline_id,
- get_advisory_packages=self.get_advisory_packages,
logger=self.log,
):
collected_advisory_count += 1
@@ -323,203 +313,3 @@ def collect_and_store_advisories(self):
continue
self.log(f"Successfully collected {collected_advisory_count:,d} advisories")
-
- def get_advisory_packages(self, advisory_data: AdvisoryData) -> list:
- """
- Return the list of packages for the given advisory.
-
- Used by ``import_advisory`` to get the list of packages for the advisory.
- """
- from vulnerabilities.improvers import default
-
- affected_purls = []
- fixed_purls = []
- for affected_package in advisory_data.affected_packages:
- package_affected_purls, package_fixed_purls = default.get_exact_purls(
- affected_package=affected_package
- )
- affected_purls.extend(package_affected_purls)
- fixed_purls.extend(package_fixed_purls)
-
- if self.unfurl_version_ranges:
- vulnerable_pvs, fixed_pvs = self.get_impacted_packages(
- affected_packages=advisory_data.affected_packages,
- advisory_date_published=advisory_data.date_published,
- )
- affected_purls.extend(vulnerable_pvs)
- fixed_purls.extend(fixed_pvs)
-
- vulnerable_packages = []
- fixed_packages = []
-
- for affected_purl in affected_purls:
- vulnerable_package, _ = PackageV2.objects.get_or_create_from_purl(purl=affected_purl)
- vulnerable_packages.append(vulnerable_package)
-
- for fixed_purl in fixed_purls:
- fixed_package, _ = PackageV2.objects.get_or_create_from_purl(purl=fixed_purl)
- fixed_packages.append(fixed_package)
-
- return vulnerable_packages, fixed_packages
-
- def get_published_package_versions(
- self, package_url: PackageURL, until: Optional[datetime] = None
- ) -> List[str]:
- """
- Return a list of versions published before `until` for the `package_url`
- """
- versions_before_until = []
- try:
- versions = package_versions.versions(str(package_url))
- for version in versions or []:
- if (
- version.release_date
- and version.release_date.tzinfo
- and until
- and until.tzinfo is None
- ):
- until = until.replace(tzinfo=timezone.utc)
- if until and version.release_date and version.release_date > until:
- continue
- versions_before_until.append(version.value)
-
- return versions_before_until
- except Exception as e:
- self.log(
- f"Failed to fetch versions for package {str(package_url)} {e!r}",
- level=logging.ERROR,
- )
- return []
-
- def get_impacted_packages(self, affected_packages, advisory_date_published):
- """
- Return a tuple of lists of affected and fixed PackageURLs
- """
- if not affected_packages:
- return [], []
-
- mergable = True
-
- # TODO: We should never had the exception in first place
- try:
- purl, affected_version_ranges, fixed_versions = AffectedPackage.merge(affected_packages)
- except UnMergeablePackageError:
- self.log(f"Cannot merge with different purls {affected_packages!r}", logging.ERROR)
- mergable = False
-
- if not mergable:
- vulnerable_packages = []
- fixed_packages = []
- for affected_package in affected_packages:
- purl = affected_package.package
- affected_version_range = affected_package.affected_version_range
- fixed_version = affected_package.fixed_version
- pkg_type = purl.type
- pkg_namespace = purl.namespace
- pkg_name = purl.name
- if not affected_version_range and fixed_version:
- fixed_packages.append(
- PackageURL(
- type=pkg_type,
- namespace=pkg_namespace,
- name=pkg_name,
- version=str(fixed_version),
- )
- )
- else:
- valid_versions = self.get_published_package_versions(
- package_url=purl, until=advisory_date_published
- )
- affected_pvs, fixed_pvs = self.resolve_package_versions(
- affected_version_range=affected_version_range,
- pkg_type=pkg_type,
- pkg_namespace=pkg_namespace,
- pkg_name=pkg_name,
- valid_versions=valid_versions,
- )
- vulnerable_packages.extend(affected_pvs)
- fixed_packages.extend(fixed_pvs)
- return vulnerable_packages, fixed_packages
- else:
- pkg_type = purl.type
- pkg_namespace = purl.namespace
- pkg_name = purl.name
- pkg_qualifiers = purl.qualifiers
- fixed_purls = [
- PackageURL(
- type=pkg_type,
- namespace=pkg_namespace,
- name=pkg_name,
- version=str(version),
- qualifiers=pkg_qualifiers,
- )
- for version in fixed_versions
- ]
- if not affected_version_ranges:
- return [], fixed_purls
- else:
- valid_versions = self.get_published_package_versions(
- package_url=purl, until=advisory_date_published
- )
- vulnerable_packages = []
- fixed_packages = []
- for affected_version_range in affected_version_ranges:
- vulnerable_pvs, fixed_pvs = self.resolve_package_versions(
- affected_version_range=affected_version_range,
- pkg_type=pkg_type,
- pkg_namespace=pkg_namespace,
- pkg_name=pkg_name,
- valid_versions=valid_versions,
- )
- vulnerable_packages.extend(vulnerable_pvs)
- fixed_packages.extend(fixed_pvs)
- return vulnerable_packages, fixed_packages
-
- def resolve_package_versions(
- self,
- affected_version_range,
- pkg_type,
- pkg_namespace,
- pkg_name,
- valid_versions,
- ):
- """
- Return a tuple of lists of ``affected_packages`` and ``fixed_packages`` PackageURL for the given `affected_version_range` and `valid_versions`.
-
- ``valid_versions`` are the valid version listed on the package registry for that package
-
- """
- aff_vers, unaff_vers = resolve_version_range(
- affected_version_range=affected_version_range,
- ignorable_versions=self.ignorable_versions,
- package_versions=valid_versions,
- )
-
- affected_purls = list(
- self.expand_verion_range_to_purls(pkg_type, pkg_namespace, pkg_name, aff_vers)
- )
-
- unaffected_purls = list(
- self.expand_verion_range_to_purls(pkg_type, pkg_namespace, pkg_name, unaff_vers)
- )
-
- fixed_packages = []
- affected_packages = []
-
- patched_packages = nearest_patched_package(
- vulnerable_packages=affected_purls, resolved_packages=unaffected_purls
- )
-
- for (
- fixed_package,
- affected_purls,
- ) in get_affected_packages_by_patched_package(patched_packages).items():
- if fixed_package:
- fixed_packages.append(fixed_package)
- affected_packages.extend(affected_purls)
-
- return affected_packages, fixed_packages
-
- def expand_verion_range_to_purls(self, pkg_type, pkg_namespace, pkg_name, versions):
- for version in versions:
- yield PackageURL(type=pkg_type, namespace=pkg_namespace, name=pkg_name, version=version)
diff --git a/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py b/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py
index c90af00c4..249133eaa 100644
--- a/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py
+++ b/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py
@@ -22,7 +22,7 @@
from univers.versions import SemverVersion
from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import AffectedPackageV2
from vulnerabilities.importer import ReferenceV2
from vulnerabilities.importer import VulnerabilitySeverity
from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
@@ -151,7 +151,6 @@ class ApacheHTTPDImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
spdx_license_expression = "Apache-2.0"
license_url = "https://www.apache.org/licenses/LICENSE-2.0"
base_url = "https://httpd.apache.org/security/json/"
- unfurl_version_ranges = True
links = []
@@ -219,7 +218,6 @@ class ApacheHTTPDImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
"pre_ajp_proxy",
]
)
- unfurl_version_ranges = True
@classmethod
def steps(cls):
@@ -292,7 +290,7 @@ def to_advisory(self, data):
affected_version_range = self.to_version_ranges(versions_data, fixed_versions)
if affected_version_range:
affected_packages.append(
- AffectedPackage(
+ AffectedPackageV2(
package=PackageURL(
type="apache",
name="httpd",
diff --git a/vulnerabilities/pipelines/v2_importers/curl_importer.py b/vulnerabilities/pipelines/v2_importers/curl_importer.py
index e3253b4b4..03610a1e0 100644
--- a/vulnerabilities/pipelines/v2_importers/curl_importer.py
+++ b/vulnerabilities/pipelines/v2_importers/curl_importer.py
@@ -12,10 +12,9 @@
from cwe2.database import Database
from packageurl import PackageURL
from univers.version_range import GenericVersionRange
-from univers.versions import SemverVersion
from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import AffectedPackageV2
from vulnerabilities.importer import ReferenceV2
from vulnerabilities.importer import VulnerabilitySeverity
from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
@@ -37,7 +36,6 @@ class CurlImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
license_url = "https://curl.se/docs/copyright.html"
repo_url = "https://github.com/curl/curl-www/"
url = "https://curl.se/docs/vuln.json"
- unfurl_version_ranges = True
@classmethod
def steps(cls):
@@ -77,16 +75,16 @@ def parse_curl_advisory(raw_data) -> AdvisoryData:
version_type = get_item(ranges, "type") if get_item(ranges, "type") else ""
fixed_version = events.get("fixed")
if version_type == "SEMVER" and fixed_version:
- fixed_version = SemverVersion(fixed_version)
+ fixed_version_range = GenericVersionRange.from_versions([fixed_version])
purl = PackageURL(type="generic", namespace="curl.se", name="curl")
versions = affected.get("versions") or []
affected_version_range = GenericVersionRange.from_versions(versions)
- affected_package = AffectedPackage(
+ affected_package = AffectedPackageV2(
package=purl,
affected_version_range=affected_version_range,
- fixed_version=fixed_version,
+ fixed_version_range=fixed_version_range,
)
database_specific = raw_data.get("database_specific") or {}
diff --git a/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py b/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py
index 0c47b9c1a..4fb95ad3b 100644
--- a/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py
+++ b/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py
@@ -17,7 +17,7 @@
from univers.version_range import HexVersionRange
from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import AffectedPackageV2
from vulnerabilities.importer import ReferenceV2
from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
from vulnerabilities.utils import is_cve
@@ -26,7 +26,7 @@
class ElixirSecurityImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
"""
- Elixir Security Advisiories Importer Pipeline
+ Elixir Security Advisories Importer Pipeline
This pipeline imports security advisories for elixir.
"""
@@ -35,7 +35,6 @@ class ElixirSecurityImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
spdx_license_expression = "CC0-1.0"
license_url = "https://github.com/dependabot/elixir-security-advisories/blob/master/LICENSE.txt"
repo_url = "git+https://github.com/dependabot/elixir-security-advisories"
- unfurl_version_ranges = True
@classmethod
def steps(cls):
@@ -118,7 +117,7 @@ def process_file(self, file, base_path) -> Iterable[AdvisoryData]:
affected_packages = []
if pkg_name:
affected_packages.append(
- AffectedPackage(
+ AffectedPackageV2(
package=PackageURL(type="hex", name=pkg_name),
affected_version_range=HexVersionRange(constraints=constraints),
)
diff --git a/vulnerabilities/pipelines/v2_importers/github_osv_importer.py b/vulnerabilities/pipelines/v2_importers/github_osv_importer.py
index ef96e4a84..36f9d06b7 100644
--- a/vulnerabilities/pipelines/v2_importers/github_osv_importer.py
+++ b/vulnerabilities/pipelines/v2_importers/github_osv_importer.py
@@ -29,7 +29,6 @@ class GithubOSVImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
spdx_license_expression = "CC-BY-4.0"
license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md"
repo_url = "git+https://github.com/github/advisory-database/"
- unfurl_version_ranges = True
@classmethod
def steps(cls):
diff --git a/vulnerabilities/pipelines/v2_importers/gitlab_importer.py b/vulnerabilities/pipelines/v2_importers/gitlab_importer.py
index ff87dd5c9..e28ab7520 100644
--- a/vulnerabilities/pipelines/v2_importers/gitlab_importer.py
+++ b/vulnerabilities/pipelines/v2_importers/gitlab_importer.py
@@ -12,7 +12,6 @@
import traceback
from pathlib import Path
from typing import Iterable
-from typing import List
from typing import Tuple
import pytz
@@ -21,12 +20,10 @@
from fetchcode.vcs import fetch_via_vcs
from packageurl import PackageURL
from univers.version_range import RANGE_CLASS_BY_SCHEMES
-from univers.version_range import VersionRange
from univers.version_range import from_gitlab_native
-from univers.versions import Version
from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import AffectedPackageV2
from vulnerabilities.importer import ReferenceV2
from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
from vulnerabilities.utils import build_description
@@ -45,7 +42,6 @@ class GitLabImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
spdx_license_expression = "MIT"
license_url = "https://gitlab.com/gitlab-org/advisories-community/-/blob/main/LICENSE"
repo_url = "git+https://gitlab.com/gitlab-org/advisories-community/"
- unfurl_version_ranges = True
@classmethod
def steps(cls):
@@ -177,27 +173,6 @@ def get_purl(package_slug, purl_type_by_gitlab_scheme, logger):
return
-def extract_affected_packages(
- affected_version_range: VersionRange,
- fixed_versions: List[Version],
- purl: PackageURL,
-) -> Iterable[AffectedPackage]:
- """
- Yield AffectedPackage objects, one for each fixed_version
-
- In case of gitlab advisory data we get a list of fixed_versions and a affected_version_range.
- Since we can not determine which package fixes which range.
- We store the all the fixed_versions with the same affected_version_range in the advisory.
- Later the advisory data is used to be inferred in the GitLabBasicImprover.
- """
- for fixed_version in fixed_versions:
- yield AffectedPackage(
- package=purl,
- fixed_version=fixed_version,
- affected_version_range=affected_version_range,
- )
-
-
def parse_gitlab_advisory(
file, base_path, gitlab_scheme_by_purl_type, purl_type_by_gitlab_scheme, logger
):
@@ -276,7 +251,7 @@ def parse_gitlab_advisory(
fixed_versions = gitlab_advisory.get("fixed_versions") or []
affected_range = gitlab_advisory.get("affected_range")
gitlab_native_schemes = set(["pypi", "gem", "npm", "go", "packagist", "conan"])
- vrc: VersionRange = RANGE_CLASS_BY_SCHEMES[purl.type]
+ vrc = RANGE_CLASS_BY_SCHEMES[purl.type]
gitlab_scheme = gitlab_scheme_by_purl_type[purl.type]
try:
if affected_range:
@@ -296,38 +271,33 @@ def parse_gitlab_advisory(
for fixed_version in fixed_versions:
try:
fixed_version = vrc.version_class(fixed_version)
- parsed_fixed_versions.append(fixed_version)
+ parsed_fixed_versions.append(fixed_version.string)
except Exception as e:
logger(
f"parse_yaml_file: fixed_version is not parsable`: {fixed_version!r} error: {e!r}\n {traceback.format_exc()}",
level=logging.ERROR,
)
- if parsed_fixed_versions:
- affected_packages = list(
- extract_affected_packages(
- affected_version_range=affected_version_range,
- fixed_versions=parsed_fixed_versions,
- purl=purl,
- )
- )
- else:
- if not affected_version_range:
- affected_packages = []
- else:
- affected_packages = [
- AffectedPackage(
- package=purl,
- affected_version_range=affected_version_range,
- )
- ]
+ if affected_version_range:
+ vrc = affected_version_range.__class__
+
+ fixed_version_range = vrc.from_versions(parsed_fixed_versions)
+ if not fixed_version_range and not affected_version_range:
+ return
+
+ affected_package = AffectedPackageV2(
+ package=purl,
+ affected_version_range=affected_version_range,
+ fixed_version_range=fixed_version_range,
+ )
+
return AdvisoryData(
advisory_id=advisory_id,
aliases=aliases,
summary=summary,
references_v2=references,
date_published=date_published,
- affected_packages=affected_packages,
+ affected_packages=[affected_package],
weaknesses=cwe_list,
url=advisory_url,
original_advisory_text=json.dumps(gitlab_advisory, indent=2, ensure_ascii=False),
diff --git a/vulnerabilities/pipelines/v2_importers/istio_importer.py b/vulnerabilities/pipelines/v2_importers/istio_importer.py
index bc544f7f8..4126ec75e 100644
--- a/vulnerabilities/pipelines/v2_importers/istio_importer.py
+++ b/vulnerabilities/pipelines/v2_importers/istio_importer.py
@@ -20,10 +20,11 @@
from univers.version_constraint import VersionConstraint
from univers.version_range import GitHubVersionRange
from univers.version_range import GolangVersionRange
+from univers.versions import GolangVersion
from univers.versions import SemverVersion
from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import AffectedPackageV2
from vulnerabilities.importer import ReferenceV2
from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
from vulnerabilities.utils import get_advisory_url
@@ -41,7 +42,6 @@ class IstioImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
spdx_license_expression = "Apache-2.0"
license_url = "https://github.com/istio/istio.io/blob/master/LICENSE"
repo_url = "git+https://github.com/istio/istio.io"
- unfurl_version_ranges = True
@classmethod
def steps(cls):
@@ -80,23 +80,27 @@ def collect_advisories(self) -> Iterable[AdvisoryData]:
release_date = (
parser.parse(published_date).replace(tzinfo=pytz.UTC) if published_date else None
)
- constraints = self.get_version_constraints(data.get("releases", []))
-
+ semver_constraints = self.get_version_constraints(data.get("releases", []))
+ golang_constraints = self.get_version_constraints(
+ data.get("releases", []), GolangVersion
+ )
cves = data.get("cves", [])
affected_packages = []
- if constraints:
- affected_packages.extend(
- [
- AffectedPackage(
- package=PackageURL(type="golang", namespace="istio.io", name="istio"),
- affected_version_range=GolangVersionRange(constraints=constraints),
- ),
- AffectedPackage(
- package=PackageURL(type="github", namespace="istio", name="istio"),
- affected_version_range=GitHubVersionRange(constraints=constraints),
- ),
- ]
+ if semver_constraints:
+ affected_packages.append(
+ AffectedPackageV2(
+ package=PackageURL(type="github", namespace="istio", name="istio"),
+ affected_version_range=GitHubVersionRange(constraints=semver_constraints),
+ ),
+ )
+
+ if golang_constraints:
+ affected_packages.append(
+ AffectedPackageV2(
+ package=PackageURL(type="golang", namespace="istio.io", name="istio"),
+ affected_version_range=GolangVersionRange(constraints=golang_constraints),
+ ),
)
title = data.get("title") or ""
@@ -127,37 +131,37 @@ def parse_markdown(self, path: Path) -> dict:
front_matter, _ = split_markdown_front_matter(text)
return saneyaml.load(front_matter)
- def get_version_constraints(self, releases: List[str]) -> List[VersionConstraint]:
+ def get_version_constraints(
+ self,
+ releases: List[str],
+ version_cls=SemverVersion,
+ ) -> List[VersionConstraint]:
constraints = []
for release in releases:
release = release.strip()
if "All releases prior" in release:
_, _, version = release.rpartition(" ")
- constraints.append(
- VersionConstraint(version=SemverVersion(version), comparator="<")
- )
+ constraints.append(VersionConstraint(version=version_cls(version), comparator="<"))
elif "All releases" in release and "and later" in release:
version = release.replace("All releases", "").replace("and later", "").strip()
if is_release(version):
constraints.append(
- VersionConstraint(version=SemverVersion(version), comparator=">=")
+ VersionConstraint(version=version_cls(version), comparator=">=")
)
elif "to" in release:
lower, _, upper = release.partition("to")
constraints.append(
- VersionConstraint(version=SemverVersion(lower.strip()), comparator=">=")
+ VersionConstraint(version=version_cls(lower.strip()), comparator=">=")
)
constraints.append(
- VersionConstraint(version=SemverVersion(upper.strip()), comparator="<=")
+ VersionConstraint(version=version_cls(upper.strip()), comparator="<=")
)
elif is_release(release):
- constraints.append(
- VersionConstraint(version=SemverVersion(release), comparator="=")
- )
+ constraints.append(VersionConstraint(version=version_cls(release), comparator="="))
return constraints
diff --git a/vulnerabilities/pipelines/v2_importers/mozilla_importer.py b/vulnerabilities/pipelines/v2_importers/mozilla_importer.py
index 668b6a498..e8a6aef7a 100644
--- a/vulnerabilities/pipelines/v2_importers/mozilla_importer.py
+++ b/vulnerabilities/pipelines/v2_importers/mozilla_importer.py
@@ -18,10 +18,10 @@
from fetchcode.vcs import fetch_via_vcs
from markdown import markdown
from packageurl import PackageURL
-from univers.versions import SemverVersion
+from univers.version_range import GenericVersionRange
from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import AffectedPackageV2
from vulnerabilities.importer import ReferenceV2
from vulnerabilities.importer import VulnerabilitySeverity
from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
@@ -189,7 +189,7 @@ def extract_description_from_html(md_text: str) -> str:
return "\n".join(description_parts).strip()
-def parse_affected_packages(pkgs: list) -> Iterable[AffectedPackage]:
+def parse_affected_packages(pkgs: list) -> Iterable[AffectedPackageV2]:
for pkg in pkgs:
if not pkg:
continue
@@ -198,14 +198,14 @@ def parse_affected_packages(pkgs: list) -> Iterable[AffectedPackage]:
if version.count(".") == 3:
continue # invalid SemVer
try:
- fixed_version = SemverVersion(version)
+ fixed_version_range = GenericVersionRange.from_versions([version])
except Exception:
logger.debug(f"Invalid version '{version}' for package '{name}'")
continue
- yield AffectedPackage(
+ yield AffectedPackageV2(
package=PackageURL(type="mozilla", name=name),
- fixed_version=fixed_version,
+ fixed_version_range=fixed_version_range,
)
diff --git a/vulnerabilities/pipelines/v2_importers/npm_importer.py b/vulnerabilities/pipelines/v2_importers/npm_importer.py
index 5f0306413..bf9de86d1 100644
--- a/vulnerabilities/pipelines/v2_importers/npm_importer.py
+++ b/vulnerabilities/pipelines/v2_importers/npm_importer.py
@@ -9,7 +9,6 @@
# Author: Navonil Das (@NavonilDas)
-import json
from pathlib import Path
from typing import Iterable
@@ -20,7 +19,7 @@
from univers.version_range import NpmVersionRange
from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import AffectedPackageV2
from vulnerabilities.importer import ReferenceV2
from vulnerabilities.importer import VulnerabilitySeverity
from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
@@ -32,7 +31,7 @@
class NpmImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
"""
- Node.js Security Working Group importer pipeline
+ Node.js Security Working Group importer pipeline.
Import advisories from nodejs security working group including node proper advisories and npm advisories.
"""
@@ -41,7 +40,6 @@ class NpmImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
spdx_license_expression = "MIT"
license_url = "https://github.com/nodejs/security-wg/blob/main/LICENSE.md"
repo_url = "git+https://github.com/nodejs/security-wg"
- unfurl_version_ranges = True
@classmethod
def steps(cls):
@@ -70,10 +68,12 @@ def to_advisory_data(self, file: Path) -> Iterable[AdvisoryData]:
self.log(f"Skipping {file.name} file")
return
data = load_json(file)
- advisory_text = None
- with open(file) as f:
- advisory_text = f.read()
+ advisory_text = file.read_text()
id = data.get("id")
+ if not id:
+ self.log(f"Advisory ID not found in {file}")
+ return
+
description = data.get("overview") or ""
summary = data.get("title") or ""
# TODO: Take care of description
@@ -100,9 +100,6 @@ def to_advisory_data(self, file: Path) -> Iterable[AdvisoryData]:
url=f"https://github.com/nodejs/security-wg/blob/main/vuln/npm/{id}.json",
)
)
- if not id:
- self.log(f"Advisory ID not found in {file}")
- return
advisory_reference = ReferenceV2(
url=f"https://github.com/nodejs/security-wg/blob/main/vuln/npm/{id}.json",
@@ -134,13 +131,12 @@ def to_advisory_data(self, file: Path) -> Iterable[AdvisoryData]:
references_v2=references,
severities=severities,
url=f"https://github.com/nodejs/security-wg/blob/main/vuln/npm/{id}.json",
- original_advisory_text=advisory_text or json.dumps(data, indent=2, ensure_ascii=False),
+ original_advisory_text=advisory_text,
)
def get_affected_package(self, data, package_name):
affected_version_range = None
unaffected_version_range = None
- fixed_version = None
vulnerable_range = data.get("vulnerable_versions") or ""
patched_range = data.get("patched_versions") or ""
@@ -157,21 +153,13 @@ def get_affected_package(self, data, package_name):
if patched_range:
unaffected_version_range = NpmVersionRange.from_native(patched_range)
- # We only store single fixed versions and not a range of fixed versions
- # If there is a single constraint in the unaffected_version_range
- # having comparator as ">=" then we store that as the fixed version
- if unaffected_version_range and len(unaffected_version_range.constraints) == 1:
- constraint = unaffected_version_range.constraints[0]
- if constraint.comparator == ">=":
- fixed_version = constraint.version
-
- return AffectedPackage(
+ return AffectedPackageV2(
package=PackageURL(
type="npm",
name=package_name,
),
affected_version_range=affected_version_range,
- fixed_version=fixed_version,
+ fixed_version_range=unaffected_version_range,
)
def clean_downloads(self):
diff --git a/vulnerabilities/pipelines/v2_importers/nvd_importer.py b/vulnerabilities/pipelines/v2_importers/nvd_importer.py
index 2b9de3bd8..876b7a905 100644
--- a/vulnerabilities/pipelines/v2_importers/nvd_importer.py
+++ b/vulnerabilities/pipelines/v2_importers/nvd_importer.py
@@ -111,7 +111,7 @@ def fetch(url, logger=None):
return json.loads(data)
-def fetch_cve_data_1_1(starting_year=2002, logger=None):
+def fetch_cve_data_1_1(starting_year=2025, logger=None):
"""
Yield tuples of (year, lists of CVE mappings) from the NVD, one for each
year since ``starting_year`` defaulting to 2002.
@@ -326,7 +326,7 @@ def to_advisory(self):
weaknesses=self.weaknesses,
severities=self.severities,
url=f"https://nvd.nist.gov/vuln/detail/{self.cve_id}",
- raw_data=json.dumps(self.cve_item, indent=2, ensure_ascii=False),
+ original_advisory_text=json.dumps(self.cve_item, indent=2, ensure_ascii=False),
)
diff --git a/vulnerabilities/pipelines/v2_importers/oss_fuzz.py b/vulnerabilities/pipelines/v2_importers/oss_fuzz.py
index 9c9d6bed8..9c5f78d90 100644
--- a/vulnerabilities/pipelines/v2_importers/oss_fuzz.py
+++ b/vulnerabilities/pipelines/v2_importers/oss_fuzz.py
@@ -25,7 +25,6 @@ class OSSFuzzImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
spdx_license_expression = "CC-BY-4.0"
license_url = "https://github.com/google/oss-fuzz-vulns/blob/main/LICENSE"
repo_url = "git+https://github.com/google/oss-fuzz-vulns"
- unfurl_version_ranges = True
@classmethod
def steps(cls):
diff --git a/vulnerabilities/pipelines/v2_importers/postgresql_importer.py b/vulnerabilities/pipelines/v2_importers/postgresql_importer.py
index 60b099ff6..2ca4c7b5b 100644
--- a/vulnerabilities/pipelines/v2_importers/postgresql_importer.py
+++ b/vulnerabilities/pipelines/v2_importers/postgresql_importer.py
@@ -18,7 +18,7 @@
from vulnerabilities import severity_systems
from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import AffectedPackageV2
from vulnerabilities.importer import ReferenceV2
from vulnerabilities.importer import VulnerabilitySeverity
from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
@@ -83,31 +83,25 @@ def to_advisories(self, data, url):
affected_packages = []
affected_version_list = [v.strip() for v in affected_col.text.split(",") if v.strip()]
fixed_version_list = [v.strip() for v in fixed_col.text.split(",") if v.strip()]
-
- if fixed_version_list:
- for fixed_version in fixed_version_list:
- affected_packages.append(
- AffectedPackage(
- package=PackageURL(
- name="postgresql", type="generic", qualifiers=pkg_qualifiers
- ),
- affected_version_range=GenericVersionRange.from_versions(
- affected_version_list
- )
- if affected_version_list
- else None,
- fixed_version=GenericVersion(fixed_version),
- )
- )
- elif affected_version_list:
+ fixed_version_range = (
+ GenericVersionRange.from_versions(fixed_version_list)
+ if fixed_version_list
+ else None
+ )
+ affected_version_range = (
+ GenericVersionRange.from_versions(affected_version_list)
+ if affected_version_list
+ else None
+ )
+
+ if affected_version_range or fixed_version_range:
affected_packages.append(
- AffectedPackage(
+ AffectedPackageV2(
package=PackageURL(
name="postgresql", type="generic", qualifiers=pkg_qualifiers
),
- affected_version_range=GenericVersionRange.from_versions(
- affected_version_list
- ),
+ affected_version_range=affected_version_range,
+ fixed_version_range=fixed_version_range,
)
)
diff --git a/vulnerabilities/pipelines/v2_importers/pypa_importer.py b/vulnerabilities/pipelines/v2_importers/pypa_importer.py
index 8f9f57ddf..fa5edaa0b 100644
--- a/vulnerabilities/pipelines/v2_importers/pypa_importer.py
+++ b/vulnerabilities/pipelines/v2_importers/pypa_importer.py
@@ -28,7 +28,6 @@ class PyPaImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
spdx_license_expression = "CC-BY-4.0"
license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE"
repo_url = "git+https://github.com/pypa/advisory-database"
- unfurl_version_ranges = True
@classmethod
def steps(cls):
diff --git a/vulnerabilities/pipelines/v2_importers/pysec_importer.py b/vulnerabilities/pipelines/v2_importers/pysec_importer.py
index ed41fdc87..bea79cb98 100644
--- a/vulnerabilities/pipelines/v2_importers/pysec_importer.py
+++ b/vulnerabilities/pipelines/v2_importers/pysec_importer.py
@@ -28,7 +28,6 @@ class PyPIImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE"
url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip"
spdx_license_expression = "CC-BY-4.0"
- unfurl_version_ranges = True
@classmethod
def steps(cls):
diff --git a/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py b/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py
index d8590b498..a596d8d65 100644
--- a/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py
+++ b/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py
@@ -69,7 +69,14 @@ def parse_cve_advisory(self, raw_data, advisory_url):
date_published = cve_metadata.get("datePublished")
if date_published:
- date_published = dateparser.parse(date_published)
+ date_published = dateparser.parse(
+ date_published,
+ settings={
+ "TIMEZONE": "UTC",
+ "RETURN_AS_TIMEZONE_AWARE": True,
+ "TO_TIMEZONE": "UTC",
+ },
+ )
# Extract containers
containers = raw_data.get("containers", {})
diff --git a/vulnerabilities/pipelines/v2_improvers/collect_commits.py b/vulnerabilities/pipelines/v2_improvers/collect_commits.py
index 32fb1ce79..d9be9781e 100644
--- a/vulnerabilities/pipelines/v2_improvers/collect_commits.py
+++ b/vulnerabilities/pipelines/v2_improvers/collect_commits.py
@@ -37,8 +37,8 @@ def steps(cls):
def collect_and_store_fix_commits(self):
affected_advisories = (
- AdvisoryV2.objects.filter(affecting_packages__isnull=False)
- .prefetch_related("affecting_packages")
+ AdvisoryV2.objects.filter(impacted_packages__affecting_packages__isnull=False)
+ .prefetch_related("impacted_packages__affecting_packages", "references")
.distinct()
)
@@ -61,24 +61,19 @@ def collect_and_store_fix_commits(self):
# Skip if already processed
if is_vcs_url_already_processed(commit_id=vcs_url):
- self.log(
- f"Skipping already processed reference: {reference.url} with VCS URL {vcs_url}"
- )
continue
# check if vcs_url has commit
- for package in adv.affecting_packages.all():
- code_fix, created = CodeFixV2.objects.get_or_create(
- commits=[vcs_url],
- advisory=adv,
- affected_package=package,
- )
-
- if created:
- created_fix_count += 1
- self.log(
- f"Created CodeFix entry for reference: {reference.url} with VCS URL {vcs_url}"
+ for impact in adv.impacted_packages.all():
+ for package in impact.affecting_packages.all():
+ code_fix, created = CodeFixV2.objects.get_or_create(
+ commits=[vcs_url],
+ advisory=adv,
+ affected_package=package,
)
+ if created:
+ created_fix_count += 1
+
self.log(f"Successfully created {created_fix_count:,d} CodeFix entries.")
diff --git a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py
index 55608f0d1..ac7caa49d 100644
--- a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py
+++ b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py
@@ -34,17 +34,13 @@ def steps(cls):
def compute_and_store_vulnerability_risk_score(self):
affected_advisories = (
- AdvisoryV2.objects.filter(affecting_packages__isnull=False)
- .prefetch_related(
- "references",
- "severities",
- "exploits",
- )
+ AdvisoryV2.objects.filter(impacted_packages__affecting_packages__isnull=False)
+ .prefetch_related("references", "severities", "exploits")
.distinct()
)
self.log(
- f"Calculating risk for {affected_advisories.count():,d} vulnerability with a affected packages records"
+ f"Calculating risk for {affected_advisories.count():,d} advisory with a affected packages records"
)
progress = LoopProgress(total_iterations=affected_advisories.count(), logger=self.log)
@@ -53,7 +49,7 @@ def compute_and_store_vulnerability_risk_score(self):
updated_vulnerability_count = 0
batch_size = 5000
- for advisory in progress.iter(affected_advisories.paginated(per_page=batch_size)):
+ for advisory in progress.iter(affected_advisories.iterator(chunk_size=batch_size)):
severities = advisory.severities.all()
references = advisory.references.all()
exploits = advisory.exploits.all()
@@ -65,9 +61,6 @@ def compute_and_store_vulnerability_risk_score(self):
)
advisory.weighted_severity = weighted_severity
advisory.exploitability = exploitability
- print(
- f"Computed risk for {advisory.advisory_id} with weighted_severity={weighted_severity} and exploitability={exploitability}"
- )
updatables.append(advisory)
if len(updatables) >= batch_size:
@@ -90,9 +83,7 @@ def compute_and_store_vulnerability_risk_score(self):
)
def compute_and_store_package_risk_score(self):
- affected_packages = (
- PackageV2.objects.filter(affected_by_advisories__isnull=False)
- ).distinct()
+ affected_packages = (PackageV2.objects.filter(affected_in_impacts__isnull=False)).distinct()
self.log(f"Calculating risk for {affected_packages.count():,d} affected package records")
@@ -106,7 +97,7 @@ def compute_and_store_package_risk_score(self):
updated_package_count = 0
batch_size = 10000
- for package in progress.iter(affected_packages.paginated(per_page=batch_size)):
+ for package in progress.iter(affected_packages.iterator(chunk_size=batch_size)):
risk_score = compute_package_risk_v2(package)
if not risk_score:
diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py
index 2736d8874..3487b74db 100644
--- a/vulnerabilities/pipes/advisory.py
+++ b/vulnerabilities/pipes/advisory.py
@@ -35,6 +35,7 @@
from vulnerabilities.models import VulnerabilityRelatedReference
from vulnerabilities.models import VulnerabilitySeverity
from vulnerabilities.models import Weakness
+from vulnerabilities.pipes.univers_utils import get_exact_purls_v2
def get_or_create_aliases(aliases: List) -> QuerySet:
@@ -43,9 +44,6 @@ def get_or_create_aliases(aliases: List) -> QuerySet:
return Alias.objects.filter(alias__in=aliases)
-from django.db.models import Q
-
-
def get_or_create_advisory_aliases(aliases: List[str]) -> List[AdvisoryAlias]:
existing = AdvisoryAlias.objects.filter(alias__in=aliases)
existing_aliases = {a.alias for a in existing}
@@ -136,12 +134,14 @@ def insert_advisory(advisory: AdvisoryData, pipeline_id: str, logger: Callable =
return advisory_obj
+@transaction.atomic
def insert_advisory_v2(
advisory: AdvisoryData,
pipeline_id: str,
- get_advisory_packages: Callable,
logger: Callable = None,
):
+ from vulnerabilities.models import ImpactedPackage
+ from vulnerabilities.models import PackageV2
from vulnerabilities.utils import compute_content_id
advisory_obj = None
@@ -150,7 +150,6 @@ def insert_advisory_v2(
severities = get_or_create_advisory_severities(severities=advisory.severities)
weaknesses = get_or_create_advisory_weaknesses(weaknesses=advisory.weaknesses)
content_id = compute_content_id(advisory_data=advisory)
- affecting_packages, fixed_by_packages = get_advisory_packages(advisory_data=advisory)
try:
default_data = {
"datasource_id": pipeline_id,
@@ -162,7 +161,7 @@ def insert_advisory_v2(
"original_advisory_text": advisory.original_advisory_text,
}
- advisory_obj, _ = AdvisoryV2.objects.get_or_create(
+ advisory_obj, created = AdvisoryV2.objects.get_or_create(
unique_content_id=content_id,
url=advisory.url,
defaults=default_data,
@@ -172,8 +171,6 @@ def insert_advisory_v2(
"references": references,
"severities": severities,
"weaknesses": weaknesses,
- "fixed_by_packages": fixed_by_packages,
- "affecting_packages": affecting_packages,
}
for field_name, values in related_fields.items():
@@ -192,6 +189,29 @@ def insert_advisory_v2(
level=logging.ERROR,
)
+ if created:
+ for affected_pkg in advisory.affected_packages:
+ impact = ImpactedPackage.objects.create(
+ advisory=advisory_obj,
+ base_purl=str(affected_pkg.package),
+ affecting_vers=str(affected_pkg.affected_version_range),
+ fixed_vers=str(affected_pkg.fixed_version_range),
+ )
+ package_affected_purls, package_fixed_purls = get_exact_purls_v2(
+ affected_package=affected_pkg,
+ logger=logger,
+ )
+ affected_packages_v2 = [
+ PackageV2.objects.get_or_create_from_purl(purl=purl)[0]
+ for purl in package_affected_purls
+ ]
+ fixed_packages_v2 = [
+ PackageV2.objects.get_or_create_from_purl(purl=purl)[0]
+ for purl in package_fixed_purls
+ ]
+ impact.affecting_packages.add(*affected_packages_v2)
+ impact.fixed_by_packages.add(*fixed_packages_v2)
+
return advisory_obj
diff --git a/vulnerabilities/pipes/univers_utils.py b/vulnerabilities/pipes/univers_utils.py
new file mode 100644
index 000000000..296ebc12f
--- /dev/null
+++ b/vulnerabilities/pipes/univers_utils.py
@@ -0,0 +1,95 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+from traceback import format_exc as traceback_format_exc
+from typing import Callable
+from typing import List
+from typing import Tuple
+
+from packageurl import PackageURL
+from univers.version_range import VersionRange
+
+from vulnerabilities.importer import AffectedPackageV2
+from vulnerabilities.utils import update_purl_version
+
+
+def get_exact_purl_from_vers(
+ version_range: VersionRange, base_purl: PackageURL
+) -> Tuple[List[PackageURL], List[PackageURL]]:
+ """Return list of exact PURLs that in and outside vers range."""
+ if not version_range:
+ return [], []
+
+ all_versions = [c.version for c in version_range.constraints if c]
+ version_not_in_range = [
+ c.version for c in version_range.constraints if c and c.comparator == "!="
+ ]
+ version_in_range = [v for v in all_versions if v and v in version_range]
+
+ purl_in_range = [
+ update_purl_version(purl=base_purl, version=str(version)) for version in version_in_range
+ ]
+ purl_not_in_range = [
+ update_purl_version(purl=base_purl, version=str(version))
+ for version in version_not_in_range
+ ]
+
+ return purl_in_range, purl_not_in_range
+
+
+def get_exact_purls_v2(
+ affected_package: AffectedPackageV2,
+ logger: Callable = None,
+) -> Tuple[List[PackageURL], PackageURL]:
+ """
+ Return a list of affected purls and the fixed package found in the ``affected_package``
+ AffectedPackageV2 disregarding any ranges.
+
+ Only exact version constraints (ie with an equality) are considered
+ For eg:
+ >>> purl = {"type": "turtle", "name": "green"}
+ >>> vers = "vers:npm/<1.0.0 | >=2.0.0 | <3.0.0"
+ >>> vers2 = "vers:npm/5.0.0"
+ >>> affected_package = AffectedPackageV2.from_dict({
+ ... "package": purl,
+ ... "affected_version_range": vers,
+ ... "fixed_version_range": vers2
+ ... })
+ >>> got = get_exact_purls_v2(affected_package)
+ >>> expected = (
+ ... [PackageURL(type='turtle', namespace=None, name='green', version='2.0.0', qualifiers={}, subpath=None)],
+ ... [PackageURL(type='turtle', namespace=None, name='green', version='5.0.0', qualifiers={}, subpath=None)]
+ ... )
+ >>> assert expected == got
+ """
+ if not affected_package:
+ return [], []
+
+ try:
+ affected_purls, fixed_purls = get_exact_purl_from_vers(
+ version_range=affected_package.affected_version_range,
+ base_purl=affected_package.package,
+ )
+
+ fixed_purls_2, affected_purls_2 = get_exact_purl_from_vers(
+ version_range=affected_package.fixed_version_range,
+ base_purl=affected_package.package,
+ )
+
+ affected_purls.extend(affected_purls_2)
+ fixed_purls.extend(fixed_purls_2)
+
+ return affected_purls, fixed_purls
+ except Exception as e:
+ logger(
+ f"Failed to get exact purls for: {affected_package!r} with error: {e!r} \n{traceback_format_exc()}",
+ level=logging.ERROR,
+ )
+ return [], []
diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py
index 56f19171e..04af140f0 100644
--- a/vulnerabilities/risk.py
+++ b/vulnerabilities/risk.py
@@ -122,8 +122,8 @@ def compute_package_risk_v2(package):
and determining the associated risk.
"""
result = []
- for advisory in package.affected_by_advisories.all():
- if risk := advisory.risk_score:
+ for impact in package.affected_in_impacts.all():
+ if risk := impact.advisory.risk_score:
result.append(float(risk))
if not result:
diff --git a/vulnerabilities/templates/advisory_package_details.html b/vulnerabilities/templates/advisory_package_details.html
index 0f4c71044..234c5bae0 100644
--- a/vulnerabilities/templates/advisory_package_details.html
+++ b/vulnerabilities/templates/advisory_package_details.html
@@ -6,83 +6,62 @@
{% load url_filters %}
{% block title %}
-VulnerableCode Advisory Package Details - {{ advisory.advisory_id }}
+VulnerableCode Advisory Package Details - {{ advisoryv2.advisory_id }}
{% endblock %}
{% block content %}
-{% if advisory %}
-
-
-
-
-
-
-
- | Affected |
- Fixed by |
-
-
-
- {% for package in affected_packages %}
-
- |
- {{ package.purl }}
- |
-
-
- {% for match in all_affected_fixed_by_matches %}
- {% if match.affected_package == package %}
- {% if match.matched_fixed_by_packages|length > 0 %}
- {% for pkg in match.matched_fixed_by_packages %}
- {{ pkg }}
-
- {% endfor %}
- {% else %}
+{% if advisoryv2 %}
+
+
+
+
+
+
+
+ | Affected |
+ Fixed by |
+
+
+
+ {% for impact in advisoryv2.impacted_packages.all %}
+
+
+ {% for package in impact.affecting_packages.all %}
+ {{ package.purl }}
+
+ {% endfor %}
+ |
+
+ {% for package in impact.fixed_by_packages.all %}
+ {{ package.purl }}
+
+ {% empty %}
There are no reported fixed by versions.
- {% endif %}
- {% endif %}
- {% endfor %}
-
- |
-
- {% empty %}
-
- |
- This vulnerability is not known to affect any packages.
- |
-
- {% endfor %}
-
-
-
-
-
+ {% endfor %}
+ |
+
+ {% empty %}
+
+ |
+ This vulnerability is not known to affect any packages.
+ |
+
+ {% endfor %}
+
+
+
+
+
{% endif %}
-
-
{% endblock %}
\ No newline at end of file
diff --git a/vulnerabilities/templates/package_details_v2.html b/vulnerabilities/templates/package_details_v2.html
index be0af62fe..1af0866ef 100644
--- a/vulnerabilities/templates/package_details_v2.html
+++ b/vulnerabilities/templates/package_details_v2.html
@@ -3,6 +3,7 @@
{% load widget_tweaks %}
{% load static %}
{% load url_filters %}
+{% load utils %}
{% block title %}
VulnerableCode Package Details - {{ package.purl }}
@@ -60,7 +61,7 @@
- {{ fixed_package_details.purl.to_string }}
+ {{ package.purl }}
|
{% if package.is_ghost %}
@@ -91,9 +92,9 @@
Next non-vulnerable version
- {% if fixed_package_details.next_non_vulnerable.version %}
- {{ fixed_package_details.next_non_vulnerable.version }}
+ {% if next_non_vulnerable.version %}
+ {{ next_non_vulnerable.version }}
{% else %}
None.
{% endif %}
@@ -104,9 +105,9 @@
Latest non-vulnerable version
|
- {% if fixed_package_details.latest_non_vulnerable.version %}
- {{ fixed_package_details.latest_non_vulnerable.version }}
+ {% if latest_non_vulnerable.version %}
+ {{ latest_non_vulnerable.version }}
{% else %}
None.
{% endif %}
@@ -175,66 +176,22 @@
{{ advisory.summary }}
|
- {% if package.purl == fixed_package_details.purl.to_string %}
- {% for key, value in fixed_package_details.items %}
- {% if key == "advisories" %}
- {% for vuln in value %}
- {% if vuln.advisory.advisory_id == advisory.advisory_id %}
- {% if vuln.fixed_by_package_details is None %}
- There are no reported fixed by versions.
- {% else %}
- {% for fixed_pkg in vuln.fixed_by_package_details %}
-
- {% if fixed_pkg.fixed_by_purl_advisories|length == 0 %}
- {{ fixed_pkg.fixed_by_purl.version }}
-
- Subject of 0 other advisories.
- {% else %}
- {{ fixed_pkg.fixed_by_purl.version }}
- {% if fixed_pkg.fixed_by_purl_advisories|length != 1 %}
-
- Subject of {{ fixed_pkg.fixed_by_purl_advisories|length }} other
- advisory.
- {% else %}
-
- Subject of {{ fixed_pkg.fixed_by_purl_advisories|length }} other
- advisory.
- {% endif %}
-
-
- {% endif %}
-
- {% endfor %}
- {% endif %}
- {% endif %}
- {% endfor %}
- {% endif %}
- {% endfor %}
- {% endif %}
+ {% with fixed=fixed_package_details|get_item:advisory.id %}
+ {% if fixed %}
+ {% for item in fixed %}
+
+ {% endfor %}
+ {% else %}
+ There are no reported fixed by versions.
+ {% endif %}
+ {% endwith %}
|
{% empty %}
diff --git a/vulnerabilities/templatetags/utils.py b/vulnerabilities/templatetags/utils.py
index 952221f48..cea889808 100644
--- a/vulnerabilities/templatetags/utils.py
+++ b/vulnerabilities/templatetags/utils.py
@@ -34,3 +34,8 @@ def active_item(context, url_name):
if request.resolver_match.url_name == url_name:
return "is-active"
return ""
+
+
+@register.filter
+def get_item(dictionary, key):
+ return dictionary.get(key)
diff --git a/vulnerabilities/tests/pipelines/test_vulnerablecode_importer_v2_pipeline.py b/vulnerabilities/tests/pipelines/test_vulnerablecode_importer_v2_pipeline.py
deleted file mode 100644
index f995f0c1f..000000000
--- a/vulnerabilities/tests/pipelines/test_vulnerablecode_importer_v2_pipeline.py
+++ /dev/null
@@ -1,180 +0,0 @@
-#
-# Copyright (c) nexB Inc. and others. All rights reserved.
-# VulnerableCode is a trademark of nexB Inc.
-# SPDX-License-Identifier: Apache-2.0
-# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
-# See https://github.com/aboutcode-org/vulnerablecode for support or download.
-# See https://aboutcode.org for more information about nexB OSS projects.
-#
-
-import logging
-from datetime import datetime
-from datetime import timedelta
-from unittest import mock
-
-import pytest
-from packageurl import PackageURL
-
-from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import UnMergeablePackageError
-from vulnerabilities.models import AdvisoryV2
-from vulnerabilities.models import PackageV2
-from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
-
-
-class DummyImporter(VulnerableCodeBaseImporterPipelineV2):
- pipeline_id = "dummy"
- log_messages = []
-
- def log(self, message, level=logging.INFO):
- self.log_messages.append((level, message))
-
- def collect_advisories(self):
- yield from self._advisories
-
- def advisories_count(self):
- return len(self._advisories)
-
-
-@pytest.fixture
-def dummy_advisory():
- return AdvisoryData(
- summary="Test advisory",
- aliases=["CVE-2025-0001"],
- references_v2=[],
- severities=[],
- weaknesses=[],
- affected_packages=[],
- advisory_id="ADV-123",
- date_published=datetime.now() - timedelta(days=10),
- url="https://example.com/advisory/1",
- )
-
-
-@pytest.fixture
-def dummy_importer(dummy_advisory):
- importer = DummyImporter()
- importer._advisories = [dummy_advisory]
- return importer
-
-
-@pytest.mark.django_db
-def test_collect_and_store_advisories(dummy_importer):
- dummy_importer.collect_and_store_advisories()
- assert len(dummy_importer.log_messages) >= 2
- assert "Successfully collected" in dummy_importer.log_messages[-1][1]
- assert AdvisoryV2.objects.count() == 1
-
-
-def test_get_advisory_packages_basic(dummy_importer):
- purl = PackageURL("pypi", None, "dummy", "1.0.0")
- affected_package = mock.Mock()
- affected_package.package = purl
- dummy_importer.unfurl_version_ranges = False
-
- with mock.patch(
- "vulnerabilities.improvers.default.get_exact_purls", return_value=([purl], [purl])
- ):
- with mock.patch.object(
- PackageV2.objects, "get_or_create_from_purl", return_value=(mock.Mock(), True)
- ) as mock_get:
- dummy_importer.get_advisory_packages(
- advisory_data=mock.Mock(affected_packages=[affected_package])
- )
- assert mock_get.call_count == 2 # one affected, one fixed
-
-
-def test_get_published_package_versions_filters(dummy_importer):
- purl = PackageURL("pypi", None, "example", None)
-
- dummy_versions = [
- mock.Mock(value="1.0.0", release_date=datetime.now() - timedelta(days=5)),
- mock.Mock(value="2.0.0", release_date=datetime.now() + timedelta(days=5)), # future
- ]
-
- with mock.patch(
- "vulnerabilities.pipelines.package_versions.versions", return_value=dummy_versions
- ):
- versions = dummy_importer.get_published_package_versions(purl, until=datetime.now())
- assert "1.0.0" in versions
- assert "2.0.0" not in versions
-
-
-def test_get_published_package_versions_failure_logs(dummy_importer):
- purl = PackageURL("pypi", None, "example", None)
- with mock.patch(
- "vulnerabilities.pipelines.package_versions.versions", side_effect=Exception("fail")
- ):
- versions = dummy_importer.get_published_package_versions(purl)
- assert versions == []
- assert any("Failed to fetch versions" in msg for lvl, msg in dummy_importer.log_messages)
-
-
-def test_expand_version_range_to_purls(dummy_importer):
- purls = list(
- dummy_importer.expand_verion_range_to_purls("npm", "lodash", "lodash", ["1.0.0", "1.1.0"])
- )
- assert all(isinstance(p, PackageURL) for p in purls)
- assert purls[0].name == "lodash"
-
-
-def test_resolve_package_versions(dummy_importer):
- dummy_importer.ignorable_versions = []
- dummy_importer.expand_verion_range_to_purls = lambda *args, **kwargs: [
- PackageURL("npm", None, "a", "1.0.0")
- ]
-
- with mock.patch(
- "vulnerabilities.pipelines.resolve_version_range", return_value=(["1.0.0"], ["1.1.0"])
- ), mock.patch(
- "vulnerabilities.pipelines.get_affected_packages_by_patched_package",
- return_value={None: [PackageURL("npm", None, "a", "1.0.0")]},
- ), mock.patch(
- "vulnerabilities.pipelines.nearest_patched_package", return_value=[]
- ):
- aff, fix = dummy_importer.resolve_package_versions(
- affected_version_range=">=1.0.0",
- pkg_type="npm",
- pkg_namespace=None,
- pkg_name="a",
- valid_versions=["1.0.0", "1.1.0"],
- )
- assert any(isinstance(p, PackageURL) for p in aff)
-
-
-def test_get_impacted_packages_mergeable(dummy_importer):
- ap = mock.Mock()
- ap.package = PackageURL("npm", None, "abc", None)
- dummy_importer.get_published_package_versions = lambda package_url, until: ["1.0.0", "1.1.0"]
- dummy_importer.resolve_package_versions = lambda **kwargs: (
- [PackageURL("npm", None, "abc", "1.0.0")],
- [PackageURL("npm", None, "abc", "1.1.0")],
- )
-
- with mock.patch(
- "vulnerabilities.importer.AffectedPackage.merge",
- return_value=(ap.package, [">=1.0.0"], ["1.1.0"]),
- ):
- aff, fix = dummy_importer.get_impacted_packages([ap], datetime.now())
- assert len(aff) == 1 and aff[0].version == "1.0.0"
- assert len(fix) == 1 and fix[0].version == "1.1.0"
-
-
-def test_get_impacted_packages_unmergeable(dummy_importer):
- ap = mock.Mock()
- ap.package = PackageURL("npm", None, "abc", None)
- ap.affected_version_range = ">=1.0.0"
- ap.fixed_version = None
-
- dummy_importer.get_published_package_versions = lambda package_url, until: ["1.0.0", "1.1.0"]
- dummy_importer.resolve_package_versions = lambda **kwargs: (
- [PackageURL("npm", None, "abc", "1.0.0")],
- [PackageURL("npm", None, "abc", "1.1.0")],
- )
-
- with mock.patch(
- "vulnerabilities.importer.AffectedPackage.merge", side_effect=UnMergeablePackageError
- ):
- aff, fix = dummy_importer.get_impacted_packages([ap], datetime.utcnow())
- assert len(aff) == 1
- assert aff[0].version == "1.0.0"
diff --git a/vulnerabilities/tests/pipelines/test_apache_httpd_importer_pipeline_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_apache_httpd_importer_pipeline_v2.py
similarity index 100%
rename from vulnerabilities/tests/pipelines/test_apache_httpd_importer_pipeline_v2.py
rename to vulnerabilities/tests/pipelines/v2_importers/test_apache_httpd_importer_pipeline_v2.py
diff --git a/vulnerabilities/tests/pipelines/test_curl_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_curl_importer_v2.py
similarity index 96%
rename from vulnerabilities/tests/pipelines/test_curl_importer_v2.py
rename to vulnerabilities/tests/pipelines/v2_importers/test_curl_importer_v2.py
index 7833d1397..6157d8bbe 100644
--- a/vulnerabilities/tests/pipelines/test_curl_importer_v2.py
+++ b/vulnerabilities/tests/pipelines/v2_importers/test_curl_importer_v2.py
@@ -16,7 +16,7 @@
from univers.versions import SemverVersion
from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import AffectedPackageV2
from vulnerabilities.pipelines.v2_importers.curl_importer import CurlImporterPipeline
from vulnerabilities.pipelines.v2_importers.curl_importer import get_cwe_from_curl_advisory
from vulnerabilities.pipelines.v2_importers.curl_importer import parse_curl_advisory
@@ -74,9 +74,9 @@ def test_collect_advisories(mock_fetch, pipeline):
# Affected package check
pkg = advisory.affected_packages[0]
- assert isinstance(pkg, AffectedPackage)
+ assert isinstance(pkg, AffectedPackageV2)
assert pkg.package == PackageURL(type="generic", namespace="curl.se", name="curl")
- assert pkg.fixed_version == SemverVersion("8.7.0")
+ assert str(pkg.fixed_version_range) == "vers:generic/8.7.0"
assert "8.6.0" in str(pkg.affected_version_range)
# References
diff --git a/vulnerabilities/tests/pipelines/test_elixir_security_v2_importer.py b/vulnerabilities/tests/pipelines/v2_importers/test_elixir_security_importer_v2.py
similarity index 100%
rename from vulnerabilities/tests/pipelines/test_elixir_security_v2_importer.py
rename to vulnerabilities/tests/pipelines/v2_importers/test_elixir_security_importer_v2.py
diff --git a/vulnerabilities/tests/pipelines/test_github_osv_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_github_osv_importer_v2.py
similarity index 100%
rename from vulnerabilities/tests/pipelines/test_github_osv_importer_v2.py
rename to vulnerabilities/tests/pipelines/v2_importers/test_github_osv_importer_v2.py
diff --git a/vulnerabilities/tests/pipelines/test_gitlab_v2_importer.py b/vulnerabilities/tests/pipelines/v2_importers/test_gitlab_importer_v2.py
similarity index 98%
rename from vulnerabilities/tests/pipelines/test_gitlab_v2_importer.py
rename to vulnerabilities/tests/pipelines/v2_importers/test_gitlab_importer_v2.py
index 195be6609..4ac781080 100644
--- a/vulnerabilities/tests/pipelines/test_gitlab_v2_importer.py
+++ b/vulnerabilities/tests/pipelines/v2_importers/test_gitlab_importer_v2.py
@@ -99,7 +99,7 @@ def test_collect_advisories(mock_gitlab_yaml, mock_vcs_response, mock_fetch_via_
assert advisory.summary == "Example vulnerability\nExample description"
assert advisory.references_v2[0].url == "https://example.com/advisory"
assert advisory.affected_packages[0].package.name == "package-name"
- assert advisory.affected_packages[0].fixed_version
+ assert str(advisory.affected_packages[0].fixed_version_range) == "vers:pypi/2.0.0"
assert advisory.weaknesses[0] == 79
diff --git a/vulnerabilities/tests/pipelines/test_istio_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_istio_importer_v2.py
similarity index 66%
rename from vulnerabilities/tests/pipelines/test_istio_importer_v2.py
rename to vulnerabilities/tests/pipelines/v2_importers/test_istio_importer_v2.py
index 162eddbbe..ba5289b1c 100644
--- a/vulnerabilities/tests/pipelines/test_istio_importer_v2.py
+++ b/vulnerabilities/tests/pipelines/v2_importers/test_istio_importer_v2.py
@@ -12,14 +12,8 @@
from textwrap import dedent
import pytest
-from packageurl import PackageURL
-from univers.version_constraint import VersionConstraint
-from univers.version_range import GitHubVersionRange
-from univers.version_range import GolangVersionRange
-from univers.versions import SemverVersion
from vulnerabilities.importer import AdvisoryData
-from vulnerabilities.importer import AffectedPackage
from vulnerabilities.importer import ReferenceV2
from vulnerabilities.pipelines.v2_importers.istio_importer import IstioImporterPipeline
@@ -77,24 +71,11 @@ def test_istio_advisory_parsing():
url="https://istio.io/latest/news/security/ISTIO-SECURITY-2019-002/",
)
- expected_versions = [
- VersionConstraint(version=SemverVersion("1.0"), comparator=">="),
- VersionConstraint(version=SemverVersion("1.0.8"), comparator="<="),
- VersionConstraint(version=SemverVersion("1.1"), comparator=">="),
- VersionConstraint(version=SemverVersion("1.1.9"), comparator="<="),
- VersionConstraint(version=SemverVersion("1.2"), comparator=">="),
- VersionConstraint(version=SemverVersion("1.2.1"), comparator="<="),
- ]
-
- expected_packages = [
- AffectedPackage(
- package=PackageURL(type="golang", namespace="istio.io", name="istio"),
- affected_version_range=GolangVersionRange(constraints=expected_versions),
- ),
- AffectedPackage(
- package=PackageURL(type="github", namespace="istio", name="istio"),
- affected_version_range=GitHubVersionRange(constraints=expected_versions),
- ),
- ]
-
- assert advisory.affected_packages == expected_packages
+ assert (
+ str(advisory.affected_packages[0].affected_version_range)
+ == "vers:github/>=1.0.0|<=1.0.8|>=1.1.0|<=1.1.9|>=1.2.0|<=1.2.1"
+ )
+ assert (
+ str(advisory.affected_packages[1].affected_version_range)
+ == "vers:golang/>=1.0.0|<=1.0.8|>=1.1.0|<=1.1.9|>=1.2.0|<=1.2.1"
+ )
diff --git a/vulnerabilities/tests/pipelines/test_mozilla_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_mozilla_importer_v2.py
similarity index 95%
rename from vulnerabilities/tests/pipelines/test_mozilla_importer_v2.py
rename to vulnerabilities/tests/pipelines/v2_importers/test_mozilla_importer_v2.py
index 556a609ac..c468b3684 100644
--- a/vulnerabilities/tests/pipelines/test_mozilla_importer_v2.py
+++ b/vulnerabilities/tests/pipelines/v2_importers/test_mozilla_importer_v2.py
@@ -14,7 +14,6 @@
from vulnerabilities.pipelines.v2_importers.mozilla_importer import get_severity_from_impact
from vulnerabilities.pipelines.v2_importers.mozilla_importer import mfsa_id_from_filename
from vulnerabilities.pipelines.v2_importers.mozilla_importer import parse_affected_packages
-from vulnerabilities.pipelines.v2_importers.mozilla_importer import parse_md_advisory
from vulnerabilities.pipelines.v2_importers.mozilla_importer import parse_yml_advisory
@@ -57,7 +56,7 @@ def test_parse_affected_packages_valid():
result = list(parse_affected_packages(packages))
assert len(result) == 2
assert result[0].package.name == "firefox"
- assert str(result[0].fixed_version) == "89.0.0"
+ assert str(result[0].fixed_version_range) == "vers:generic/89.0.0"
def test_parse_affected_packages_invalid():
diff --git a/vulnerabilities/tests/pipelines/test_npm_importer_pipeline_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_npm_importer_pipeline_v2.py
similarity index 96%
rename from vulnerabilities/tests/pipelines/test_npm_importer_pipeline_v2.py
rename to vulnerabilities/tests/pipelines/v2_importers/test_npm_importer_pipeline_v2.py
index 7941c9b69..67fcc2970 100644
--- a/vulnerabilities/tests/pipelines/test_npm_importer_pipeline_v2.py
+++ b/vulnerabilities/tests/pipelines/v2_importers/test_npm_importer_pipeline_v2.py
@@ -102,7 +102,7 @@ def test_to_advisory_data_full(tmp_path):
pkg = adv.affected_packages[0]
assert pkg.package == PackageURL(type="npm", name="mypkg")
assert isinstance(pkg.affected_version_range, NpmVersionRange)
- assert pkg.fixed_version == SemverVersion("1.2.4")
+ assert str(pkg.fixed_version_range) == "vers:npm/>=1.2.4"
assert set(adv.aliases) == {"CVE-123", "CVE-124"}
@@ -121,8 +121,8 @@ def test_get_affected_package_special_and_standard():
{"vulnerable_versions": "<=99.999.99999", "patched_versions": "<0.0.0"}, "pkg"
)
assert isinstance(pkg.affected_version_range, NpmVersionRange)
- assert pkg.fixed_version is None
+ assert pkg.fixed_version_range is None
data2 = {"vulnerable_versions": "<=2.0.0", "patched_versions": ">=2.0.1"}
pkg2 = p.get_affected_package(data2, "pkg2")
assert isinstance(pkg2.affected_version_range, NpmVersionRange)
- assert pkg2.fixed_version == SemverVersion("2.0.1")
+ assert str(pkg2.fixed_version_range) == "vers:npm/>=2.0.1"
diff --git a/vulnerabilities/tests/pipelines/test_oss_fuzz_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_oss_fuzz_v2.py
similarity index 100%
rename from vulnerabilities/tests/pipelines/test_oss_fuzz_v2.py
rename to vulnerabilities/tests/pipelines/v2_importers/test_oss_fuzz_v2.py
diff --git a/vulnerabilities/tests/pipelines/test_postgresql_v2_importer.py b/vulnerabilities/tests/pipelines/v2_importers/test_postgresql_importer_v2.py
similarity index 94%
rename from vulnerabilities/tests/pipelines/test_postgresql_v2_importer.py
rename to vulnerabilities/tests/pipelines/v2_importers/test_postgresql_importer_v2.py
index c138c0746..ce3873930 100644
--- a/vulnerabilities/tests/pipelines/test_postgresql_v2_importer.py
+++ b/vulnerabilities/tests/pipelines/v2_importers/test_postgresql_importer_v2.py
@@ -87,20 +87,20 @@ def test_collect_advisories(mock_get, importer):
assert "Description of the issue" in advisory.summary
assert len(advisory.references_v2) > 0
assert advisory.affected_packages[0].package.name == "postgresql"
- assert str(advisory.affected_packages[0].fixed_version) == "10.2"
+ assert str(advisory.affected_packages[0].fixed_version_range) == "vers:generic/10.2.0"
assert advisory.affected_packages[0].affected_version_range.contains(SemverVersion("10.0.0"))
assert advisory.affected_packages[0].affected_version_range.contains(SemverVersion("10.1.0"))
@patch("vulnerabilities.pipelines.v2_importers.postgresql_importer.requests.get")
-def test_collect_advisories_with_no_fixed_version(mock_get, importer):
+def test_collect_advisories_with_no_fixed_version_range(mock_get, importer):
mock_get.return_value.content = HTML_NO_FIX_ADVISORY.encode("utf-8")
advisories = list(importer.collect_advisories())
assert len(advisories) == 1
advisory = advisories[0]
assert advisory.advisory_id == "CVE-2023-5678"
- assert advisory.affected_packages[0].fixed_version is None
+ assert advisory.affected_packages[0].fixed_version_range is None
assert advisory.affected_packages[0].affected_version_range.contains(SemverVersion("9.5"))
assert advisory.affected_packages[0].affected_version_range.contains(SemverVersion("9.6"))
diff --git a/vulnerabilities/tests/pipelines/test_pypa_v2_importer_pipeline.py b/vulnerabilities/tests/pipelines/v2_importers/test_pypa_importer_pipeline_v2.py
similarity index 100%
rename from vulnerabilities/tests/pipelines/test_pypa_v2_importer_pipeline.py
rename to vulnerabilities/tests/pipelines/v2_importers/test_pypa_importer_pipeline_v2.py
diff --git a/vulnerabilities/tests/pipelines/test_pysec_v2_importer.py b/vulnerabilities/tests/pipelines/v2_importers/test_pysec_importer_v2.py
similarity index 100%
rename from vulnerabilities/tests/pipelines/test_pysec_v2_importer.py
rename to vulnerabilities/tests/pipelines/v2_importers/test_pysec_importer_v2.py
diff --git a/vulnerabilities/tests/pipelines/test_vulnrichment_v2_importer.py b/vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py
similarity index 100%
rename from vulnerabilities/tests/pipelines/test_vulnrichment_v2_importer.py
rename to vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py
diff --git a/vulnerabilities/tests/pipelines/test_xen_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_xen_importer_v2.py
similarity index 100%
rename from vulnerabilities/tests/pipelines/test_xen_importer_v2.py
rename to vulnerabilities/tests/pipelines/v2_importers/test_xen_importer_v2.py
diff --git a/vulnerabilities/tests/pipelines/test_collect_commits_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_collect_commits_v2.py
similarity index 84%
rename from vulnerabilities/tests/pipelines/test_collect_commits_v2.py
rename to vulnerabilities/tests/pipelines/v2_improvers/test_collect_commits_v2.py
index dddec9084..d79ca0eb0 100644
--- a/vulnerabilities/tests/pipelines/test_collect_commits_v2.py
+++ b/vulnerabilities/tests/pipelines/v2_improvers/test_collect_commits_v2.py
@@ -1,11 +1,19 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+
from datetime import datetime
-from unittest.mock import patch
import pytest
from vulnerabilities.models import AdvisoryReference
from vulnerabilities.models import AdvisoryV2
from vulnerabilities.models import CodeFixV2
+from vulnerabilities.models import ImpactedPackage
from vulnerabilities.models import PackageV2
from vulnerabilities.pipelines.v2_improvers.collect_commits import CollectFixCommitsPipeline
from vulnerabilities.pipelines.v2_improvers.collect_commits import is_vcs_url
@@ -59,8 +67,8 @@ def test_is_vcs_url_already_processed_true():
name="foo",
version="1.0",
)
- advisory.affecting_packages.add(package)
- advisory.save()
+ impact = ImpactedPackage.objects.create(advisory=advisory)
+ impact.affecting_packages.add(package)
CodeFixV2.objects.create(
commits=["https://github.com/user/repo/commit/abc123"],
advisory=advisory,
@@ -87,9 +95,9 @@ def test_collect_fix_commits_pipeline_creates_entry():
reference = AdvisoryReference.objects.create(
url="https://github.com/test/testpkg/commit/abc123"
)
- advisory.affecting_packages.add(package)
+ impact = ImpactedPackage.objects.create(advisory=advisory)
+ impact.affecting_packages.add(package)
advisory.references.add(reference)
- advisory.save()
pipeline = CollectFixCommitsPipeline()
pipeline.collect_and_store_fix_commits()
@@ -117,13 +125,11 @@ def test_collect_fix_commits_pipeline_skips_non_commit_urls():
name="otherpkg",
version="2.0",
)
-
- advisory.affecting_packages.add(package)
+ impact = ImpactedPackage.objects.create(advisory=advisory)
+ impact.affecting_packages.add(package)
reference = AdvisoryReference.objects.create(url="https://github.com/test/testpkg/issues/12")
-
advisory.references.add(reference)
- advisory.save()
pipeline = CollectFixCommitsPipeline()
pipeline.collect_and_store_fix_commits()
diff --git a/vulnerabilities/tests/pipelines/test_compute_package_risk_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_package_risk_v2.py
similarity index 93%
rename from vulnerabilities/tests/pipelines/test_compute_package_risk_v2.py
rename to vulnerabilities/tests/pipelines/v2_improvers/test_compute_package_risk_v2.py
index 4dbfb222a..db6ffd5d3 100644
--- a/vulnerabilities/tests/pipelines/test_compute_package_risk_v2.py
+++ b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_package_risk_v2.py
@@ -14,6 +14,7 @@
from vulnerabilities.models import AdvisorySeverity
from vulnerabilities.models import AdvisoryV2
from vulnerabilities.models import AdvisoryWeakness
+from vulnerabilities.models import ImpactedPackage
from vulnerabilities.models import PackageV2
from vulnerabilities.pipelines.v2_improvers.compute_package_risk import ComputePackageRiskPipeline
from vulnerabilities.severity_systems import CVSSV3
@@ -54,8 +55,8 @@ def test_simple_risk_pipeline():
weaknesses = AdvisoryWeakness.objects.create(cwe_id=119)
adv.weaknesses.add(weaknesses)
- adv.affecting_packages.add(pkg)
- adv.save()
+ impact = ImpactedPackage.objects.create(advisory=adv)
+ impact.affecting_packages.add(pkg)
improver = ComputePackageRiskPipeline()
improver.execute()
diff --git a/vulnerabilities/tests/pipelines/test_compute_version_rank_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_version_rank_v2.py
similarity index 100%
rename from vulnerabilities/tests/pipelines/test_compute_version_rank_v2.py
rename to vulnerabilities/tests/pipelines/v2_improvers/test_compute_version_rank_v2.py
diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_exploitdb_v2.py
similarity index 94%
rename from vulnerabilities/tests/pipelines/test_enhance_with_exploitdb_v2.py
rename to vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_exploitdb_v2.py
index 865356158..41f96d706 100644
--- a/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb_v2.py
+++ b/vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_exploitdb_v2.py
@@ -20,7 +20,7 @@
from vulnerabilities.pipelines.v2_improvers.enhance_with_exploitdb import ExploitDBImproverPipeline
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-TEST_DATA = os.path.join(BASE_DIR, "../test_data", "exploitdb_improver/files_exploits.csv")
+TEST_DATA = os.path.join(BASE_DIR, "../../test_data", "exploitdb_improver/files_exploits.csv")
@pytest.mark.django_db
diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_kev_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_kev_v2.py
similarity index 96%
rename from vulnerabilities/tests/pipelines/test_enhance_with_kev_v2.py
rename to vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_kev_v2.py
index bd58fa5fd..ab4df9cf2 100644
--- a/vulnerabilities/tests/pipelines/test_enhance_with_kev_v2.py
+++ b/vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_kev_v2.py
@@ -21,7 +21,7 @@
from vulnerabilities.utils import load_json
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-TEST_DATA = os.path.join(BASE_DIR, "../test_data", "kev_data.json")
+TEST_DATA = os.path.join(BASE_DIR, "../../test_data", "kev_data.json")
@pytest.mark.django_db
diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_metasploit_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_metasploit_v2.py
similarity index 94%
rename from vulnerabilities/tests/pipelines/test_enhance_with_metasploit_v2.py
rename to vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_metasploit_v2.py
index c20437145..447dea9d3 100644
--- a/vulnerabilities/tests/pipelines/test_enhance_with_metasploit_v2.py
+++ b/vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_metasploit_v2.py
@@ -23,7 +23,9 @@
from vulnerabilities.utils import load_json
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-TEST_DATA = os.path.join(BASE_DIR, "../test_data", "metasploit_improver/modules_metadata_base.json")
+TEST_DATA = os.path.join(
+ BASE_DIR, "../../test_data", "metasploit_improver/modules_metadata_base.json"
+)
@pytest.mark.django_db
diff --git a/vulnerabilities/tests/pipelines/test_flag_ghost_packages_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_flag_ghost_packages_v2.py
similarity index 100%
rename from vulnerabilities/tests/pipelines/test_flag_ghost_packages_v2.py
rename to vulnerabilities/tests/pipelines/v2_improvers/test_flag_ghost_packages_v2.py
diff --git a/vulnerabilities/tests/pipes/test_vulnerablecode_importer_pipeline_v2.py b/vulnerabilities/tests/pipes/test_vulnerablecode_importer_pipeline_v2.py
new file mode 100644
index 000000000..aa5a2298e
--- /dev/null
+++ b/vulnerabilities/tests/pipes/test_vulnerablecode_importer_pipeline_v2.py
@@ -0,0 +1,95 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+from datetime import datetime
+from datetime import timedelta
+from unittest.mock import patch
+
+import pytest
+from packageurl import PackageURL
+from univers.version_range import VersionRange
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AffectedPackageV2
+from vulnerabilities.models import AdvisoryV2
+from vulnerabilities.models import ImpactedPackage
+from vulnerabilities.models import PackageV2
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+
+
+class DummyImporter(VulnerableCodeBaseImporterPipelineV2):
+ pipeline_id = "dummy"
+ log_messages = []
+
+ def log(self, message, level=logging.INFO):
+ self.log_messages.append((level, message))
+
+ def collect_advisories(self):
+ yield from self._advisories
+
+ def advisories_count(self):
+ return len(self._advisories)
+
+
+@pytest.fixture
+def dummy_advisory():
+ return AdvisoryData(
+ summary="Test advisory",
+ aliases=["CVE-2025-0001"],
+ references_v2=[],
+ severities=[],
+ weaknesses=[],
+ affected_packages=[
+ AffectedPackageV2(
+ package=PackageURL.from_string("pkg:npm/foobar"),
+ affected_version_range=VersionRange.from_string("vers:npm/<=1.2.3"),
+ fixed_version_range=VersionRange.from_string("vers:npm/1.2.4"),
+ ),
+ AffectedPackageV2(
+ package=PackageURL.from_string("pkg:npm/foobar"),
+ affected_version_range=VersionRange.from_string("vers:npm/<=3.2.3"),
+ fixed_version_range=VersionRange.from_string("vers:npm/3.2.4"),
+ ),
+ ],
+ advisory_id="ADV-123",
+ date_published=datetime.now() - timedelta(days=10),
+ url="https://example.com/advisory/1",
+ )
+
+
+@pytest.fixture
+def dummy_importer(dummy_advisory):
+ importer = DummyImporter()
+ importer._advisories = [dummy_advisory]
+ return importer
+
+
+@pytest.mark.django_db
+def test_collect_and_store_advisories(dummy_importer):
+ dummy_importer.collect_and_store_advisories()
+ assert len(dummy_importer.log_messages) >= 2
+ assert "Successfully collected" in dummy_importer.log_messages[-1][1]
+ assert AdvisoryV2.objects.count() == 1
+
+
+@pytest.mark.django_db
+@patch("vulnerabilities.pipes.advisory.get_exact_purls_v2", side_effect=Exception("error"))
+def test_advisory_import_atomicity_no_partial_adv_import(mock_exception, dummy_importer):
+ dummy_importer.collect_and_store_advisories()
+ assert AdvisoryV2.objects.count() == 0
+ assert ImpactedPackage.objects.count() == 0
+
+
+@pytest.mark.django_db
+def test_advisory_import_atomicity(dummy_importer):
+ dummy_importer.collect_and_store_advisories()
+ assert AdvisoryV2.objects.count() == 1
+ assert ImpactedPackage.objects.count() == 2
+ assert PackageV2.objects.count() == 4
diff --git a/vulnerabilities/tests/test_api_v2.py b/vulnerabilities/tests/test_api_v2.py
index 432c7c10f..662499ed9 100644
--- a/vulnerabilities/tests/test_api_v2.py
+++ b/vulnerabilities/tests/test_api_v2.py
@@ -838,6 +838,8 @@ def test_filter_codefix_by_advisory_id_not_found(self):
class AdvisoriesPackageV2Tests(APITestCase):
def setUp(self):
+ from vulnerabilities.models import ImpactedPackage
+
self.advisory = AdvisoryV2.objects.create(
datasource_id="ghsa",
advisory_id="GHSA-1234",
@@ -848,18 +850,14 @@ def setUp(self):
)
self.package = PackageV2.objects.from_purl(purl="pkg:pypi/sample@1.0.0")
+ self.impact = ImpactedPackage.objects.create(advisory=self.advisory)
+ self.impact.affecting_packages.add(self.package)
- self.user = ApiUser.objects.create_api_user(username="e@mail.com")
- self.auth = f"Token {self.user.auth_token.key}"
self.client = APIClient(enforce_csrf_checks=True)
- self.client.credentials(HTTP_AUTHORIZATION=self.auth)
-
- self.package.affected_by_advisories.add(self.advisory)
- self.package.save()
def test_list_with_purl_filter(self):
url = reverse("advisories-package-v2-list")
- with self.assertNumQueries(18):
+ with self.assertNumQueries(17):
response = self.client.get(url, {"purl": "pkg:pypi/sample@1.0.0"})
assert response.status_code == 200
assert "packages" in response.data["results"]
@@ -868,7 +866,7 @@ def test_list_with_purl_filter(self):
def test_bulk_lookup(self):
url = reverse("advisories-package-v2-bulk-lookup")
- with self.assertNumQueries(13):
+ with self.assertNumQueries(16):
response = self.client.post(url, {"purls": ["pkg:pypi/sample@1.0.0"]}, format="json")
assert response.status_code == 200
assert "packages" in response.data
@@ -878,7 +876,7 @@ def test_bulk_lookup(self):
def test_bulk_search_plain(self):
url = reverse("advisories-package-v2-bulk-search")
payload = {"purls": ["pkg:pypi/sample@1.0.0"], "plain_purl": True, "purl_only": False}
- with self.assertNumQueries(13):
+ with self.assertNumQueries(16):
response = self.client.post(url, payload, format="json")
assert response.status_code == 200
assert "packages" in response.data
@@ -887,21 +885,21 @@ def test_bulk_search_plain(self):
def test_bulk_search_purl_only(self):
url = reverse("advisories-package-v2-bulk-search")
payload = {"purls": ["pkg:pypi/sample@1.0.0"], "plain_purl": False, "purl_only": True}
- with self.assertNumQueries(13):
+ with self.assertNumQueries(14):
response = self.client.post(url, payload, format="json")
assert response.status_code == 200
assert "pkg:pypi/sample@1.0.0" in response.data
def test_lookup_single_package(self):
url = reverse("advisories-package-v2-lookup")
- with self.assertNumQueries(11):
+ with self.assertNumQueries(12):
response = self.client.post(url, {"purl": "pkg:pypi/sample@1.0.0"}, format="json")
assert response.status_code == 200
assert any(pkg["purl"] == "pkg:pypi/sample@1.0.0" for pkg in response.data)
def test_get_all_vulnerable_purls(self):
url = reverse("advisories-package-v2-all")
- with self.assertNumQueries(6):
+ with self.assertNumQueries(3):
response = self.client.get(url)
assert response.status_code == 200
assert "pkg:pypi/sample@1.0.0" in response.data
diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py
index 3aec1f56c..b65726a5d 100644
--- a/vulnerabilities/utils.py
+++ b/vulnerabilities/utils.py
@@ -604,7 +604,6 @@ def compute_content_id(advisory_data):
# Normalize fields
from vulnerabilities.importer import AdvisoryData
- from vulnerabilities.importer import AdvisoryDataV2
from vulnerabilities.models import Advisory
if isinstance(advisory_data, Advisory):
diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py
index 07eafbdf0..f4cd99dbe 100644
--- a/vulnerabilities/views.py
+++ b/vulnerabilities/views.py
@@ -34,6 +34,7 @@
from vulnerabilities.forms import PackageSearchForm
from vulnerabilities.forms import PipelineSchedulePackageForm
from vulnerabilities.forms import VulnerabilitySearchForm
+from vulnerabilities.models import ImpactedPackage
from vulnerabilities.models import PipelineRun
from vulnerabilities.models import PipelineSchedule
from vulnerabilities.severity_systems import EPSS
@@ -186,18 +187,49 @@ class PackageV2Details(DetailView):
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
package = self.object
+ next_non_vulnerable, latest_non_vulnerable = package.get_non_vulnerable_versions()
+ fixed_pkg_details = {}
+ for impact in package.affected_in_impacts.all():
+ if impact.advisory.id not in fixed_pkg_details:
+ fixed_pkg_details[impact.advisory.id] = []
+ fixed_pkg_details[impact.advisory.id].extend(
+ [
+ {"pkg": pkg, "affected_count": pkg.affected_in_impacts.count()}
+ for pkg in impact.fixed_by_packages.all()
+ ]
+ )
context["package"] = package
- context["affected_by_advisories"] = package.affected_by_advisories.order_by("advisory_id")
- # Ghost package should not fix any vulnerability.
- context["fixing_advisories"] = (
- None if package.is_ghost else package.fixing_advisories.order_by("advisory_id")
- )
+ context["next_non_vulnerable"] = next_non_vulnerable
+ context["latest_non_vulnerable"] = latest_non_vulnerable
+ context["affected_by_advisories"] = {
+ impact.advisory for impact in package.affected_in_impacts.all()
+ }
+ context["fixing_advisories"] = {
+ impact.advisory for impact in package.fixed_in_impacts.all()
+ }
+
context["package_search_form"] = PackageSearchForm(self.request.GET)
- context["fixed_package_details"] = package.fixed_package_details
+ context["fixed_package_details"] = fixed_pkg_details
# context["history"] = list(package.history)
return context
+ def get_queryset(self):
+ return (
+ super()
+ .get_queryset()
+ .prefetch_related(
+ Prefetch(
+ "affected_in_impacts",
+ queryset=ImpactedPackage.objects.select_related("advisory"),
+ ),
+ Prefetch(
+ "fixed_in_impacts",
+ queryset=ImpactedPackage.objects.select_related("advisory"),
+ ),
+ )
+ )
+
def get_object(self, queryset=None):
if queryset is None:
queryset = self.get_queryset()
@@ -582,37 +614,25 @@ def get_queryset(self):
.get_queryset()
.prefetch_related(
Prefetch(
- "affecting_packages",
- queryset=models.PackageV2.objects.only("type", "namespace", "name", "version"),
- ),
- Prefetch(
- "fixed_by_packages",
- queryset=models.PackageV2.objects.only("type", "namespace", "name", "version"),
- ),
+ "impacted_packages",
+ queryset=models.ImpactedPackage.objects.prefetch_related(
+ Prefetch(
+ "affecting_packages",
+ queryset=models.PackageV2.objects.only(
+ "type", "namespace", "name", "version"
+ ),
+ ),
+ Prefetch(
+ "fixed_by_packages",
+ queryset=models.PackageV2.objects.only(
+ "type", "namespace", "name", "version"
+ ),
+ ),
+ ),
+ )
)
)
- def get_context_data(self, **kwargs):
- """
- Build context with preloaded QuerySets and minimize redundant queries.
- """
- context = super().get_context_data(**kwargs)
- advisory = self.object
- (
- sorted_fixed_by_packages,
- sorted_affected_packages,
- all_affected_fixed_by_matches,
- ) = advisory.aggregate_fixed_and_affected_packages()
- context.update(
- {
- "affected_packages": sorted_affected_packages,
- "fixed_by_packages": sorted_fixed_by_packages,
- "all_affected_fixed_by_matches": all_affected_fixed_by_matches,
- "advisory": advisory,
- }
- )
- return context
-
class PipelineScheduleListView(ListView, FormMixin):
model = PipelineSchedule