diff --git a/vulnerabilities/api_v2.py b/vulnerabilities/api_v2.py index c45dbfebe..ba41d0906 100644 --- a/vulnerabilities/api_v2.py +++ b/vulnerabilities/api_v2.py @@ -31,6 +31,7 @@ from vulnerabilities.models import AdvisoryWeakness from vulnerabilities.models import CodeFix from vulnerabilities.models import CodeFixV2 +from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import Package from vulnerabilities.models import PackageV2 from vulnerabilities.models import PipelineRun @@ -42,6 +43,10 @@ from vulnerabilities.throttling import PermissionBasedUserRateThrottle +class CharInFilter(filters.BaseInFilter, filters.CharFilter): + pass + + class WeaknessV2Serializer(serializers.ModelSerializer): cwe_id = serializers.CharField() name = serializers.CharField() @@ -306,8 +311,8 @@ class AdvisoryPackageV2Serializer(serializers.ModelSerializer): risk_score = serializers.FloatField(read_only=True) affected_by_vulnerabilities = serializers.SerializerMethodField() fixing_vulnerabilities = serializers.SerializerMethodField() - next_non_vulnerable_version = serializers.CharField(read_only=True) - latest_non_vulnerable_version = serializers.CharField(read_only=True) + next_non_vulnerable_version = serializers.SerializerMethodField() + latest_non_vulnerable_version = serializers.SerializerMethodField() class Meta: model = Package @@ -320,36 +325,36 @@ class Meta: "risk_score", ] - def get_affected_by_vulnerabilities(self, obj): - """ - Return a dictionary with vulnerabilities as keys and their details, including fixed_by_packages. - """ + def get_affected_by_vulnerabilities(self, package): + """Return a dictionary with advisory as keys and their details, including fixed_by_packages.""" result = {} request = self.context.get("request") - for adv in getattr(obj, "prefetched_affected_advisories", []): - fixed_by_package = adv.fixed_by_packages.first() - purl = None - if fixed_by_package: - purl = fixed_by_package.package_url - # Get code fixed for a vulnerability - code_fixes = CodeFixV2.objects.filter(advisory=adv).distinct() + for impact in package.affected_in_impacts.all(): + advisory = impact.advisory + fixed_by_packages = [pkg.purl for pkg in impact.fixed_by_packages.all()] + code_fixes = CodeFixV2.objects.filter(advisory=advisory).distinct() code_fix_urls = [ reverse("advisory-codefix-detail", args=[code_fix.id], request=request) for code_fix in code_fixes ] - - result[adv.avid] = { - "advisory_id": adv.avid, - "fixed_by_packages": purl, + result[advisory.avid] = { + "advisory_id": advisory.avid, + "fixed_by_packages": fixed_by_packages, "code_fixes": code_fix_urls, } + return result - def get_fixing_vulnerabilities(self, obj): - # Ghost package should not fix any vulnerability. - if obj.is_ghost: - return [] - return [adv.avid for adv in obj.fixing_advisories.all()] + def get_fixing_vulnerabilities(self, package): + return [impact.advisory.avid for impact in package.fixed_in_impacts.all()] + + def get_next_non_vulnerable_version(self, package): + if next_non_vulnerable := package.get_non_vulnerable_versions()[0]: + return next_non_vulnerable.version + + def get_latest_non_vulnerable_version(self, package): + if latest_non_vulnerable := package.get_non_vulnerable_versions()[-1]: + return latest_non_vulnerable.version class PackageurlListSerializer(serializers.Serializer): @@ -381,9 +386,24 @@ class PackageV2FilterSet(filters.FilterSet): class AdvisoryPackageV2FilterSet(filters.FilterSet): - affected_by_vulnerability = filters.CharFilter(field_name="affected_by_advisory__advisory_id") - fixing_vulnerability = filters.CharFilter(field_name="fixing_advisories__advisory_id") - purl = filters.CharFilter(field_name="package_url") + affected_by_advisory = filters.CharFilter( + field_name="affected_in_impacts__advisory__avid", + label="Affected By Advisory ID", + help_text="Filter packages affected by a specific Advisory ID.", + ) + + fixing_advisory = filters.CharFilter( + field_name="fixed_in_impacts__advisory__avid", + label="Fixed By Advisory ID", + help_text="Filter packages fixed by a specific Advisory ID.", + ) + + purls = CharInFilter( + field_name="package_url", + lookup_expr="in", + label="Package URL", + help_text="Filter by one or more Package URLs. Multi-value supported (comma-separated).", + ) class PackageV2ViewSet(viewsets.ReadOnlyModelViewSet): @@ -968,40 +988,39 @@ def get_view_name(self): class AdvisoriesPackageV2ViewSet(viewsets.ReadOnlyModelViewSet): - queryset = PackageV2.objects.all().prefetch_related( - Prefetch( - "affected_by_advisories", - queryset=AdvisoryV2.objects.prefetch_related("fixed_by_packages"), - to_attr="prefetched_affected_advisories", - ) - ) + queryset = PackageV2.objects.all() serializer_class = AdvisoryPackageV2Serializer - filter_backends = (filters.DjangoFilterBackend,) + filter_backends = [filters.DjangoFilterBackend] filterset_class = AdvisoryPackageV2FilterSet def get_queryset(self): - queryset = super().get_queryset() - package_purls = self.request.query_params.getlist("purl") - affected_by_advisory = self.request.query_params.get("affected_by_advisory") - fixing_advisory = self.request.query_params.get("fixing_advisory") - if package_purls: - queryset = queryset.filter(package_url__in=package_purls) - if affected_by_advisory: - queryset = queryset.filter(affected_by_advisories__advisory_id=affected_by_advisory) - if fixing_advisory: - queryset = queryset.filter(fixing_advisories__advisory=fixing_advisory) - return queryset.with_is_vulnerable() + return ( + super() + .get_queryset() + .prefetch_related( + Prefetch( + "affected_in_impacts", + queryset=ImpactedPackage.objects.select_related("advisory").prefetch_related( + "fixed_by_packages", + ), + ), + Prefetch( + "fixed_in_impacts", + queryset=ImpactedPackage.objects.select_related("advisory"), + ), + ) + .with_is_vulnerable() + ) def list(self, request, *args, **kwargs): - queryset = self.get_queryset() - # Apply pagination - page = self.paginate_queryset(queryset) + filtered_queryset = self.filter_queryset(self.get_queryset()) + page = self.paginate_queryset(filtered_queryset) + + advisories = set() if page is not None: - # Collect only vulnerabilities for packages in the current page - advisories = set() for package in page: - advisories.update(package.affected_by_advisories.all()) - advisories.update(package.fixing_advisories.all()) + advisories.update({impact.advisory for impact in package.affected_in_impacts.all()}) + advisories.update({impact.advisory for impact in package.fixed_in_impacts.all()}) # Serialize the vulnerabilities with advisory_id and advisory label as keys advisory_data = {f"{adv.avid}": AdvisoryV2Serializer(adv).data for adv in advisories} @@ -1009,15 +1028,14 @@ def list(self, request, *args, **kwargs): # Serialize the current page of packages serializer = self.get_serializer(page, many=True) data = serializer.data - print(data) + # Use 'self.get_paginated_response' to include pagination data return self.get_paginated_response({"advisories": advisory_data, "packages": data}) # If pagination is not applied, collect vulnerabilities for all packages - advisories = set() for package in queryset: - advisories.update(package.affected_by_vulnerabilities.all()) - advisories.update(package.fixing_vulnerabilities.all()) + advisories.update({impact.advisory for impact in package.affected_in_impacts.all()}) + advisories.update({impact.advisory for impact in package.fixed_in_impacts.all()}) advisory_data = {f"{adv.avid}": AdvisoryV2Serializer(adv).data for adv in advisories} @@ -1053,13 +1071,28 @@ def bulk_lookup(self, request): purls = validated_data.get("purls") # Fetch packages matching the provided purls - packages = PackageV2.objects.for_purls(purls).with_is_vulnerable() + packages = ( + PackageV2.objects.for_purls(purls) + .prefetch_related( + Prefetch( + "affected_in_impacts", + queryset=ImpactedPackage.objects.select_related("advisory").prefetch_related( + "fixed_by_packages", + ), + ), + Prefetch( + "fixed_in_impacts", + queryset=ImpactedPackage.objects.select_related("advisory"), + ), + ) + .with_is_vulnerable() + ) # Collect vulnerabilities associated with these packages advisories = set() for package in packages: - advisories.update(package.affected_by_advisories.all()) - advisories.update(package.fixing_advisories.all()) + advisories.update({impact.advisory for impact in package.affected_in_impacts.all()}) + advisories.update({impact.advisory for impact in package.fixed_in_impacts.all()}) # Serialize vulnerabilities with vulnerability_id as keys advisory_data = {adv.avid: AdvisoryV2Serializer(adv).data for adv in advisories} @@ -1124,6 +1157,20 @@ def bulk_search(self, request): PackageV2.objects.filter(plain_package_url__in=plain_purls) .order_by("plain_package_url") .distinct("plain_package_url") + .prefetch_related( + Prefetch( + "affected_in_impacts", + queryset=ImpactedPackage.objects.select_related( + "advisory" + ).prefetch_related( + "fixed_by_packages", + ), + ), + Prefetch( + "fixed_in_impacts", + queryset=ImpactedPackage.objects.select_related("advisory"), + ), + ) .with_is_vulnerable() ) @@ -1132,14 +1179,16 @@ def bulk_search(self, request): # Collect vulnerabilities associated with these packages advisories = set() for package in packages: - advisories.update(package.affected_by_advisories.all()) - advisories.update(package.fixing_advisories.all()) + advisories.update({impact.advisory for impact in package.affected_in_impacts.all()}) + advisories.update({impact.advisory for impact in package.fixed_in_impacts.all()}) advisory_data = {adv.avid: AdvisoryV2Serializer(adv).data for adv in advisories} if not purl_only: package_data = AdvisoryPackageV2Serializer( - packages, many=True, context={"request": request} + packages, + many=True, + context={"request": request}, ).data return Response( { @@ -1154,20 +1203,39 @@ def bulk_search(self, request): vulnerable_purls = [str(package.plain_package_url) for package in vulnerable_purls] return Response(data=vulnerable_purls) - query = PackageV2.objects.filter(package_url__in=purls).distinct().with_is_vulnerable() + query = ( + PackageV2.objects.filter(package_url__in=purls) + .order_by("plain_package_url") + .distinct("plain_package_url") + .prefetch_related( + Prefetch( + "affected_in_impacts", + queryset=ImpactedPackage.objects.select_related("advisory").prefetch_related( + "fixed_by_packages", + ), + ), + Prefetch( + "fixed_in_impacts", + queryset=ImpactedPackage.objects.select_related("advisory"), + ), + ) + .with_is_vulnerable() + ) packages = query # Collect vulnerabilities associated with these packages advisories = set() for package in packages: - advisories.update(package.affected_by_advisories.all()) - advisories.update(package.fixing_advisories.all()) + advisories.update({impact.advisory for impact in package.affected_in_impacts.all()}) + advisories.update({impact.advisory for impact in package.fixed_in_impacts.all()}) advisory_data = {adv.advisory_id: AdvisoryV2Serializer(adv).data for adv in advisories} if not purl_only: package_data = AdvisoryPackageV2Serializer( - packages, many=True, context={"request": request} + packages, + many=True, + context={"request": request}, ).data return Response( { diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index 52f9d62dd..da7f743da 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -11,11 +11,8 @@ import datetime import functools import logging -import os -import shutil import traceback import xml.etree.ElementTree as ET -from pathlib import Path from typing import Iterable from typing import List from typing import Mapping @@ -339,6 +336,88 @@ def from_dict(cls, affected_pkg: dict): ) +@functools.total_ordering +@dataclasses.dataclass(eq=True) +class AffectedPackageV2: + """ + Relate a Package URL with a range of affected versions and fixed versions. + The Package URL must *not* have a version. + AffectedPackage must contain either ``affected_version_range`` or ``fixed_version_range``. + """ + + package: PackageURL + affected_version_range: Optional[VersionRange] = None + fixed_version_range: Optional[VersionRange] = None + + def __post_init__(self): + if self.package.version: + raise ValueError(f"Affected Package URL {self.package!r} cannot have a version.") + + if not (self.affected_version_range or self.fixed_version_range): + raise ValueError( + f"Affected Package {self.package!r} should have either fixed version range or an " + "affected version range." + ) + + def __lt__(self, other): + if not isinstance(other, AffectedPackageV2): + return NotImplemented + return self._cmp_key() < other._cmp_key() + + # TODO: Add cache + def _cmp_key(self): + return ( + str(self.package), + str(self.affected_version_range or ""), + str(self.fixed_version_range or ""), + ) + + def to_dict(self): + """Return a serializable dict that can be converted back using self.from_dict""" + + affected_version_range = ( + str(self.affected_version_range) if self.affected_version_range else None + ) + fixed_version_range = str(self.fixed_version_range) if self.fixed_version_range else None + return { + "package": purl_to_dict(self.package), + "affected_version_range": affected_version_range, + "fixed_version_range": fixed_version_range, + } + + @classmethod + def from_dict(cls, affected_pkg: dict): + """Return an AffectedPackage object from dict generated by self.to_dict""" + + package = PackageURL(**affected_pkg["package"]) + affected_version_range = None + fixed_version_range = None + affected_range = affected_pkg["affected_version_range"] + fixed_range = affected_pkg["fixed_version_range"] + + try: + affected_version_range = VersionRange.from_string(affected_range) + fixed_version_range = VersionRange.from_string(fixed_range) + except: + tb = traceback.format_exc() + logger.error( + f"Cannot create AffectedPackage with invalid or unknown range: {affected_pkg!r} with error: {tb!r}" + ) + return + + if not fixed_version_range and not affected_version_range: + logger.error( + f"Cannot create AffectedPackage without fixed or affected range: {affected_pkg!r}" + ) + return + + return cls( + package=package, + affected_version_range=affected_version_range, + fixed_version_range=fixed_version_range, + ) + + @dataclasses.dataclass(order=True) class AdvisoryData: """ @@ -355,7 +434,9 @@ class AdvisoryData: advisory_id: str = "" aliases: List[str] = dataclasses.field(default_factory=list) summary: Optional[str] = "" - affected_packages: List[AffectedPackage] = dataclasses.field(default_factory=list) + affected_packages: Union[List[AffectedPackage], List[AffectedPackageV2]] = dataclasses.field( + default_factory=list + ) references: List[Reference] = dataclasses.field(default_factory=list) references_v2: List[ReferenceV2] = dataclasses.field(default_factory=list) date_published: Optional[datetime.datetime] = None @@ -365,8 +446,6 @@ class AdvisoryData: original_advisory_text: Optional[str] = None def __post_init__(self): - if self.date_published and not self.date_published.tzinfo: - logger.warning(f"AdvisoryData with no tzinfo: {self!r}") if self.summary: self.summary = self.clean_summary(self.summary) @@ -392,13 +471,19 @@ def to_dict(self): @classmethod def from_dict(cls, advisory_data): date_published = advisory_data["date_published"] + affected_packages = advisory_data["affected_packages"] + affected_package_cls = AffectedPackage + if affected_packages: + affected_package_cls = ( + AffectedPackageV2 + if "fixed_version_range" in affected_packages[0] + else AffectedPackage + ) transformed = { "aliases": advisory_data["aliases"], "summary": advisory_data["summary"], "affected_packages": [ - AffectedPackage.from_dict(pkg) - for pkg in advisory_data["affected_packages"] - if pkg is not None + affected_package_cls.from_dict(pkg) for pkg in affected_packages if pkg is not None ], "references": [Reference.from_dict(ref) for ref in advisory_data["references"]], "date_published": datetime.datetime.fromisoformat(date_published) diff --git a/vulnerabilities/importers/osv.py b/vulnerabilities/importers/osv.py index be27492d7..8bd4ad274 100644 --- a/vulnerabilities/importers/osv.py +++ b/vulnerabilities/importers/osv.py @@ -23,6 +23,7 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import Reference from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity @@ -144,19 +145,27 @@ def parse_advisory_data_v2( supported_ecosystem=purl.type, ) + fixed_versions = [] + fixed_version_range = None for fixed_range in affected_pkg.get("ranges") or []: fixed_version = get_fixed_versions( fixed_range=fixed_range, raw_id=advisory_id, supported_ecosystem=purl.type ) + fixed_versions.extend([v.string for v in fixed_version]) - for version in fixed_version: - affected_packages.append( - AffectedPackage( - package=purl, - affected_version_range=affected_version_range, - fixed_version=version, - ) + fixed_version_range = ( + get_fixed_version_range(fixed_versions, purl.type) if fixed_versions else None + ) + + if fixed_version_range or affected_version_range: + affected_packages.append( + AffectedPackageV2( + package=purl, + affected_version_range=affected_version_range, + fixed_version_range=fixed_version_range, ) + ) + database_specific = raw_data.get("database_specific") or {} cwe_ids = database_specific.get("cwe_ids") or [] weaknesses = list(map(get_cwe_id, cwe_ids)) @@ -334,6 +343,13 @@ def get_affected_version_range(affected_pkg, raw_id, supported_ecosystem): ) +def get_fixed_version_range(versions, ecosystem): + try: + return RANGE_CLASS_BY_SCHEMES[ecosystem].from_versions(versions) + except Exception as e: + logger.error(f"Failed to create VersionRange from: {versions}: error:{e!r}") + + def get_fixed_versions(fixed_range, raw_id, supported_ecosystem) -> List[Version]: """ Return a list of unique fixed univers Versions given a ``fixed_range`` diff --git a/vulnerabilities/migrations/0100_remove_advisoryv2_affecting_packages_and_more.py b/vulnerabilities/migrations/0100_remove_advisoryv2_affecting_packages_and_more.py new file mode 100644 index 000000000..9a00e4731 --- /dev/null +++ b/vulnerabilities/migrations/0100_remove_advisoryv2_affecting_packages_and_more.py @@ -0,0 +1,87 @@ +# Generated by Django 4.2.22 on 2025-07-28 18:30 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0099_advisoryv2_original_advisory_text"), + ] + + operations = [ + migrations.RemoveField( + model_name="advisoryv2", + name="affecting_packages", + ), + migrations.RemoveField( + model_name="advisoryv2", + name="fixed_by_packages", + ), + migrations.CreateModel( + name="ImpactedPackage", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "base_purl", + models.CharField( + blank=True, + help_text="Version less PURL related to impacted range.", + max_length=500, + ), + ), + ( + "affecting_vers", + models.TextField( + blank=True, + help_text="VersionRange expression for package vulnerable to this impact.", + ), + ), + ( + "fixed_vers", + models.TextField( + blank=True, + help_text="VersionRange expression for packages fixing the vulnerable package in this impact.", + ), + ), + ( + "created_at", + models.DateTimeField( + auto_now_add=True, + db_index=True, + help_text="Timestamp indicating when this impact was added.", + ), + ), + ( + "advisory", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="impacted_packages", + to="vulnerabilities.advisoryv2", + ), + ), + ( + "affecting_packages", + models.ManyToManyField( + help_text="Packages vulnerable to this impact.", + related_name="affected_in_impacts", + to="vulnerabilities.packagev2", + ), + ), + ( + "fixed_by_packages", + models.ManyToManyField( + help_text="Packages vulnerable to this impact.", + related_name="fixed_in_impacts", + to="vulnerabilities.packagev2", + ), + ), + ], + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index c4a302536..d51b2fce6 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2532,6 +2532,20 @@ class Meta: verbose_name_plural = "Advisory severities" ordering = ["url", "scoring_system", "value"] + def to_dict(self): + return { + "system": self.scoring_system, + "value": self.value, + "scoring_elements": self.scoring_elements, + "published_at": self.published_at, + "url": self.url, + } + + def to_vulnerability_severity_data(self): + from vulnerabilities.importer import VulnerabilitySeverity + + return VulnerabilitySeverity.from_dict(self.to_dict()) + class AdvisoryWeakness(models.Model): """ @@ -2625,6 +2639,18 @@ def is_cpe(self): """ return self.reference_id.startswith("cpe") + def to_dict(self): + return { + "reference_id": self.reference_id, + "reference_type": self.reference_type, + "url": self.url, + } + + def to_reference_v2_data(self): + from vulnerabilities.importer import ReferenceV2 + + return ReferenceV2.from_dict(self.to_dict()) + class AdvisoryAlias(models.Model): alias = models.CharField( @@ -2750,18 +2776,6 @@ class AdvisoryV2(models.Model): help_text="Raw advisory data as collected from the upstream datasource.", ) - affecting_packages = models.ManyToManyField( - "PackageV2", - related_name="affected_by_advisories", - help_text="A list of packages that are affected by this advisory.", - ) - - fixed_by_packages = models.ManyToManyField( - "PackageV2", - related_name="fixing_advisories", - help_text="A list of packages that are reported by this advisory.", - ) - status = models.IntegerField( choices=AdvisoryStatusType.choices, default=AdvisoryStatusType.PUBLISHED ) @@ -2816,21 +2830,19 @@ def get_absolute_url(self): """ return reverse("advisory_details", args=[self.avid]) - def to_advisory_data(self) -> "AdvisoryDataV2": - from vulnerabilities.importer import AdvisoryDataV2 - from vulnerabilities.importer import AffectedPackage - from vulnerabilities.importer import ReferenceV2 + def to_advisory_data(self) -> "AdvisoryData": + from vulnerabilities.importer import AdvisoryData - return AdvisoryDataV2( + return AdvisoryData( aliases=[item.alias for item in self.aliases.all()], summary=self.summary, affected_packages=[ - AffectedPackage.from_dict(pkg) for pkg in self.affected_packages if pkg + impacted.to_affected_package_data() for impacted in self.impacted_packages.all() ], - references=[ReferenceV2.from_dict(ref) for ref in self.references], + references_v2=[ref.to_reference_v2_data() for ref in self.references.all()], date_published=self.date_published, - weaknesses=self.weaknesses, - severities=self.severities, + weaknesses=[weak.cwe_id for weak in self.weaknesses.all()], + severities=[sev.to_vulnerability_severity_data() for sev in self.severities.all()], url=self.url, ) @@ -2841,66 +2853,69 @@ def get_aliases(self): """ return self.aliases.all() - def aggregate_fixed_and_affected_packages(self): - from vulnerabilities.utils import get_purl_version_class + alias = get_aliases - sorted_fixed_by_packages = self.fixed_by_packages.filter(is_ghost=False).order_by( - "type", "namespace", "name", "qualifiers", "subpath" - ) - if sorted_fixed_by_packages: - sorted_fixed_by_packages.first().calculate_version_rank +class ImpactedPackage(models.Model): + """ + Represents a single impact for an advisory, including affected range and fixed version and + associated package relations. + """ - sorted_affected_packages = self.affecting_packages.all() + advisory = models.ForeignKey( + AdvisoryV2, + related_name="impacted_packages", + on_delete=models.CASCADE, + ) - if sorted_affected_packages: - sorted_affected_packages.first().calculate_version_rank + base_purl = models.CharField( + max_length=500, + blank=True, + help_text="Version less PURL related to impacted range.", + ) - grouped_fixed_by_packages = { - key: list(group) - for key, group in groupby( - sorted_fixed_by_packages, - key=attrgetter("type", "namespace", "name", "qualifiers", "subpath"), - ) - } + affecting_vers = models.TextField( + blank=True, + help_text="VersionRange expression for package vulnerable to this impact.", + ) - all_affected_fixed_by_matches = [] + fixed_vers = models.TextField( + blank=True, + help_text="VersionRange expression for packages fixing the vulnerable package in this impact.", + ) - for sorted_affected_package in sorted_affected_packages: - affected_fixed_by_matches = { - "affected_package": sorted_affected_package, - "matched_fixed_by_packages": [], - } + affecting_packages = models.ManyToManyField( + "PackageV2", + related_name="affected_in_impacts", + help_text="Packages vulnerable to this impact.", + ) - # Build the key to find matching group - key = ( - sorted_affected_package.type, - sorted_affected_package.namespace, - sorted_affected_package.name, - sorted_affected_package.qualifiers, - sorted_affected_package.subpath, - ) + fixed_by_packages = models.ManyToManyField( + "PackageV2", + related_name="fixed_in_impacts", + help_text="Packages vulnerable to this impact.", + ) - # Get matching group from pre-grouped fixed_by_packages - matching_fixed_packages = grouped_fixed_by_packages.get(key, []) + created_at = models.DateTimeField( + auto_now_add=True, + db_index=True, + help_text="Timestamp indicating when this impact was added.", + ) - # Get version classes for comparison - affected_version_class = get_purl_version_class(sorted_affected_package) - affected_version = affected_version_class(sorted_affected_package.version) + def to_dict(self): + from vulnerabilities.utils import purl_to_dict - # Compare versions and filter valid matches - matched_fixed_by_packages = [ - fixed_by_package.purl - for fixed_by_package in matching_fixed_packages - if get_purl_version_class(fixed_by_package)(fixed_by_package.version) - > affected_version - ] + return { + "package": purl_to_dict(self.base_purl), + "affected_version_range": self.affecting_vers, + "fixed_version_range": self.fixed_vers, + } - affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages - all_affected_fixed_by_matches.append(affected_fixed_by_matches) - return sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches + def to_affected_package_data(self): + """Return `AffectedPackageV2` data from the impact.""" + from vulnerabilities.importer import AffectedPackageV2 - alias = get_aliases + return AffectedPackageV2.from_dict(self.to_dict()) class ToDoRelatedAdvisory(models.Model): @@ -2942,10 +2957,12 @@ def search(self, query: str = None): def with_vulnerability_counts(self): return self.annotate( vulnerability_count=Count( - "affected_by_advisories", + "affected_in_impacts__advisory", + distinct=True, ), patched_vulnerability_count=Count( - "fixing_advisories", + "fixed_in_impacts__advisory", + distinct=True, ), ) @@ -2963,7 +2980,7 @@ def get_fixed_by_package_versions(self, purl: PackageURL, fix=True): } if fix: - filter_dict["fixing_advisories__isnull"] = False + filter_dict["fixed_in_impacts__isnull"] = False # TODO: why do we need distinct return PackageV2.objects.filter(**filter_dict).distinct() @@ -2996,7 +3013,7 @@ def for_purls(self, purls=()): def _vulnerable(self, vulnerable=True): """ - Filter to select only vulnerable or non-vulnearble packages. + Filter to select only vulnerable or non-vulnerable packages. """ return self.with_is_vulnerable().filter(is_vulnerable=vulnerable) @@ -3004,14 +3021,16 @@ def vulnerable(self): """ Return only packages that are vulnerable. """ - return self.filter(affected_by_advisories__isnull=False) + return self.filter(affected_in_impacts__isnull=False) def with_is_vulnerable(self): """ Annotate Package with ``is_vulnerable`` boolean attribute. """ return self.annotate( - is_vulnerable=Exists(AdvisoryV2.objects.filter(affecting_packages__pk=OuterRef("pk"))) + is_vulnerable=Exists( + ImpactedPackage.objects.filter(affecting_packages__pk=OuterRef("pk")) + ) ) def from_purl(self, purl: Union[PackageURL, str]): @@ -3118,23 +3137,6 @@ def calculate_version_rank(self): PackageV2.objects.bulk_update(sorted_packages, fields=["version_rank"]) return self.version_rank - @property - def fixed_package_details(self): - """ - Return a mapping of vulnerabilities that affect this package and the next and - latest non-vulnerable versions. - """ - package_details = {} - package_details["purl"] = PackageURL.from_string(self.purl) - - next_non_vulnerable, latest_non_vulnerable = self.get_non_vulnerable_versions() - package_details["next_non_vulnerable"] = next_non_vulnerable - package_details["latest_non_vulnerable"] = latest_non_vulnerable - - package_details["advisories"] = self.get_affecting_vulnerabilities() - - return package_details - def get_non_vulnerable_versions(self): """ Return a tuple of the next and latest non-vulnerable versions as Package instance. @@ -3146,17 +3148,12 @@ def get_non_vulnerable_versions(self): self, fix=False ).only_non_vulnerable() - later_non_vulnerable_versions = non_vulnerable_versions.filter( - version_rank__gt=self.version_rank - ) - - later_non_vulnerable_versions = list(later_non_vulnerable_versions) + later_non_vulnerable = non_vulnerable_versions.filter( + version_rank__gte=self.version_rank + ).order_by("version_rank") - if later_non_vulnerable_versions: - sorted_versions = later_non_vulnerable_versions - next_non_vulnerable = sorted_versions[0] - latest_non_vulnerable = sorted_versions[-1] - return next_non_vulnerable, latest_non_vulnerable + if later_non_vulnerable.exists(): + return later_non_vulnerable.first(), later_non_vulnerable.last() return None, None @@ -3175,65 +3172,6 @@ def get_absolute_url(self): def current_version(self): return self.version_class(self.version) - def get_affecting_vulnerabilities(self): - """ - Return a list of vulnerabilities that affect this package together with information regarding - the versions that fix the vulnerabilities. - """ - if self.version_rank == 0: - self.calculate_version_rank - package_details_advs = [] - - fixed_by_packages = PackageV2.objects.get_fixed_by_package_versions(self, fix=True) - - package_advisories = self.affected_by_advisories.prefetch_related( - Prefetch( - "fixed_by_packages", - queryset=fixed_by_packages, - to_attr="fixed_packages", - ) - ) - - for adv in package_advisories: - package_details_advs.append({"advisory": adv}) - later_fixed_packages = [] - - for fixed_pkg in adv.fixed_by_packages.all(): - if fixed_pkg not in fixed_by_packages: - continue - fixed_version = self.version_class(fixed_pkg.version) - if fixed_version > self.current_version: - later_fixed_packages.append(fixed_pkg) - - next_fixed_package_vulns = [] - - sort_fixed_by_packages_by_version = [] - if later_fixed_packages: - sort_fixed_by_packages_by_version = sorted( - later_fixed_packages, key=lambda p: p.version_rank - ) - - fixed_by_pkgs = [] - - for vuln_details in package_details_advs: - if vuln_details["advisory"] != adv: - continue - vuln_details["fixed_by_purl"] = [] - vuln_details["fixed_by_purl_advisories"] = [] - - for fixed_by_pkg in sort_fixed_by_packages_by_version: - fixed_by_package_details = {} - fixed_by_purl = PackageURL.from_string(fixed_by_pkg.purl) - next_fixed_package_vulns = list(fixed_by_pkg.affected_by_advisories.all()) - - fixed_by_package_details["fixed_by_purl"] = fixed_by_purl - fixed_by_package_details["fixed_by_purl_advisories"] = next_fixed_package_vulns - fixed_by_pkgs.append(fixed_by_package_details) - - vuln_details["fixed_by_package_details"] = fixed_by_pkgs - - return package_details_advs - class AdvisoryExploit(models.Model): """ diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py index 6232a294d..c642748b0 100644 --- a/vulnerabilities/pipelines/__init__.py +++ b/vulnerabilities/pipelines/__init__.py @@ -15,29 +15,18 @@ from traceback import format_exc as traceback_format_exc from typing import Iterable from typing import List -from typing import Optional from aboutcode.pipeline import LoopProgress from aboutcode.pipeline import PipelineDefinition from aboutcode.pipeline import humanize_time -from fetchcode import package_versions -from packageurl import PackageURL from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import UnMergeablePackageError from vulnerabilities.improver import MAX_CONFIDENCE from vulnerabilities.models import Advisory -from vulnerabilities.models import PackageV2 from vulnerabilities.models import PipelineRun from vulnerabilities.pipes.advisory import import_advisory from vulnerabilities.pipes.advisory import insert_advisory from vulnerabilities.pipes.advisory import insert_advisory_v2 -from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage -from vulnerabilities.utils import classproperty -from vulnerabilities.utils import get_affected_packages_by_patched_package -from vulnerabilities.utils import nearest_patched_package -from vulnerabilities.utils import resolve_version_range module_logger = logging.getLogger(__name__) @@ -271,13 +260,15 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline): license_url = None spdx_license_expression = None repo_url = None - advisory_confidence = MAX_CONFIDENCE ignorable_versions = [] - unfurl_version_ranges = False @classmethod def steps(cls): - return (cls.collect_and_store_advisories,) + return ( + # Add step for downloading/cloning resource as required. + cls.collect_and_store_advisories, + # Add step for removing downloaded/cloned resource as required. + ) def collect_advisories(self) -> Iterable[AdvisoryData]: """ @@ -311,7 +302,6 @@ def collect_and_store_advisories(self): if _obj := insert_advisory_v2( advisory=advisory, pipeline_id=self.pipeline_id, - get_advisory_packages=self.get_advisory_packages, logger=self.log, ): collected_advisory_count += 1 @@ -323,203 +313,3 @@ def collect_and_store_advisories(self): continue self.log(f"Successfully collected {collected_advisory_count:,d} advisories") - - def get_advisory_packages(self, advisory_data: AdvisoryData) -> list: - """ - Return the list of packages for the given advisory. - - Used by ``import_advisory`` to get the list of packages for the advisory. - """ - from vulnerabilities.improvers import default - - affected_purls = [] - fixed_purls = [] - for affected_package in advisory_data.affected_packages: - package_affected_purls, package_fixed_purls = default.get_exact_purls( - affected_package=affected_package - ) - affected_purls.extend(package_affected_purls) - fixed_purls.extend(package_fixed_purls) - - if self.unfurl_version_ranges: - vulnerable_pvs, fixed_pvs = self.get_impacted_packages( - affected_packages=advisory_data.affected_packages, - advisory_date_published=advisory_data.date_published, - ) - affected_purls.extend(vulnerable_pvs) - fixed_purls.extend(fixed_pvs) - - vulnerable_packages = [] - fixed_packages = [] - - for affected_purl in affected_purls: - vulnerable_package, _ = PackageV2.objects.get_or_create_from_purl(purl=affected_purl) - vulnerable_packages.append(vulnerable_package) - - for fixed_purl in fixed_purls: - fixed_package, _ = PackageV2.objects.get_or_create_from_purl(purl=fixed_purl) - fixed_packages.append(fixed_package) - - return vulnerable_packages, fixed_packages - - def get_published_package_versions( - self, package_url: PackageURL, until: Optional[datetime] = None - ) -> List[str]: - """ - Return a list of versions published before `until` for the `package_url` - """ - versions_before_until = [] - try: - versions = package_versions.versions(str(package_url)) - for version in versions or []: - if ( - version.release_date - and version.release_date.tzinfo - and until - and until.tzinfo is None - ): - until = until.replace(tzinfo=timezone.utc) - if until and version.release_date and version.release_date > until: - continue - versions_before_until.append(version.value) - - return versions_before_until - except Exception as e: - self.log( - f"Failed to fetch versions for package {str(package_url)} {e!r}", - level=logging.ERROR, - ) - return [] - - def get_impacted_packages(self, affected_packages, advisory_date_published): - """ - Return a tuple of lists of affected and fixed PackageURLs - """ - if not affected_packages: - return [], [] - - mergable = True - - # TODO: We should never had the exception in first place - try: - purl, affected_version_ranges, fixed_versions = AffectedPackage.merge(affected_packages) - except UnMergeablePackageError: - self.log(f"Cannot merge with different purls {affected_packages!r}", logging.ERROR) - mergable = False - - if not mergable: - vulnerable_packages = [] - fixed_packages = [] - for affected_package in affected_packages: - purl = affected_package.package - affected_version_range = affected_package.affected_version_range - fixed_version = affected_package.fixed_version - pkg_type = purl.type - pkg_namespace = purl.namespace - pkg_name = purl.name - if not affected_version_range and fixed_version: - fixed_packages.append( - PackageURL( - type=pkg_type, - namespace=pkg_namespace, - name=pkg_name, - version=str(fixed_version), - ) - ) - else: - valid_versions = self.get_published_package_versions( - package_url=purl, until=advisory_date_published - ) - affected_pvs, fixed_pvs = self.resolve_package_versions( - affected_version_range=affected_version_range, - pkg_type=pkg_type, - pkg_namespace=pkg_namespace, - pkg_name=pkg_name, - valid_versions=valid_versions, - ) - vulnerable_packages.extend(affected_pvs) - fixed_packages.extend(fixed_pvs) - return vulnerable_packages, fixed_packages - else: - pkg_type = purl.type - pkg_namespace = purl.namespace - pkg_name = purl.name - pkg_qualifiers = purl.qualifiers - fixed_purls = [ - PackageURL( - type=pkg_type, - namespace=pkg_namespace, - name=pkg_name, - version=str(version), - qualifiers=pkg_qualifiers, - ) - for version in fixed_versions - ] - if not affected_version_ranges: - return [], fixed_purls - else: - valid_versions = self.get_published_package_versions( - package_url=purl, until=advisory_date_published - ) - vulnerable_packages = [] - fixed_packages = [] - for affected_version_range in affected_version_ranges: - vulnerable_pvs, fixed_pvs = self.resolve_package_versions( - affected_version_range=affected_version_range, - pkg_type=pkg_type, - pkg_namespace=pkg_namespace, - pkg_name=pkg_name, - valid_versions=valid_versions, - ) - vulnerable_packages.extend(vulnerable_pvs) - fixed_packages.extend(fixed_pvs) - return vulnerable_packages, fixed_packages - - def resolve_package_versions( - self, - affected_version_range, - pkg_type, - pkg_namespace, - pkg_name, - valid_versions, - ): - """ - Return a tuple of lists of ``affected_packages`` and ``fixed_packages`` PackageURL for the given `affected_version_range` and `valid_versions`. - - ``valid_versions`` are the valid version listed on the package registry for that package - - """ - aff_vers, unaff_vers = resolve_version_range( - affected_version_range=affected_version_range, - ignorable_versions=self.ignorable_versions, - package_versions=valid_versions, - ) - - affected_purls = list( - self.expand_verion_range_to_purls(pkg_type, pkg_namespace, pkg_name, aff_vers) - ) - - unaffected_purls = list( - self.expand_verion_range_to_purls(pkg_type, pkg_namespace, pkg_name, unaff_vers) - ) - - fixed_packages = [] - affected_packages = [] - - patched_packages = nearest_patched_package( - vulnerable_packages=affected_purls, resolved_packages=unaffected_purls - ) - - for ( - fixed_package, - affected_purls, - ) in get_affected_packages_by_patched_package(patched_packages).items(): - if fixed_package: - fixed_packages.append(fixed_package) - affected_packages.extend(affected_purls) - - return affected_packages, fixed_packages - - def expand_verion_range_to_purls(self, pkg_type, pkg_namespace, pkg_name, versions): - for version in versions: - yield PackageURL(type=pkg_type, namespace=pkg_namespace, name=pkg_name, version=version) diff --git a/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py b/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py index c90af00c4..249133eaa 100644 --- a/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py +++ b/vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py @@ -22,7 +22,7 @@ from univers.versions import SemverVersion from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 @@ -151,7 +151,6 @@ class ApacheHTTPDImporterPipeline(VulnerableCodeBaseImporterPipelineV2): spdx_license_expression = "Apache-2.0" license_url = "https://www.apache.org/licenses/LICENSE-2.0" base_url = "https://httpd.apache.org/security/json/" - unfurl_version_ranges = True links = [] @@ -219,7 +218,6 @@ class ApacheHTTPDImporterPipeline(VulnerableCodeBaseImporterPipelineV2): "pre_ajp_proxy", ] ) - unfurl_version_ranges = True @classmethod def steps(cls): @@ -292,7 +290,7 @@ def to_advisory(self, data): affected_version_range = self.to_version_ranges(versions_data, fixed_versions) if affected_version_range: affected_packages.append( - AffectedPackage( + AffectedPackageV2( package=PackageURL( type="apache", name="httpd", diff --git a/vulnerabilities/pipelines/v2_importers/curl_importer.py b/vulnerabilities/pipelines/v2_importers/curl_importer.py index e3253b4b4..03610a1e0 100644 --- a/vulnerabilities/pipelines/v2_importers/curl_importer.py +++ b/vulnerabilities/pipelines/v2_importers/curl_importer.py @@ -12,10 +12,9 @@ from cwe2.database import Database from packageurl import PackageURL from univers.version_range import GenericVersionRange -from univers.versions import SemverVersion from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 @@ -37,7 +36,6 @@ class CurlImporterPipeline(VulnerableCodeBaseImporterPipelineV2): license_url = "https://curl.se/docs/copyright.html" repo_url = "https://github.com/curl/curl-www/" url = "https://curl.se/docs/vuln.json" - unfurl_version_ranges = True @classmethod def steps(cls): @@ -77,16 +75,16 @@ def parse_curl_advisory(raw_data) -> AdvisoryData: version_type = get_item(ranges, "type") if get_item(ranges, "type") else "" fixed_version = events.get("fixed") if version_type == "SEMVER" and fixed_version: - fixed_version = SemverVersion(fixed_version) + fixed_version_range = GenericVersionRange.from_versions([fixed_version]) purl = PackageURL(type="generic", namespace="curl.se", name="curl") versions = affected.get("versions") or [] affected_version_range = GenericVersionRange.from_versions(versions) - affected_package = AffectedPackage( + affected_package = AffectedPackageV2( package=purl, affected_version_range=affected_version_range, - fixed_version=fixed_version, + fixed_version_range=fixed_version_range, ) database_specific = raw_data.get("database_specific") or {} diff --git a/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py b/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py index 0c47b9c1a..4fb95ad3b 100644 --- a/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py +++ b/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py @@ -17,7 +17,7 @@ from univers.version_range import HexVersionRange from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 from vulnerabilities.utils import is_cve @@ -26,7 +26,7 @@ class ElixirSecurityImporterPipeline(VulnerableCodeBaseImporterPipelineV2): """ - Elixir Security Advisiories Importer Pipeline + Elixir Security Advisories Importer Pipeline This pipeline imports security advisories for elixir. """ @@ -35,7 +35,6 @@ class ElixirSecurityImporterPipeline(VulnerableCodeBaseImporterPipelineV2): spdx_license_expression = "CC0-1.0" license_url = "https://github.com/dependabot/elixir-security-advisories/blob/master/LICENSE.txt" repo_url = "git+https://github.com/dependabot/elixir-security-advisories" - unfurl_version_ranges = True @classmethod def steps(cls): @@ -118,7 +117,7 @@ def process_file(self, file, base_path) -> Iterable[AdvisoryData]: affected_packages = [] if pkg_name: affected_packages.append( - AffectedPackage( + AffectedPackageV2( package=PackageURL(type="hex", name=pkg_name), affected_version_range=HexVersionRange(constraints=constraints), ) diff --git a/vulnerabilities/pipelines/v2_importers/github_osv_importer.py b/vulnerabilities/pipelines/v2_importers/github_osv_importer.py index ef96e4a84..36f9d06b7 100644 --- a/vulnerabilities/pipelines/v2_importers/github_osv_importer.py +++ b/vulnerabilities/pipelines/v2_importers/github_osv_importer.py @@ -29,7 +29,6 @@ class GithubOSVImporterPipeline(VulnerableCodeBaseImporterPipelineV2): spdx_license_expression = "CC-BY-4.0" license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md" repo_url = "git+https://github.com/github/advisory-database/" - unfurl_version_ranges = True @classmethod def steps(cls): diff --git a/vulnerabilities/pipelines/v2_importers/gitlab_importer.py b/vulnerabilities/pipelines/v2_importers/gitlab_importer.py index ff87dd5c9..e28ab7520 100644 --- a/vulnerabilities/pipelines/v2_importers/gitlab_importer.py +++ b/vulnerabilities/pipelines/v2_importers/gitlab_importer.py @@ -12,7 +12,6 @@ import traceback from pathlib import Path from typing import Iterable -from typing import List from typing import Tuple import pytz @@ -21,12 +20,10 @@ from fetchcode.vcs import fetch_via_vcs from packageurl import PackageURL from univers.version_range import RANGE_CLASS_BY_SCHEMES -from univers.version_range import VersionRange from univers.version_range import from_gitlab_native -from univers.versions import Version from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 from vulnerabilities.utils import build_description @@ -45,7 +42,6 @@ class GitLabImporterPipeline(VulnerableCodeBaseImporterPipelineV2): spdx_license_expression = "MIT" license_url = "https://gitlab.com/gitlab-org/advisories-community/-/blob/main/LICENSE" repo_url = "git+https://gitlab.com/gitlab-org/advisories-community/" - unfurl_version_ranges = True @classmethod def steps(cls): @@ -177,27 +173,6 @@ def get_purl(package_slug, purl_type_by_gitlab_scheme, logger): return -def extract_affected_packages( - affected_version_range: VersionRange, - fixed_versions: List[Version], - purl: PackageURL, -) -> Iterable[AffectedPackage]: - """ - Yield AffectedPackage objects, one for each fixed_version - - In case of gitlab advisory data we get a list of fixed_versions and a affected_version_range. - Since we can not determine which package fixes which range. - We store the all the fixed_versions with the same affected_version_range in the advisory. - Later the advisory data is used to be inferred in the GitLabBasicImprover. - """ - for fixed_version in fixed_versions: - yield AffectedPackage( - package=purl, - fixed_version=fixed_version, - affected_version_range=affected_version_range, - ) - - def parse_gitlab_advisory( file, base_path, gitlab_scheme_by_purl_type, purl_type_by_gitlab_scheme, logger ): @@ -276,7 +251,7 @@ def parse_gitlab_advisory( fixed_versions = gitlab_advisory.get("fixed_versions") or [] affected_range = gitlab_advisory.get("affected_range") gitlab_native_schemes = set(["pypi", "gem", "npm", "go", "packagist", "conan"]) - vrc: VersionRange = RANGE_CLASS_BY_SCHEMES[purl.type] + vrc = RANGE_CLASS_BY_SCHEMES[purl.type] gitlab_scheme = gitlab_scheme_by_purl_type[purl.type] try: if affected_range: @@ -296,38 +271,33 @@ def parse_gitlab_advisory( for fixed_version in fixed_versions: try: fixed_version = vrc.version_class(fixed_version) - parsed_fixed_versions.append(fixed_version) + parsed_fixed_versions.append(fixed_version.string) except Exception as e: logger( f"parse_yaml_file: fixed_version is not parsable`: {fixed_version!r} error: {e!r}\n {traceback.format_exc()}", level=logging.ERROR, ) - if parsed_fixed_versions: - affected_packages = list( - extract_affected_packages( - affected_version_range=affected_version_range, - fixed_versions=parsed_fixed_versions, - purl=purl, - ) - ) - else: - if not affected_version_range: - affected_packages = [] - else: - affected_packages = [ - AffectedPackage( - package=purl, - affected_version_range=affected_version_range, - ) - ] + if affected_version_range: + vrc = affected_version_range.__class__ + + fixed_version_range = vrc.from_versions(parsed_fixed_versions) + if not fixed_version_range and not affected_version_range: + return + + affected_package = AffectedPackageV2( + package=purl, + affected_version_range=affected_version_range, + fixed_version_range=fixed_version_range, + ) + return AdvisoryData( advisory_id=advisory_id, aliases=aliases, summary=summary, references_v2=references, date_published=date_published, - affected_packages=affected_packages, + affected_packages=[affected_package], weaknesses=cwe_list, url=advisory_url, original_advisory_text=json.dumps(gitlab_advisory, indent=2, ensure_ascii=False), diff --git a/vulnerabilities/pipelines/v2_importers/istio_importer.py b/vulnerabilities/pipelines/v2_importers/istio_importer.py index bc544f7f8..4126ec75e 100644 --- a/vulnerabilities/pipelines/v2_importers/istio_importer.py +++ b/vulnerabilities/pipelines/v2_importers/istio_importer.py @@ -20,10 +20,11 @@ from univers.version_constraint import VersionConstraint from univers.version_range import GitHubVersionRange from univers.version_range import GolangVersionRange +from univers.versions import GolangVersion from univers.versions import SemverVersion from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 from vulnerabilities.utils import get_advisory_url @@ -41,7 +42,6 @@ class IstioImporterPipeline(VulnerableCodeBaseImporterPipelineV2): spdx_license_expression = "Apache-2.0" license_url = "https://github.com/istio/istio.io/blob/master/LICENSE" repo_url = "git+https://github.com/istio/istio.io" - unfurl_version_ranges = True @classmethod def steps(cls): @@ -80,23 +80,27 @@ def collect_advisories(self) -> Iterable[AdvisoryData]: release_date = ( parser.parse(published_date).replace(tzinfo=pytz.UTC) if published_date else None ) - constraints = self.get_version_constraints(data.get("releases", [])) - + semver_constraints = self.get_version_constraints(data.get("releases", [])) + golang_constraints = self.get_version_constraints( + data.get("releases", []), GolangVersion + ) cves = data.get("cves", []) affected_packages = [] - if constraints: - affected_packages.extend( - [ - AffectedPackage( - package=PackageURL(type="golang", namespace="istio.io", name="istio"), - affected_version_range=GolangVersionRange(constraints=constraints), - ), - AffectedPackage( - package=PackageURL(type="github", namespace="istio", name="istio"), - affected_version_range=GitHubVersionRange(constraints=constraints), - ), - ] + if semver_constraints: + affected_packages.append( + AffectedPackageV2( + package=PackageURL(type="github", namespace="istio", name="istio"), + affected_version_range=GitHubVersionRange(constraints=semver_constraints), + ), + ) + + if golang_constraints: + affected_packages.append( + AffectedPackageV2( + package=PackageURL(type="golang", namespace="istio.io", name="istio"), + affected_version_range=GolangVersionRange(constraints=golang_constraints), + ), ) title = data.get("title") or "" @@ -127,37 +131,37 @@ def parse_markdown(self, path: Path) -> dict: front_matter, _ = split_markdown_front_matter(text) return saneyaml.load(front_matter) - def get_version_constraints(self, releases: List[str]) -> List[VersionConstraint]: + def get_version_constraints( + self, + releases: List[str], + version_cls=SemverVersion, + ) -> List[VersionConstraint]: constraints = [] for release in releases: release = release.strip() if "All releases prior" in release: _, _, version = release.rpartition(" ") - constraints.append( - VersionConstraint(version=SemverVersion(version), comparator="<") - ) + constraints.append(VersionConstraint(version=version_cls(version), comparator="<")) elif "All releases" in release and "and later" in release: version = release.replace("All releases", "").replace("and later", "").strip() if is_release(version): constraints.append( - VersionConstraint(version=SemverVersion(version), comparator=">=") + VersionConstraint(version=version_cls(version), comparator=">=") ) elif "to" in release: lower, _, upper = release.partition("to") constraints.append( - VersionConstraint(version=SemverVersion(lower.strip()), comparator=">=") + VersionConstraint(version=version_cls(lower.strip()), comparator=">=") ) constraints.append( - VersionConstraint(version=SemverVersion(upper.strip()), comparator="<=") + VersionConstraint(version=version_cls(upper.strip()), comparator="<=") ) elif is_release(release): - constraints.append( - VersionConstraint(version=SemverVersion(release), comparator="=") - ) + constraints.append(VersionConstraint(version=version_cls(release), comparator="=")) return constraints diff --git a/vulnerabilities/pipelines/v2_importers/mozilla_importer.py b/vulnerabilities/pipelines/v2_importers/mozilla_importer.py index 668b6a498..e8a6aef7a 100644 --- a/vulnerabilities/pipelines/v2_importers/mozilla_importer.py +++ b/vulnerabilities/pipelines/v2_importers/mozilla_importer.py @@ -18,10 +18,10 @@ from fetchcode.vcs import fetch_via_vcs from markdown import markdown from packageurl import PackageURL -from univers.versions import SemverVersion +from univers.version_range import GenericVersionRange from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 @@ -189,7 +189,7 @@ def extract_description_from_html(md_text: str) -> str: return "\n".join(description_parts).strip() -def parse_affected_packages(pkgs: list) -> Iterable[AffectedPackage]: +def parse_affected_packages(pkgs: list) -> Iterable[AffectedPackageV2]: for pkg in pkgs: if not pkg: continue @@ -198,14 +198,14 @@ def parse_affected_packages(pkgs: list) -> Iterable[AffectedPackage]: if version.count(".") == 3: continue # invalid SemVer try: - fixed_version = SemverVersion(version) + fixed_version_range = GenericVersionRange.from_versions([version]) except Exception: logger.debug(f"Invalid version '{version}' for package '{name}'") continue - yield AffectedPackage( + yield AffectedPackageV2( package=PackageURL(type="mozilla", name=name), - fixed_version=fixed_version, + fixed_version_range=fixed_version_range, ) diff --git a/vulnerabilities/pipelines/v2_importers/npm_importer.py b/vulnerabilities/pipelines/v2_importers/npm_importer.py index 5f0306413..bf9de86d1 100644 --- a/vulnerabilities/pipelines/v2_importers/npm_importer.py +++ b/vulnerabilities/pipelines/v2_importers/npm_importer.py @@ -9,7 +9,6 @@ # Author: Navonil Das (@NavonilDas) -import json from pathlib import Path from typing import Iterable @@ -20,7 +19,7 @@ from univers.version_range import NpmVersionRange from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 @@ -32,7 +31,7 @@ class NpmImporterPipeline(VulnerableCodeBaseImporterPipelineV2): """ - Node.js Security Working Group importer pipeline + Node.js Security Working Group importer pipeline. Import advisories from nodejs security working group including node proper advisories and npm advisories. """ @@ -41,7 +40,6 @@ class NpmImporterPipeline(VulnerableCodeBaseImporterPipelineV2): spdx_license_expression = "MIT" license_url = "https://github.com/nodejs/security-wg/blob/main/LICENSE.md" repo_url = "git+https://github.com/nodejs/security-wg" - unfurl_version_ranges = True @classmethod def steps(cls): @@ -70,10 +68,12 @@ def to_advisory_data(self, file: Path) -> Iterable[AdvisoryData]: self.log(f"Skipping {file.name} file") return data = load_json(file) - advisory_text = None - with open(file) as f: - advisory_text = f.read() + advisory_text = file.read_text() id = data.get("id") + if not id: + self.log(f"Advisory ID not found in {file}") + return + description = data.get("overview") or "" summary = data.get("title") or "" # TODO: Take care of description @@ -100,9 +100,6 @@ def to_advisory_data(self, file: Path) -> Iterable[AdvisoryData]: url=f"https://github.com/nodejs/security-wg/blob/main/vuln/npm/{id}.json", ) ) - if not id: - self.log(f"Advisory ID not found in {file}") - return advisory_reference = ReferenceV2( url=f"https://github.com/nodejs/security-wg/blob/main/vuln/npm/{id}.json", @@ -134,13 +131,12 @@ def to_advisory_data(self, file: Path) -> Iterable[AdvisoryData]: references_v2=references, severities=severities, url=f"https://github.com/nodejs/security-wg/blob/main/vuln/npm/{id}.json", - original_advisory_text=advisory_text or json.dumps(data, indent=2, ensure_ascii=False), + original_advisory_text=advisory_text, ) def get_affected_package(self, data, package_name): affected_version_range = None unaffected_version_range = None - fixed_version = None vulnerable_range = data.get("vulnerable_versions") or "" patched_range = data.get("patched_versions") or "" @@ -157,21 +153,13 @@ def get_affected_package(self, data, package_name): if patched_range: unaffected_version_range = NpmVersionRange.from_native(patched_range) - # We only store single fixed versions and not a range of fixed versions - # If there is a single constraint in the unaffected_version_range - # having comparator as ">=" then we store that as the fixed version - if unaffected_version_range and len(unaffected_version_range.constraints) == 1: - constraint = unaffected_version_range.constraints[0] - if constraint.comparator == ">=": - fixed_version = constraint.version - - return AffectedPackage( + return AffectedPackageV2( package=PackageURL( type="npm", name=package_name, ), affected_version_range=affected_version_range, - fixed_version=fixed_version, + fixed_version_range=unaffected_version_range, ) def clean_downloads(self): diff --git a/vulnerabilities/pipelines/v2_importers/nvd_importer.py b/vulnerabilities/pipelines/v2_importers/nvd_importer.py index 2b9de3bd8..876b7a905 100644 --- a/vulnerabilities/pipelines/v2_importers/nvd_importer.py +++ b/vulnerabilities/pipelines/v2_importers/nvd_importer.py @@ -111,7 +111,7 @@ def fetch(url, logger=None): return json.loads(data) -def fetch_cve_data_1_1(starting_year=2002, logger=None): +def fetch_cve_data_1_1(starting_year=2025, logger=None): """ Yield tuples of (year, lists of CVE mappings) from the NVD, one for each year since ``starting_year`` defaulting to 2002. @@ -326,7 +326,7 @@ def to_advisory(self): weaknesses=self.weaknesses, severities=self.severities, url=f"https://nvd.nist.gov/vuln/detail/{self.cve_id}", - raw_data=json.dumps(self.cve_item, indent=2, ensure_ascii=False), + original_advisory_text=json.dumps(self.cve_item, indent=2, ensure_ascii=False), ) diff --git a/vulnerabilities/pipelines/v2_importers/oss_fuzz.py b/vulnerabilities/pipelines/v2_importers/oss_fuzz.py index 9c9d6bed8..9c5f78d90 100644 --- a/vulnerabilities/pipelines/v2_importers/oss_fuzz.py +++ b/vulnerabilities/pipelines/v2_importers/oss_fuzz.py @@ -25,7 +25,6 @@ class OSSFuzzImporterPipeline(VulnerableCodeBaseImporterPipelineV2): spdx_license_expression = "CC-BY-4.0" license_url = "https://github.com/google/oss-fuzz-vulns/blob/main/LICENSE" repo_url = "git+https://github.com/google/oss-fuzz-vulns" - unfurl_version_ranges = True @classmethod def steps(cls): diff --git a/vulnerabilities/pipelines/v2_importers/postgresql_importer.py b/vulnerabilities/pipelines/v2_importers/postgresql_importer.py index 60b099ff6..2ca4c7b5b 100644 --- a/vulnerabilities/pipelines/v2_importers/postgresql_importer.py +++ b/vulnerabilities/pipelines/v2_importers/postgresql_importer.py @@ -18,7 +18,7 @@ from vulnerabilities import severity_systems from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 @@ -83,31 +83,25 @@ def to_advisories(self, data, url): affected_packages = [] affected_version_list = [v.strip() for v in affected_col.text.split(",") if v.strip()] fixed_version_list = [v.strip() for v in fixed_col.text.split(",") if v.strip()] - - if fixed_version_list: - for fixed_version in fixed_version_list: - affected_packages.append( - AffectedPackage( - package=PackageURL( - name="postgresql", type="generic", qualifiers=pkg_qualifiers - ), - affected_version_range=GenericVersionRange.from_versions( - affected_version_list - ) - if affected_version_list - else None, - fixed_version=GenericVersion(fixed_version), - ) - ) - elif affected_version_list: + fixed_version_range = ( + GenericVersionRange.from_versions(fixed_version_list) + if fixed_version_list + else None + ) + affected_version_range = ( + GenericVersionRange.from_versions(affected_version_list) + if affected_version_list + else None + ) + + if affected_version_range or fixed_version_range: affected_packages.append( - AffectedPackage( + AffectedPackageV2( package=PackageURL( name="postgresql", type="generic", qualifiers=pkg_qualifiers ), - affected_version_range=GenericVersionRange.from_versions( - affected_version_list - ), + affected_version_range=affected_version_range, + fixed_version_range=fixed_version_range, ) ) diff --git a/vulnerabilities/pipelines/v2_importers/pypa_importer.py b/vulnerabilities/pipelines/v2_importers/pypa_importer.py index 8f9f57ddf..fa5edaa0b 100644 --- a/vulnerabilities/pipelines/v2_importers/pypa_importer.py +++ b/vulnerabilities/pipelines/v2_importers/pypa_importer.py @@ -28,7 +28,6 @@ class PyPaImporterPipeline(VulnerableCodeBaseImporterPipelineV2): spdx_license_expression = "CC-BY-4.0" license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" repo_url = "git+https://github.com/pypa/advisory-database" - unfurl_version_ranges = True @classmethod def steps(cls): diff --git a/vulnerabilities/pipelines/v2_importers/pysec_importer.py b/vulnerabilities/pipelines/v2_importers/pysec_importer.py index ed41fdc87..bea79cb98 100644 --- a/vulnerabilities/pipelines/v2_importers/pysec_importer.py +++ b/vulnerabilities/pipelines/v2_importers/pysec_importer.py @@ -28,7 +28,6 @@ class PyPIImporterPipeline(VulnerableCodeBaseImporterPipelineV2): license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip" spdx_license_expression = "CC-BY-4.0" - unfurl_version_ranges = True @classmethod def steps(cls): diff --git a/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py b/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py index d8590b498..a596d8d65 100644 --- a/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py +++ b/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py @@ -69,7 +69,14 @@ def parse_cve_advisory(self, raw_data, advisory_url): date_published = cve_metadata.get("datePublished") if date_published: - date_published = dateparser.parse(date_published) + date_published = dateparser.parse( + date_published, + settings={ + "TIMEZONE": "UTC", + "RETURN_AS_TIMEZONE_AWARE": True, + "TO_TIMEZONE": "UTC", + }, + ) # Extract containers containers = raw_data.get("containers", {}) diff --git a/vulnerabilities/pipelines/v2_improvers/collect_commits.py b/vulnerabilities/pipelines/v2_improvers/collect_commits.py index 32fb1ce79..d9be9781e 100644 --- a/vulnerabilities/pipelines/v2_improvers/collect_commits.py +++ b/vulnerabilities/pipelines/v2_improvers/collect_commits.py @@ -37,8 +37,8 @@ def steps(cls): def collect_and_store_fix_commits(self): affected_advisories = ( - AdvisoryV2.objects.filter(affecting_packages__isnull=False) - .prefetch_related("affecting_packages") + AdvisoryV2.objects.filter(impacted_packages__affecting_packages__isnull=False) + .prefetch_related("impacted_packages__affecting_packages", "references") .distinct() ) @@ -61,24 +61,19 @@ def collect_and_store_fix_commits(self): # Skip if already processed if is_vcs_url_already_processed(commit_id=vcs_url): - self.log( - f"Skipping already processed reference: {reference.url} with VCS URL {vcs_url}" - ) continue # check if vcs_url has commit - for package in adv.affecting_packages.all(): - code_fix, created = CodeFixV2.objects.get_or_create( - commits=[vcs_url], - advisory=adv, - affected_package=package, - ) - - if created: - created_fix_count += 1 - self.log( - f"Created CodeFix entry for reference: {reference.url} with VCS URL {vcs_url}" + for impact in adv.impacted_packages.all(): + for package in impact.affecting_packages.all(): + code_fix, created = CodeFixV2.objects.get_or_create( + commits=[vcs_url], + advisory=adv, + affected_package=package, ) + if created: + created_fix_count += 1 + self.log(f"Successfully created {created_fix_count:,d} CodeFix entries.") diff --git a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py index 55608f0d1..ac7caa49d 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py @@ -34,17 +34,13 @@ def steps(cls): def compute_and_store_vulnerability_risk_score(self): affected_advisories = ( - AdvisoryV2.objects.filter(affecting_packages__isnull=False) - .prefetch_related( - "references", - "severities", - "exploits", - ) + AdvisoryV2.objects.filter(impacted_packages__affecting_packages__isnull=False) + .prefetch_related("references", "severities", "exploits") .distinct() ) self.log( - f"Calculating risk for {affected_advisories.count():,d} vulnerability with a affected packages records" + f"Calculating risk for {affected_advisories.count():,d} advisory with a affected packages records" ) progress = LoopProgress(total_iterations=affected_advisories.count(), logger=self.log) @@ -53,7 +49,7 @@ def compute_and_store_vulnerability_risk_score(self): updated_vulnerability_count = 0 batch_size = 5000 - for advisory in progress.iter(affected_advisories.paginated(per_page=batch_size)): + for advisory in progress.iter(affected_advisories.iterator(chunk_size=batch_size)): severities = advisory.severities.all() references = advisory.references.all() exploits = advisory.exploits.all() @@ -65,9 +61,6 @@ def compute_and_store_vulnerability_risk_score(self): ) advisory.weighted_severity = weighted_severity advisory.exploitability = exploitability - print( - f"Computed risk for {advisory.advisory_id} with weighted_severity={weighted_severity} and exploitability={exploitability}" - ) updatables.append(advisory) if len(updatables) >= batch_size: @@ -90,9 +83,7 @@ def compute_and_store_vulnerability_risk_score(self): ) def compute_and_store_package_risk_score(self): - affected_packages = ( - PackageV2.objects.filter(affected_by_advisories__isnull=False) - ).distinct() + affected_packages = (PackageV2.objects.filter(affected_in_impacts__isnull=False)).distinct() self.log(f"Calculating risk for {affected_packages.count():,d} affected package records") @@ -106,7 +97,7 @@ def compute_and_store_package_risk_score(self): updated_package_count = 0 batch_size = 10000 - for package in progress.iter(affected_packages.paginated(per_page=batch_size)): + for package in progress.iter(affected_packages.iterator(chunk_size=batch_size)): risk_score = compute_package_risk_v2(package) if not risk_score: diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 2736d8874..3487b74db 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -35,6 +35,7 @@ from vulnerabilities.models import VulnerabilityRelatedReference from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness +from vulnerabilities.pipes.univers_utils import get_exact_purls_v2 def get_or_create_aliases(aliases: List) -> QuerySet: @@ -43,9 +44,6 @@ def get_or_create_aliases(aliases: List) -> QuerySet: return Alias.objects.filter(alias__in=aliases) -from django.db.models import Q - - def get_or_create_advisory_aliases(aliases: List[str]) -> List[AdvisoryAlias]: existing = AdvisoryAlias.objects.filter(alias__in=aliases) existing_aliases = {a.alias for a in existing} @@ -136,12 +134,14 @@ def insert_advisory(advisory: AdvisoryData, pipeline_id: str, logger: Callable = return advisory_obj +@transaction.atomic def insert_advisory_v2( advisory: AdvisoryData, pipeline_id: str, - get_advisory_packages: Callable, logger: Callable = None, ): + from vulnerabilities.models import ImpactedPackage + from vulnerabilities.models import PackageV2 from vulnerabilities.utils import compute_content_id advisory_obj = None @@ -150,7 +150,6 @@ def insert_advisory_v2( severities = get_or_create_advisory_severities(severities=advisory.severities) weaknesses = get_or_create_advisory_weaknesses(weaknesses=advisory.weaknesses) content_id = compute_content_id(advisory_data=advisory) - affecting_packages, fixed_by_packages = get_advisory_packages(advisory_data=advisory) try: default_data = { "datasource_id": pipeline_id, @@ -162,7 +161,7 @@ def insert_advisory_v2( "original_advisory_text": advisory.original_advisory_text, } - advisory_obj, _ = AdvisoryV2.objects.get_or_create( + advisory_obj, created = AdvisoryV2.objects.get_or_create( unique_content_id=content_id, url=advisory.url, defaults=default_data, @@ -172,8 +171,6 @@ def insert_advisory_v2( "references": references, "severities": severities, "weaknesses": weaknesses, - "fixed_by_packages": fixed_by_packages, - "affecting_packages": affecting_packages, } for field_name, values in related_fields.items(): @@ -192,6 +189,29 @@ def insert_advisory_v2( level=logging.ERROR, ) + if created: + for affected_pkg in advisory.affected_packages: + impact = ImpactedPackage.objects.create( + advisory=advisory_obj, + base_purl=str(affected_pkg.package), + affecting_vers=str(affected_pkg.affected_version_range), + fixed_vers=str(affected_pkg.fixed_version_range), + ) + package_affected_purls, package_fixed_purls = get_exact_purls_v2( + affected_package=affected_pkg, + logger=logger, + ) + affected_packages_v2 = [ + PackageV2.objects.get_or_create_from_purl(purl=purl)[0] + for purl in package_affected_purls + ] + fixed_packages_v2 = [ + PackageV2.objects.get_or_create_from_purl(purl=purl)[0] + for purl in package_fixed_purls + ] + impact.affecting_packages.add(*affected_packages_v2) + impact.fixed_by_packages.add(*fixed_packages_v2) + return advisory_obj diff --git a/vulnerabilities/pipes/univers_utils.py b/vulnerabilities/pipes/univers_utils.py new file mode 100644 index 000000000..296ebc12f --- /dev/null +++ b/vulnerabilities/pipes/univers_utils.py @@ -0,0 +1,95 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +from traceback import format_exc as traceback_format_exc +from typing import Callable +from typing import List +from typing import Tuple + +from packageurl import PackageURL +from univers.version_range import VersionRange + +from vulnerabilities.importer import AffectedPackageV2 +from vulnerabilities.utils import update_purl_version + + +def get_exact_purl_from_vers( + version_range: VersionRange, base_purl: PackageURL +) -> Tuple[List[PackageURL], List[PackageURL]]: + """Return list of exact PURLs that in and outside vers range.""" + if not version_range: + return [], [] + + all_versions = [c.version for c in version_range.constraints if c] + version_not_in_range = [ + c.version for c in version_range.constraints if c and c.comparator == "!=" + ] + version_in_range = [v for v in all_versions if v and v in version_range] + + purl_in_range = [ + update_purl_version(purl=base_purl, version=str(version)) for version in version_in_range + ] + purl_not_in_range = [ + update_purl_version(purl=base_purl, version=str(version)) + for version in version_not_in_range + ] + + return purl_in_range, purl_not_in_range + + +def get_exact_purls_v2( + affected_package: AffectedPackageV2, + logger: Callable = None, +) -> Tuple[List[PackageURL], PackageURL]: + """ + Return a list of affected purls and the fixed package found in the ``affected_package`` + AffectedPackageV2 disregarding any ranges. + + Only exact version constraints (ie with an equality) are considered + For eg: + >>> purl = {"type": "turtle", "name": "green"} + >>> vers = "vers:npm/<1.0.0 | >=2.0.0 | <3.0.0" + >>> vers2 = "vers:npm/5.0.0" + >>> affected_package = AffectedPackageV2.from_dict({ + ... "package": purl, + ... "affected_version_range": vers, + ... "fixed_version_range": vers2 + ... }) + >>> got = get_exact_purls_v2(affected_package) + >>> expected = ( + ... [PackageURL(type='turtle', namespace=None, name='green', version='2.0.0', qualifiers={}, subpath=None)], + ... [PackageURL(type='turtle', namespace=None, name='green', version='5.0.0', qualifiers={}, subpath=None)] + ... ) + >>> assert expected == got + """ + if not affected_package: + return [], [] + + try: + affected_purls, fixed_purls = get_exact_purl_from_vers( + version_range=affected_package.affected_version_range, + base_purl=affected_package.package, + ) + + fixed_purls_2, affected_purls_2 = get_exact_purl_from_vers( + version_range=affected_package.fixed_version_range, + base_purl=affected_package.package, + ) + + affected_purls.extend(affected_purls_2) + fixed_purls.extend(fixed_purls_2) + + return affected_purls, fixed_purls + except Exception as e: + logger( + f"Failed to get exact purls for: {affected_package!r} with error: {e!r} \n{traceback_format_exc()}", + level=logging.ERROR, + ) + return [], [] diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index 56f19171e..04af140f0 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -122,8 +122,8 @@ def compute_package_risk_v2(package): and determining the associated risk. """ result = [] - for advisory in package.affected_by_advisories.all(): - if risk := advisory.risk_score: + for impact in package.affected_in_impacts.all(): + if risk := impact.advisory.risk_score: result.append(float(risk)) if not result: diff --git a/vulnerabilities/templates/advisory_package_details.html b/vulnerabilities/templates/advisory_package_details.html index 0f4c71044..234c5bae0 100644 --- a/vulnerabilities/templates/advisory_package_details.html +++ b/vulnerabilities/templates/advisory_package_details.html @@ -6,83 +6,62 @@ {% load url_filters %} {% block title %} -VulnerableCode Advisory Package Details - {{ advisory.advisory_id }} +VulnerableCode Advisory Package Details - {{ advisoryv2.advisory_id }} {% endblock %} {% block content %} -{% if advisory %} -
-
-
-
- Vulnerable and Fixing Package details for Advisory: - - {{ advisory.advisory_id }} - -
-
-
- - - - - - - - - {% for package in affected_packages %} - - - + + {% empty %} + + + + {% endfor %} + +
AffectedFixed by
- {{ package.purl }} - - - {% for match in all_affected_fixed_by_matches %} - {% if match.affected_package == package %} - {% if match.matched_fixed_by_packages|length > 0 %} - {% for pkg in match.matched_fixed_by_packages %} - {{ pkg }} -
- {% endfor %} - {% else %} +{% if advisoryv2 %} +
+
+
+
+ Vulnerable and Fixing Package details for Advisory: + + {{ advisoryv2.advisory_id }} + +
+
+
+ + + + + + + + + {% for impact in advisoryv2.impacted_packages.all %} + + + - - {% empty %} - - - - {% endfor %} - -
AffectedFixed by
+ {% for package in impact.affecting_packages.all %} + {{ package.purl }} +
+ {% endfor %} +
+ {% for package in impact.fixed_by_packages.all %} + {{ package.purl }} +
+ {% empty %} There are no reported fixed by versions. - {% endif %} - {% endif %} - {% endfor %} - -
- This vulnerability is not known to affect any packages. -
-
-
-
+ {% endfor %} +
+ This vulnerability is not known to affect any packages. +
+
+
+
{% endif %} - - {% endblock %} \ No newline at end of file diff --git a/vulnerabilities/templates/package_details_v2.html b/vulnerabilities/templates/package_details_v2.html index be0af62fe..1af0866ef 100644 --- a/vulnerabilities/templates/package_details_v2.html +++ b/vulnerabilities/templates/package_details_v2.html @@ -3,6 +3,7 @@ {% load widget_tweaks %} {% load static %} {% load url_filters %} +{% load utils %} {% block title %} VulnerableCode Package Details - {{ package.purl }} @@ -60,7 +61,7 @@ - {{ fixed_package_details.purl.to_string }} + {{ package.purl }} {% if package.is_ghost %} @@ -91,9 +92,9 @@ Next non-vulnerable version - {% if fixed_package_details.next_non_vulnerable.version %} - {{ fixed_package_details.next_non_vulnerable.version }} + {% if next_non_vulnerable.version %} + {{ next_non_vulnerable.version }} {% else %} None. {% endif %} @@ -104,9 +105,9 @@ Latest non-vulnerable version - {% if fixed_package_details.latest_non_vulnerable.version %} - {{ fixed_package_details.latest_non_vulnerable.version }} + {% if latest_non_vulnerable.version %} + {{ latest_non_vulnerable.version }} {% else %} None. {% endif %} @@ -175,66 +176,22 @@ {{ advisory.summary }} - {% if package.purl == fixed_package_details.purl.to_string %} - {% for key, value in fixed_package_details.items %} - {% if key == "advisories" %} - {% for vuln in value %} - {% if vuln.advisory.advisory_id == advisory.advisory_id %} - {% if vuln.fixed_by_package_details is None %} - There are no reported fixed by versions. - {% else %} - {% for fixed_pkg in vuln.fixed_by_package_details %} -
- {% if fixed_pkg.fixed_by_purl_advisories|length == 0 %} - {{ fixed_pkg.fixed_by_purl.version }} -
- Subject of 0 other advisories. - {% else %} - {{ fixed_pkg.fixed_by_purl.version }} - {% if fixed_pkg.fixed_by_purl_advisories|length != 1 %} -
- Subject of {{ fixed_pkg.fixed_by_purl_advisories|length }} other - advisory. - {% else %} -
- Subject of {{ fixed_pkg.fixed_by_purl_advisories|length }} other - advisory. - {% endif %} - - - {% endif %} -
- {% endfor %} - {% endif %} - {% endif %} - {% endfor %} - {% endif %} - {% endfor %} - {% endif %} + {% with fixed=fixed_package_details|get_item:advisory.id %} + {% if fixed %} + {% for item in fixed %} +
+ {{ item.pkg.version }} +
+ + Subject of {{ item.affected_count }} other advisories. + +
+ {% endfor %} + {% else %} + There are no reported fixed by versions. + {% endif %} + {% endwith %} {% empty %} diff --git a/vulnerabilities/templatetags/utils.py b/vulnerabilities/templatetags/utils.py index 952221f48..cea889808 100644 --- a/vulnerabilities/templatetags/utils.py +++ b/vulnerabilities/templatetags/utils.py @@ -34,3 +34,8 @@ def active_item(context, url_name): if request.resolver_match.url_name == url_name: return "is-active" return "" + + +@register.filter +def get_item(dictionary, key): + return dictionary.get(key) diff --git a/vulnerabilities/tests/pipelines/test_vulnerablecode_importer_v2_pipeline.py b/vulnerabilities/tests/pipelines/test_vulnerablecode_importer_v2_pipeline.py deleted file mode 100644 index f995f0c1f..000000000 --- a/vulnerabilities/tests/pipelines/test_vulnerablecode_importer_v2_pipeline.py +++ /dev/null @@ -1,180 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -from datetime import datetime -from datetime import timedelta -from unittest import mock - -import pytest -from packageurl import PackageURL - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import UnMergeablePackageError -from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.models import PackageV2 -from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 - - -class DummyImporter(VulnerableCodeBaseImporterPipelineV2): - pipeline_id = "dummy" - log_messages = [] - - def log(self, message, level=logging.INFO): - self.log_messages.append((level, message)) - - def collect_advisories(self): - yield from self._advisories - - def advisories_count(self): - return len(self._advisories) - - -@pytest.fixture -def dummy_advisory(): - return AdvisoryData( - summary="Test advisory", - aliases=["CVE-2025-0001"], - references_v2=[], - severities=[], - weaknesses=[], - affected_packages=[], - advisory_id="ADV-123", - date_published=datetime.now() - timedelta(days=10), - url="https://example.com/advisory/1", - ) - - -@pytest.fixture -def dummy_importer(dummy_advisory): - importer = DummyImporter() - importer._advisories = [dummy_advisory] - return importer - - -@pytest.mark.django_db -def test_collect_and_store_advisories(dummy_importer): - dummy_importer.collect_and_store_advisories() - assert len(dummy_importer.log_messages) >= 2 - assert "Successfully collected" in dummy_importer.log_messages[-1][1] - assert AdvisoryV2.objects.count() == 1 - - -def test_get_advisory_packages_basic(dummy_importer): - purl = PackageURL("pypi", None, "dummy", "1.0.0") - affected_package = mock.Mock() - affected_package.package = purl - dummy_importer.unfurl_version_ranges = False - - with mock.patch( - "vulnerabilities.improvers.default.get_exact_purls", return_value=([purl], [purl]) - ): - with mock.patch.object( - PackageV2.objects, "get_or_create_from_purl", return_value=(mock.Mock(), True) - ) as mock_get: - dummy_importer.get_advisory_packages( - advisory_data=mock.Mock(affected_packages=[affected_package]) - ) - assert mock_get.call_count == 2 # one affected, one fixed - - -def test_get_published_package_versions_filters(dummy_importer): - purl = PackageURL("pypi", None, "example", None) - - dummy_versions = [ - mock.Mock(value="1.0.0", release_date=datetime.now() - timedelta(days=5)), - mock.Mock(value="2.0.0", release_date=datetime.now() + timedelta(days=5)), # future - ] - - with mock.patch( - "vulnerabilities.pipelines.package_versions.versions", return_value=dummy_versions - ): - versions = dummy_importer.get_published_package_versions(purl, until=datetime.now()) - assert "1.0.0" in versions - assert "2.0.0" not in versions - - -def test_get_published_package_versions_failure_logs(dummy_importer): - purl = PackageURL("pypi", None, "example", None) - with mock.patch( - "vulnerabilities.pipelines.package_versions.versions", side_effect=Exception("fail") - ): - versions = dummy_importer.get_published_package_versions(purl) - assert versions == [] - assert any("Failed to fetch versions" in msg for lvl, msg in dummy_importer.log_messages) - - -def test_expand_version_range_to_purls(dummy_importer): - purls = list( - dummy_importer.expand_verion_range_to_purls("npm", "lodash", "lodash", ["1.0.0", "1.1.0"]) - ) - assert all(isinstance(p, PackageURL) for p in purls) - assert purls[0].name == "lodash" - - -def test_resolve_package_versions(dummy_importer): - dummy_importer.ignorable_versions = [] - dummy_importer.expand_verion_range_to_purls = lambda *args, **kwargs: [ - PackageURL("npm", None, "a", "1.0.0") - ] - - with mock.patch( - "vulnerabilities.pipelines.resolve_version_range", return_value=(["1.0.0"], ["1.1.0"]) - ), mock.patch( - "vulnerabilities.pipelines.get_affected_packages_by_patched_package", - return_value={None: [PackageURL("npm", None, "a", "1.0.0")]}, - ), mock.patch( - "vulnerabilities.pipelines.nearest_patched_package", return_value=[] - ): - aff, fix = dummy_importer.resolve_package_versions( - affected_version_range=">=1.0.0", - pkg_type="npm", - pkg_namespace=None, - pkg_name="a", - valid_versions=["1.0.0", "1.1.0"], - ) - assert any(isinstance(p, PackageURL) for p in aff) - - -def test_get_impacted_packages_mergeable(dummy_importer): - ap = mock.Mock() - ap.package = PackageURL("npm", None, "abc", None) - dummy_importer.get_published_package_versions = lambda package_url, until: ["1.0.0", "1.1.0"] - dummy_importer.resolve_package_versions = lambda **kwargs: ( - [PackageURL("npm", None, "abc", "1.0.0")], - [PackageURL("npm", None, "abc", "1.1.0")], - ) - - with mock.patch( - "vulnerabilities.importer.AffectedPackage.merge", - return_value=(ap.package, [">=1.0.0"], ["1.1.0"]), - ): - aff, fix = dummy_importer.get_impacted_packages([ap], datetime.now()) - assert len(aff) == 1 and aff[0].version == "1.0.0" - assert len(fix) == 1 and fix[0].version == "1.1.0" - - -def test_get_impacted_packages_unmergeable(dummy_importer): - ap = mock.Mock() - ap.package = PackageURL("npm", None, "abc", None) - ap.affected_version_range = ">=1.0.0" - ap.fixed_version = None - - dummy_importer.get_published_package_versions = lambda package_url, until: ["1.0.0", "1.1.0"] - dummy_importer.resolve_package_versions = lambda **kwargs: ( - [PackageURL("npm", None, "abc", "1.0.0")], - [PackageURL("npm", None, "abc", "1.1.0")], - ) - - with mock.patch( - "vulnerabilities.importer.AffectedPackage.merge", side_effect=UnMergeablePackageError - ): - aff, fix = dummy_importer.get_impacted_packages([ap], datetime.utcnow()) - assert len(aff) == 1 - assert aff[0].version == "1.0.0" diff --git a/vulnerabilities/tests/pipelines/test_apache_httpd_importer_pipeline_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_apache_httpd_importer_pipeline_v2.py similarity index 100% rename from vulnerabilities/tests/pipelines/test_apache_httpd_importer_pipeline_v2.py rename to vulnerabilities/tests/pipelines/v2_importers/test_apache_httpd_importer_pipeline_v2.py diff --git a/vulnerabilities/tests/pipelines/test_curl_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_curl_importer_v2.py similarity index 96% rename from vulnerabilities/tests/pipelines/test_curl_importer_v2.py rename to vulnerabilities/tests/pipelines/v2_importers/test_curl_importer_v2.py index 7833d1397..6157d8bbe 100644 --- a/vulnerabilities/tests/pipelines/test_curl_importer_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_curl_importer_v2.py @@ -16,7 +16,7 @@ from univers.versions import SemverVersion from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.pipelines.v2_importers.curl_importer import CurlImporterPipeline from vulnerabilities.pipelines.v2_importers.curl_importer import get_cwe_from_curl_advisory from vulnerabilities.pipelines.v2_importers.curl_importer import parse_curl_advisory @@ -74,9 +74,9 @@ def test_collect_advisories(mock_fetch, pipeline): # Affected package check pkg = advisory.affected_packages[0] - assert isinstance(pkg, AffectedPackage) + assert isinstance(pkg, AffectedPackageV2) assert pkg.package == PackageURL(type="generic", namespace="curl.se", name="curl") - assert pkg.fixed_version == SemverVersion("8.7.0") + assert str(pkg.fixed_version_range) == "vers:generic/8.7.0" assert "8.6.0" in str(pkg.affected_version_range) # References diff --git a/vulnerabilities/tests/pipelines/test_elixir_security_v2_importer.py b/vulnerabilities/tests/pipelines/v2_importers/test_elixir_security_importer_v2.py similarity index 100% rename from vulnerabilities/tests/pipelines/test_elixir_security_v2_importer.py rename to vulnerabilities/tests/pipelines/v2_importers/test_elixir_security_importer_v2.py diff --git a/vulnerabilities/tests/pipelines/test_github_osv_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_github_osv_importer_v2.py similarity index 100% rename from vulnerabilities/tests/pipelines/test_github_osv_importer_v2.py rename to vulnerabilities/tests/pipelines/v2_importers/test_github_osv_importer_v2.py diff --git a/vulnerabilities/tests/pipelines/test_gitlab_v2_importer.py b/vulnerabilities/tests/pipelines/v2_importers/test_gitlab_importer_v2.py similarity index 98% rename from vulnerabilities/tests/pipelines/test_gitlab_v2_importer.py rename to vulnerabilities/tests/pipelines/v2_importers/test_gitlab_importer_v2.py index 195be6609..4ac781080 100644 --- a/vulnerabilities/tests/pipelines/test_gitlab_v2_importer.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_gitlab_importer_v2.py @@ -99,7 +99,7 @@ def test_collect_advisories(mock_gitlab_yaml, mock_vcs_response, mock_fetch_via_ assert advisory.summary == "Example vulnerability\nExample description" assert advisory.references_v2[0].url == "https://example.com/advisory" assert advisory.affected_packages[0].package.name == "package-name" - assert advisory.affected_packages[0].fixed_version + assert str(advisory.affected_packages[0].fixed_version_range) == "vers:pypi/2.0.0" assert advisory.weaknesses[0] == 79 diff --git a/vulnerabilities/tests/pipelines/test_istio_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_istio_importer_v2.py similarity index 66% rename from vulnerabilities/tests/pipelines/test_istio_importer_v2.py rename to vulnerabilities/tests/pipelines/v2_importers/test_istio_importer_v2.py index 162eddbbe..ba5289b1c 100644 --- a/vulnerabilities/tests/pipelines/test_istio_importer_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_istio_importer_v2.py @@ -12,14 +12,8 @@ from textwrap import dedent import pytest -from packageurl import PackageURL -from univers.version_constraint import VersionConstraint -from univers.version_range import GitHubVersionRange -from univers.version_range import GolangVersionRange -from univers.versions import SemverVersion from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import ReferenceV2 from vulnerabilities.pipelines.v2_importers.istio_importer import IstioImporterPipeline @@ -77,24 +71,11 @@ def test_istio_advisory_parsing(): url="https://istio.io/latest/news/security/ISTIO-SECURITY-2019-002/", ) - expected_versions = [ - VersionConstraint(version=SemverVersion("1.0"), comparator=">="), - VersionConstraint(version=SemverVersion("1.0.8"), comparator="<="), - VersionConstraint(version=SemverVersion("1.1"), comparator=">="), - VersionConstraint(version=SemverVersion("1.1.9"), comparator="<="), - VersionConstraint(version=SemverVersion("1.2"), comparator=">="), - VersionConstraint(version=SemverVersion("1.2.1"), comparator="<="), - ] - - expected_packages = [ - AffectedPackage( - package=PackageURL(type="golang", namespace="istio.io", name="istio"), - affected_version_range=GolangVersionRange(constraints=expected_versions), - ), - AffectedPackage( - package=PackageURL(type="github", namespace="istio", name="istio"), - affected_version_range=GitHubVersionRange(constraints=expected_versions), - ), - ] - - assert advisory.affected_packages == expected_packages + assert ( + str(advisory.affected_packages[0].affected_version_range) + == "vers:github/>=1.0.0|<=1.0.8|>=1.1.0|<=1.1.9|>=1.2.0|<=1.2.1" + ) + assert ( + str(advisory.affected_packages[1].affected_version_range) + == "vers:golang/>=1.0.0|<=1.0.8|>=1.1.0|<=1.1.9|>=1.2.0|<=1.2.1" + ) diff --git a/vulnerabilities/tests/pipelines/test_mozilla_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_mozilla_importer_v2.py similarity index 95% rename from vulnerabilities/tests/pipelines/test_mozilla_importer_v2.py rename to vulnerabilities/tests/pipelines/v2_importers/test_mozilla_importer_v2.py index 556a609ac..c468b3684 100644 --- a/vulnerabilities/tests/pipelines/test_mozilla_importer_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_mozilla_importer_v2.py @@ -14,7 +14,6 @@ from vulnerabilities.pipelines.v2_importers.mozilla_importer import get_severity_from_impact from vulnerabilities.pipelines.v2_importers.mozilla_importer import mfsa_id_from_filename from vulnerabilities.pipelines.v2_importers.mozilla_importer import parse_affected_packages -from vulnerabilities.pipelines.v2_importers.mozilla_importer import parse_md_advisory from vulnerabilities.pipelines.v2_importers.mozilla_importer import parse_yml_advisory @@ -57,7 +56,7 @@ def test_parse_affected_packages_valid(): result = list(parse_affected_packages(packages)) assert len(result) == 2 assert result[0].package.name == "firefox" - assert str(result[0].fixed_version) == "89.0.0" + assert str(result[0].fixed_version_range) == "vers:generic/89.0.0" def test_parse_affected_packages_invalid(): diff --git a/vulnerabilities/tests/pipelines/test_npm_importer_pipeline_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_npm_importer_pipeline_v2.py similarity index 96% rename from vulnerabilities/tests/pipelines/test_npm_importer_pipeline_v2.py rename to vulnerabilities/tests/pipelines/v2_importers/test_npm_importer_pipeline_v2.py index 7941c9b69..67fcc2970 100644 --- a/vulnerabilities/tests/pipelines/test_npm_importer_pipeline_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_npm_importer_pipeline_v2.py @@ -102,7 +102,7 @@ def test_to_advisory_data_full(tmp_path): pkg = adv.affected_packages[0] assert pkg.package == PackageURL(type="npm", name="mypkg") assert isinstance(pkg.affected_version_range, NpmVersionRange) - assert pkg.fixed_version == SemverVersion("1.2.4") + assert str(pkg.fixed_version_range) == "vers:npm/>=1.2.4" assert set(adv.aliases) == {"CVE-123", "CVE-124"} @@ -121,8 +121,8 @@ def test_get_affected_package_special_and_standard(): {"vulnerable_versions": "<=99.999.99999", "patched_versions": "<0.0.0"}, "pkg" ) assert isinstance(pkg.affected_version_range, NpmVersionRange) - assert pkg.fixed_version is None + assert pkg.fixed_version_range is None data2 = {"vulnerable_versions": "<=2.0.0", "patched_versions": ">=2.0.1"} pkg2 = p.get_affected_package(data2, "pkg2") assert isinstance(pkg2.affected_version_range, NpmVersionRange) - assert pkg2.fixed_version == SemverVersion("2.0.1") + assert str(pkg2.fixed_version_range) == "vers:npm/>=2.0.1" diff --git a/vulnerabilities/tests/pipelines/test_oss_fuzz_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_oss_fuzz_v2.py similarity index 100% rename from vulnerabilities/tests/pipelines/test_oss_fuzz_v2.py rename to vulnerabilities/tests/pipelines/v2_importers/test_oss_fuzz_v2.py diff --git a/vulnerabilities/tests/pipelines/test_postgresql_v2_importer.py b/vulnerabilities/tests/pipelines/v2_importers/test_postgresql_importer_v2.py similarity index 94% rename from vulnerabilities/tests/pipelines/test_postgresql_v2_importer.py rename to vulnerabilities/tests/pipelines/v2_importers/test_postgresql_importer_v2.py index c138c0746..ce3873930 100644 --- a/vulnerabilities/tests/pipelines/test_postgresql_v2_importer.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_postgresql_importer_v2.py @@ -87,20 +87,20 @@ def test_collect_advisories(mock_get, importer): assert "Description of the issue" in advisory.summary assert len(advisory.references_v2) > 0 assert advisory.affected_packages[0].package.name == "postgresql" - assert str(advisory.affected_packages[0].fixed_version) == "10.2" + assert str(advisory.affected_packages[0].fixed_version_range) == "vers:generic/10.2.0" assert advisory.affected_packages[0].affected_version_range.contains(SemverVersion("10.0.0")) assert advisory.affected_packages[0].affected_version_range.contains(SemverVersion("10.1.0")) @patch("vulnerabilities.pipelines.v2_importers.postgresql_importer.requests.get") -def test_collect_advisories_with_no_fixed_version(mock_get, importer): +def test_collect_advisories_with_no_fixed_version_range(mock_get, importer): mock_get.return_value.content = HTML_NO_FIX_ADVISORY.encode("utf-8") advisories = list(importer.collect_advisories()) assert len(advisories) == 1 advisory = advisories[0] assert advisory.advisory_id == "CVE-2023-5678" - assert advisory.affected_packages[0].fixed_version is None + assert advisory.affected_packages[0].fixed_version_range is None assert advisory.affected_packages[0].affected_version_range.contains(SemverVersion("9.5")) assert advisory.affected_packages[0].affected_version_range.contains(SemverVersion("9.6")) diff --git a/vulnerabilities/tests/pipelines/test_pypa_v2_importer_pipeline.py b/vulnerabilities/tests/pipelines/v2_importers/test_pypa_importer_pipeline_v2.py similarity index 100% rename from vulnerabilities/tests/pipelines/test_pypa_v2_importer_pipeline.py rename to vulnerabilities/tests/pipelines/v2_importers/test_pypa_importer_pipeline_v2.py diff --git a/vulnerabilities/tests/pipelines/test_pysec_v2_importer.py b/vulnerabilities/tests/pipelines/v2_importers/test_pysec_importer_v2.py similarity index 100% rename from vulnerabilities/tests/pipelines/test_pysec_v2_importer.py rename to vulnerabilities/tests/pipelines/v2_importers/test_pysec_importer_v2.py diff --git a/vulnerabilities/tests/pipelines/test_vulnrichment_v2_importer.py b/vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py similarity index 100% rename from vulnerabilities/tests/pipelines/test_vulnrichment_v2_importer.py rename to vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py diff --git a/vulnerabilities/tests/pipelines/test_xen_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_xen_importer_v2.py similarity index 100% rename from vulnerabilities/tests/pipelines/test_xen_importer_v2.py rename to vulnerabilities/tests/pipelines/v2_importers/test_xen_importer_v2.py diff --git a/vulnerabilities/tests/pipelines/test_collect_commits_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_collect_commits_v2.py similarity index 84% rename from vulnerabilities/tests/pipelines/test_collect_commits_v2.py rename to vulnerabilities/tests/pipelines/v2_improvers/test_collect_commits_v2.py index dddec9084..d79ca0eb0 100644 --- a/vulnerabilities/tests/pipelines/test_collect_commits_v2.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_collect_commits_v2.py @@ -1,11 +1,19 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. + from datetime import datetime -from unittest.mock import patch import pytest from vulnerabilities.models import AdvisoryReference from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import CodeFixV2 +from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines.v2_improvers.collect_commits import CollectFixCommitsPipeline from vulnerabilities.pipelines.v2_improvers.collect_commits import is_vcs_url @@ -59,8 +67,8 @@ def test_is_vcs_url_already_processed_true(): name="foo", version="1.0", ) - advisory.affecting_packages.add(package) - advisory.save() + impact = ImpactedPackage.objects.create(advisory=advisory) + impact.affecting_packages.add(package) CodeFixV2.objects.create( commits=["https://github.com/user/repo/commit/abc123"], advisory=advisory, @@ -87,9 +95,9 @@ def test_collect_fix_commits_pipeline_creates_entry(): reference = AdvisoryReference.objects.create( url="https://github.com/test/testpkg/commit/abc123" ) - advisory.affecting_packages.add(package) + impact = ImpactedPackage.objects.create(advisory=advisory) + impact.affecting_packages.add(package) advisory.references.add(reference) - advisory.save() pipeline = CollectFixCommitsPipeline() pipeline.collect_and_store_fix_commits() @@ -117,13 +125,11 @@ def test_collect_fix_commits_pipeline_skips_non_commit_urls(): name="otherpkg", version="2.0", ) - - advisory.affecting_packages.add(package) + impact = ImpactedPackage.objects.create(advisory=advisory) + impact.affecting_packages.add(package) reference = AdvisoryReference.objects.create(url="https://github.com/test/testpkg/issues/12") - advisory.references.add(reference) - advisory.save() pipeline = CollectFixCommitsPipeline() pipeline.collect_and_store_fix_commits() diff --git a/vulnerabilities/tests/pipelines/test_compute_package_risk_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_package_risk_v2.py similarity index 93% rename from vulnerabilities/tests/pipelines/test_compute_package_risk_v2.py rename to vulnerabilities/tests/pipelines/v2_improvers/test_compute_package_risk_v2.py index 4dbfb222a..db6ffd5d3 100644 --- a/vulnerabilities/tests/pipelines/test_compute_package_risk_v2.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_package_risk_v2.py @@ -14,6 +14,7 @@ from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import AdvisoryWeakness +from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines.v2_improvers.compute_package_risk import ComputePackageRiskPipeline from vulnerabilities.severity_systems import CVSSV3 @@ -54,8 +55,8 @@ def test_simple_risk_pipeline(): weaknesses = AdvisoryWeakness.objects.create(cwe_id=119) adv.weaknesses.add(weaknesses) - adv.affecting_packages.add(pkg) - adv.save() + impact = ImpactedPackage.objects.create(advisory=adv) + impact.affecting_packages.add(pkg) improver = ComputePackageRiskPipeline() improver.execute() diff --git a/vulnerabilities/tests/pipelines/test_compute_version_rank_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_version_rank_v2.py similarity index 100% rename from vulnerabilities/tests/pipelines/test_compute_version_rank_v2.py rename to vulnerabilities/tests/pipelines/v2_improvers/test_compute_version_rank_v2.py diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_exploitdb_v2.py similarity index 94% rename from vulnerabilities/tests/pipelines/test_enhance_with_exploitdb_v2.py rename to vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_exploitdb_v2.py index 865356158..41f96d706 100644 --- a/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb_v2.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_exploitdb_v2.py @@ -20,7 +20,7 @@ from vulnerabilities.pipelines.v2_improvers.enhance_with_exploitdb import ExploitDBImproverPipeline BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "../test_data", "exploitdb_improver/files_exploits.csv") +TEST_DATA = os.path.join(BASE_DIR, "../../test_data", "exploitdb_improver/files_exploits.csv") @pytest.mark.django_db diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_kev_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_kev_v2.py similarity index 96% rename from vulnerabilities/tests/pipelines/test_enhance_with_kev_v2.py rename to vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_kev_v2.py index bd58fa5fd..ab4df9cf2 100644 --- a/vulnerabilities/tests/pipelines/test_enhance_with_kev_v2.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_kev_v2.py @@ -21,7 +21,7 @@ from vulnerabilities.utils import load_json BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "../test_data", "kev_data.json") +TEST_DATA = os.path.join(BASE_DIR, "../../test_data", "kev_data.json") @pytest.mark.django_db diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_metasploit_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_metasploit_v2.py similarity index 94% rename from vulnerabilities/tests/pipelines/test_enhance_with_metasploit_v2.py rename to vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_metasploit_v2.py index c20437145..447dea9d3 100644 --- a/vulnerabilities/tests/pipelines/test_enhance_with_metasploit_v2.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_enhance_with_metasploit_v2.py @@ -23,7 +23,9 @@ from vulnerabilities.utils import load_json BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "../test_data", "metasploit_improver/modules_metadata_base.json") +TEST_DATA = os.path.join( + BASE_DIR, "../../test_data", "metasploit_improver/modules_metadata_base.json" +) @pytest.mark.django_db diff --git a/vulnerabilities/tests/pipelines/test_flag_ghost_packages_v2.py b/vulnerabilities/tests/pipelines/v2_improvers/test_flag_ghost_packages_v2.py similarity index 100% rename from vulnerabilities/tests/pipelines/test_flag_ghost_packages_v2.py rename to vulnerabilities/tests/pipelines/v2_improvers/test_flag_ghost_packages_v2.py diff --git a/vulnerabilities/tests/pipes/test_vulnerablecode_importer_pipeline_v2.py b/vulnerabilities/tests/pipes/test_vulnerablecode_importer_pipeline_v2.py new file mode 100644 index 000000000..aa5a2298e --- /dev/null +++ b/vulnerabilities/tests/pipes/test_vulnerablecode_importer_pipeline_v2.py @@ -0,0 +1,95 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +from datetime import datetime +from datetime import timedelta +from unittest.mock import patch + +import pytest +from packageurl import PackageURL +from univers.version_range import VersionRange + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackageV2 +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import PackageV2 +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 + + +class DummyImporter(VulnerableCodeBaseImporterPipelineV2): + pipeline_id = "dummy" + log_messages = [] + + def log(self, message, level=logging.INFO): + self.log_messages.append((level, message)) + + def collect_advisories(self): + yield from self._advisories + + def advisories_count(self): + return len(self._advisories) + + +@pytest.fixture +def dummy_advisory(): + return AdvisoryData( + summary="Test advisory", + aliases=["CVE-2025-0001"], + references_v2=[], + severities=[], + weaknesses=[], + affected_packages=[ + AffectedPackageV2( + package=PackageURL.from_string("pkg:npm/foobar"), + affected_version_range=VersionRange.from_string("vers:npm/<=1.2.3"), + fixed_version_range=VersionRange.from_string("vers:npm/1.2.4"), + ), + AffectedPackageV2( + package=PackageURL.from_string("pkg:npm/foobar"), + affected_version_range=VersionRange.from_string("vers:npm/<=3.2.3"), + fixed_version_range=VersionRange.from_string("vers:npm/3.2.4"), + ), + ], + advisory_id="ADV-123", + date_published=datetime.now() - timedelta(days=10), + url="https://example.com/advisory/1", + ) + + +@pytest.fixture +def dummy_importer(dummy_advisory): + importer = DummyImporter() + importer._advisories = [dummy_advisory] + return importer + + +@pytest.mark.django_db +def test_collect_and_store_advisories(dummy_importer): + dummy_importer.collect_and_store_advisories() + assert len(dummy_importer.log_messages) >= 2 + assert "Successfully collected" in dummy_importer.log_messages[-1][1] + assert AdvisoryV2.objects.count() == 1 + + +@pytest.mark.django_db +@patch("vulnerabilities.pipes.advisory.get_exact_purls_v2", side_effect=Exception("error")) +def test_advisory_import_atomicity_no_partial_adv_import(mock_exception, dummy_importer): + dummy_importer.collect_and_store_advisories() + assert AdvisoryV2.objects.count() == 0 + assert ImpactedPackage.objects.count() == 0 + + +@pytest.mark.django_db +def test_advisory_import_atomicity(dummy_importer): + dummy_importer.collect_and_store_advisories() + assert AdvisoryV2.objects.count() == 1 + assert ImpactedPackage.objects.count() == 2 + assert PackageV2.objects.count() == 4 diff --git a/vulnerabilities/tests/test_api_v2.py b/vulnerabilities/tests/test_api_v2.py index 432c7c10f..662499ed9 100644 --- a/vulnerabilities/tests/test_api_v2.py +++ b/vulnerabilities/tests/test_api_v2.py @@ -838,6 +838,8 @@ def test_filter_codefix_by_advisory_id_not_found(self): class AdvisoriesPackageV2Tests(APITestCase): def setUp(self): + from vulnerabilities.models import ImpactedPackage + self.advisory = AdvisoryV2.objects.create( datasource_id="ghsa", advisory_id="GHSA-1234", @@ -848,18 +850,14 @@ def setUp(self): ) self.package = PackageV2.objects.from_purl(purl="pkg:pypi/sample@1.0.0") + self.impact = ImpactedPackage.objects.create(advisory=self.advisory) + self.impact.affecting_packages.add(self.package) - self.user = ApiUser.objects.create_api_user(username="e@mail.com") - self.auth = f"Token {self.user.auth_token.key}" self.client = APIClient(enforce_csrf_checks=True) - self.client.credentials(HTTP_AUTHORIZATION=self.auth) - - self.package.affected_by_advisories.add(self.advisory) - self.package.save() def test_list_with_purl_filter(self): url = reverse("advisories-package-v2-list") - with self.assertNumQueries(18): + with self.assertNumQueries(17): response = self.client.get(url, {"purl": "pkg:pypi/sample@1.0.0"}) assert response.status_code == 200 assert "packages" in response.data["results"] @@ -868,7 +866,7 @@ def test_list_with_purl_filter(self): def test_bulk_lookup(self): url = reverse("advisories-package-v2-bulk-lookup") - with self.assertNumQueries(13): + with self.assertNumQueries(16): response = self.client.post(url, {"purls": ["pkg:pypi/sample@1.0.0"]}, format="json") assert response.status_code == 200 assert "packages" in response.data @@ -878,7 +876,7 @@ def test_bulk_lookup(self): def test_bulk_search_plain(self): url = reverse("advisories-package-v2-bulk-search") payload = {"purls": ["pkg:pypi/sample@1.0.0"], "plain_purl": True, "purl_only": False} - with self.assertNumQueries(13): + with self.assertNumQueries(16): response = self.client.post(url, payload, format="json") assert response.status_code == 200 assert "packages" in response.data @@ -887,21 +885,21 @@ def test_bulk_search_plain(self): def test_bulk_search_purl_only(self): url = reverse("advisories-package-v2-bulk-search") payload = {"purls": ["pkg:pypi/sample@1.0.0"], "plain_purl": False, "purl_only": True} - with self.assertNumQueries(13): + with self.assertNumQueries(14): response = self.client.post(url, payload, format="json") assert response.status_code == 200 assert "pkg:pypi/sample@1.0.0" in response.data def test_lookup_single_package(self): url = reverse("advisories-package-v2-lookup") - with self.assertNumQueries(11): + with self.assertNumQueries(12): response = self.client.post(url, {"purl": "pkg:pypi/sample@1.0.0"}, format="json") assert response.status_code == 200 assert any(pkg["purl"] == "pkg:pypi/sample@1.0.0" for pkg in response.data) def test_get_all_vulnerable_purls(self): url = reverse("advisories-package-v2-all") - with self.assertNumQueries(6): + with self.assertNumQueries(3): response = self.client.get(url) assert response.status_code == 200 assert "pkg:pypi/sample@1.0.0" in response.data diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 3aec1f56c..b65726a5d 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -604,7 +604,6 @@ def compute_content_id(advisory_data): # Normalize fields from vulnerabilities.importer import AdvisoryData - from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.models import Advisory if isinstance(advisory_data, Advisory): diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 07eafbdf0..f4cd99dbe 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -34,6 +34,7 @@ from vulnerabilities.forms import PackageSearchForm from vulnerabilities.forms import PipelineSchedulePackageForm from vulnerabilities.forms import VulnerabilitySearchForm +from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import PipelineRun from vulnerabilities.models import PipelineSchedule from vulnerabilities.severity_systems import EPSS @@ -186,18 +187,49 @@ class PackageV2Details(DetailView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) package = self.object + next_non_vulnerable, latest_non_vulnerable = package.get_non_vulnerable_versions() + fixed_pkg_details = {} + for impact in package.affected_in_impacts.all(): + if impact.advisory.id not in fixed_pkg_details: + fixed_pkg_details[impact.advisory.id] = [] + fixed_pkg_details[impact.advisory.id].extend( + [ + {"pkg": pkg, "affected_count": pkg.affected_in_impacts.count()} + for pkg in impact.fixed_by_packages.all() + ] + ) context["package"] = package - context["affected_by_advisories"] = package.affected_by_advisories.order_by("advisory_id") - # Ghost package should not fix any vulnerability. - context["fixing_advisories"] = ( - None if package.is_ghost else package.fixing_advisories.order_by("advisory_id") - ) + context["next_non_vulnerable"] = next_non_vulnerable + context["latest_non_vulnerable"] = latest_non_vulnerable + context["affected_by_advisories"] = { + impact.advisory for impact in package.affected_in_impacts.all() + } + context["fixing_advisories"] = { + impact.advisory for impact in package.fixed_in_impacts.all() + } + context["package_search_form"] = PackageSearchForm(self.request.GET) - context["fixed_package_details"] = package.fixed_package_details + context["fixed_package_details"] = fixed_pkg_details # context["history"] = list(package.history) return context + def get_queryset(self): + return ( + super() + .get_queryset() + .prefetch_related( + Prefetch( + "affected_in_impacts", + queryset=ImpactedPackage.objects.select_related("advisory"), + ), + Prefetch( + "fixed_in_impacts", + queryset=ImpactedPackage.objects.select_related("advisory"), + ), + ) + ) + def get_object(self, queryset=None): if queryset is None: queryset = self.get_queryset() @@ -582,37 +614,25 @@ def get_queryset(self): .get_queryset() .prefetch_related( Prefetch( - "affecting_packages", - queryset=models.PackageV2.objects.only("type", "namespace", "name", "version"), - ), - Prefetch( - "fixed_by_packages", - queryset=models.PackageV2.objects.only("type", "namespace", "name", "version"), - ), + "impacted_packages", + queryset=models.ImpactedPackage.objects.prefetch_related( + Prefetch( + "affecting_packages", + queryset=models.PackageV2.objects.only( + "type", "namespace", "name", "version" + ), + ), + Prefetch( + "fixed_by_packages", + queryset=models.PackageV2.objects.only( + "type", "namespace", "name", "version" + ), + ), + ), + ) ) ) - def get_context_data(self, **kwargs): - """ - Build context with preloaded QuerySets and minimize redundant queries. - """ - context = super().get_context_data(**kwargs) - advisory = self.object - ( - sorted_fixed_by_packages, - sorted_affected_packages, - all_affected_fixed_by_matches, - ) = advisory.aggregate_fixed_and_affected_packages() - context.update( - { - "affected_packages": sorted_affected_packages, - "fixed_by_packages": sorted_fixed_by_packages, - "all_affected_fixed_by_matches": all_affected_fixed_by_matches, - "advisory": advisory, - } - ) - return context - class PipelineScheduleListView(ListView, FormMixin): model = PipelineSchedule