From 41f44b15022e5e3af5da2e80511beeabfc7d7fca Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 15:18:05 +0530 Subject: [PATCH 01/10] Optimize vulnerabilities view Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index a2df48634..ee65d8cdc 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -32,12 +32,36 @@ from vulnerabilities.forms import VulnerabilitySearchForm from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS +from vulnerabilities.utils import get_purl_version_class from vulnerablecode import __version__ as VULNERABLECODE_VERSION from vulnerablecode.settings import env PAGE_SIZE = 20 +def purl_sort_key(purl: models.Package): + """ + Return a sort key for the built-in sorted() function when sorting a list + of Package objects. If the Package ``type`` is supported by univers, apply + the univers version class to the Package ``version``, and otherwise use the + ``version`` attribute as is. + """ + purl_version_class = get_purl_version_class(purl) + purl_sort_version = purl.version + if purl_version_class: + purl_sort_version = purl_version_class(purl.version) + return (purl.type, purl.namespace, purl.name, purl_sort_version, purl.qualifiers, purl.subpath) + + +def get_purl_version_class(purl: models.Package): + RANGE_CLASS_BY_SCHEMES["apk"] = AlpineLinuxVersionRange + purl_version_class = None + check_version_class = RANGE_CLASS_BY_SCHEMES.get(purl.type, None) + if check_version_class: + purl_version_class = check_version_class.version_class + return purl_version_class + + class PackageSearch(ListView): model = models.Package template_name = "packages.html" @@ -345,4 +369,4 @@ def get_context_data(self, **kwargs): "all_affected_fixed_by_matches": all_affected_fixed_by_matches, } ) - return context + return context \ No newline at end of file From 85c38ff09425a242fc00795a06d44a22e218f4a9 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 16:06:23 +0530 Subject: [PATCH 02/10] Fix formatting Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index ee65d8cdc..4346ce2e0 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -369,4 +369,4 @@ def get_context_data(self, **kwargs): "all_affected_fixed_by_matches": all_affected_fixed_by_matches, } ) - return context \ No newline at end of file + return context From 97870fc6d8803d627748c3bf4b57a3c66a238489 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 18:46:10 +0530 Subject: [PATCH 03/10] Fix tests Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 8 +------- vulnerabilities/utils.py | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 7bfc1ba11..39a0c16e5 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -382,20 +382,14 @@ def get_related_purls(self): return [p.package_url for p in self.packages.distinct().all()] def aggregate_fixed_and_affected_packages(self): - from vulnerabilities.utils import get_purl_version_class + from vulnerabilities.views import get_purl_version_class sorted_fixed_by_packages = self.fixed_by_packages.filter(is_ghost=False).order_by( "type", "namespace", "name", "qualifiers", "subpath" ) - if sorted_fixed_by_packages: - sorted_fixed_by_packages.first().calculate_version_rank - sorted_affected_packages = self.affected_packages.all() - if sorted_affected_packages: - sorted_affected_packages.first().calculate_version_rank - grouped_fixed_by_packages = { key: list(group) for key, group in groupby( diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 8c777610d..37a33f4fe 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -567,7 +567,7 @@ def normalize_purl(purl: Union[PackageURL, str]): def get_purl_version_class(purl): - RANGE_CLASS_BY_SCHEMES["apk"] = AlpineLinuxVersionRange + RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange purl_version_class = None check_version_class = RANGE_CLASS_BY_SCHEMES.get(purl.type, None) if check_version_class: From d96eb9af5b2fce08168699df595259ab0d383629 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 8 Jan 2025 17:51:11 +0530 Subject: [PATCH 04/10] Move severities to a different tab Signed-off-by: Tushar Goel --- .../templates/vulnerability_details.html | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index 7001c8f3b..509c515fe 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -40,6 +40,13 @@ +
  • + + + Severities ({{ severities|length }}) + + +
  • @@ -232,25 +239,16 @@ {% for ref in references %} - {% if ref.reference_id %} - {{ ref.reference_id }} - {% else %} - - {% endif %} - - {% if ref.reference_type %} - {{ ref.get_reference_type_display }} - {% else %} - - {% endif %} - - {{ ref.url }} + System + Score + Found at - {% empty %} + {% for severity in severities %} - - There are no known references. + {{ severity.scoring_system }} + {{ severity.value }} + + {{ severity.url }} {% endfor %} From cbf8e9c811fb71476a50b2547e471108e9006ad2 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 20 Jan 2025 19:35:51 +0530 Subject: [PATCH 05/10] Fix views Signed-off-by: Tushar Goel --- vulnerabilities/templates/vulnerability_details.html | 1 + 1 file changed, 1 insertion(+) diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index 509c515fe..10844abec 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -543,6 +543,7 @@

    No EPSS data available for this vulnerability.

    {% endif %} +
    From 8e4efb587b73e138c58565f7e4501dd85c9a3b11 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 11 Dec 2024 14:58:16 +0530 Subject: [PATCH 06/10] Remove form to create API Keys Signed-off-by: Tushar Goel --- vulnerabilities/templates/api_user_creation_form.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/templates/api_user_creation_form.html b/vulnerabilities/templates/api_user_creation_form.html index c7b2291f0..7714b8601 100644 --- a/vulnerabilities/templates/api_user_creation_form.html +++ b/vulnerabilities/templates/api_user_creation_form.html @@ -35,7 +35,7 @@


    -
    + {% endblock %} From 9e691ad178e5d3ed870120905790023b39091321 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 21 Jan 2025 13:51:53 +0530 Subject: [PATCH 07/10] Optimize EPSS Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 4346ce2e0..83ae281b7 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -224,6 +224,15 @@ def get_context_data(self, **kwargs): NotImplementedError, ): logging.error(f"CVSSMalformedError for {severity.scoring_elements}") + + epss_severity = vulnerability.severities.filter(scoring_system='epss').first() + epss_data = None + if epss_severity: + epss_data = { + "percentile": epss_severity.scoring_elements, + "score": epss_severity.value, + "published_at": epss_severity.published_at, + } epss_severity = vulnerability.severities.filter(scoring_system="epss").first() epss_data = None From b06f85abdefc4d5830318949d12eb18cd32e75ab Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 21 Jan 2025 17:55:04 +0530 Subject: [PATCH 08/10] Faster Weakness Lookup Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 39a0c16e5..0c2624781 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -57,6 +57,9 @@ from vulnerabilities.utils import normalize_purl from vulnerabilities.utils import purl_to_dict from vulnerablecode import __version__ as VULNERABLECODE_VERSION +from cwe2.weakness import Weakness as DBWeakness +from cwe2.mappings import xml_database_path +import xml.etree.ElementTree as ET logger = logging.getLogger(__name__) @@ -470,6 +473,21 @@ def get_severity_vectors_and_values(self): return severity_vectors, severity_values +def get_cwes(self): + """Yield CWE Weakness objects""" + for cwe_category in self.cwe_files: + cwe_category.seek(0) + reader = csv.DictReader(cwe_category) + for row in reader: + yield DBWeakness(*list(row.values())[0:-1]) + tree = ET.parse(xml_database_path) + root = tree.getroot() + for tag_num in [1, 2]: # Categories , Views + tag = root[tag_num] + for child in tag: + yield DBWeakness(*[child.attrib["ID"], child.attrib.get("Name"),None,child.attrib.get("Status"),child[0].text]) + +Database.get_cwes = get_cwes def get_cwes(self): """Yield CWE Weakness objects""" From dc2d70b64554589499ef6ccf82bd8c03e58b02b4 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 21 Jan 2025 17:57:59 +0530 Subject: [PATCH 09/10] Faster Weakness Lookup Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 15 +++++++++++---- vulnerabilities/views.py | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 0c2624781..1c43715f4 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -57,9 +57,6 @@ from vulnerabilities.utils import normalize_purl from vulnerabilities.utils import purl_to_dict from vulnerablecode import __version__ as VULNERABLECODE_VERSION -from cwe2.weakness import Weakness as DBWeakness -from cwe2.mappings import xml_database_path -import xml.etree.ElementTree as ET logger = logging.getLogger(__name__) @@ -473,6 +470,7 @@ def get_severity_vectors_and_values(self): return severity_vectors, severity_values + def get_cwes(self): """Yield CWE Weakness objects""" for cwe_category in self.cwe_files: @@ -485,7 +483,16 @@ def get_cwes(self): for tag_num in [1, 2]: # Categories , Views tag = root[tag_num] for child in tag: - yield DBWeakness(*[child.attrib["ID"], child.attrib.get("Name"),None,child.attrib.get("Status"),child[0].text]) + yield DBWeakness( + *[ + child.attrib["ID"], + child.attrib.get("Name"), + None, + child.attrib.get("Status"), + child[0].text, + ] + ) + Database.get_cwes = get_cwes diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 83ae281b7..a93a48087 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -224,8 +224,8 @@ def get_context_data(self, **kwargs): NotImplementedError, ): logging.error(f"CVSSMalformedError for {severity.scoring_elements}") - - epss_severity = vulnerability.severities.filter(scoring_system='epss').first() + + epss_severity = vulnerability.severities.filter(scoring_system="epss").first() epss_data = None if epss_severity: epss_data = { From 3c9ac0a584c3c90f739c65db63bc4b0d082852ea Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 13 Mar 2025 15:05:56 +0530 Subject: [PATCH 10/10] Adjust according to rebased changes Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 1 + .../templates/vulnerability_details.html | 35 ++++++++++--------- .../vulnerability_package_details.html | 2 +- vulnerabilities/views.py | 9 ----- 4 files changed, 20 insertions(+), 27 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 1c43715f4..fecde5c0b 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -496,6 +496,7 @@ def get_cwes(self): Database.get_cwes = get_cwes + def get_cwes(self): """Yield CWE Weakness objects""" for cwe_category in self.cwe_files: diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index 10844abec..da8466706 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -40,13 +40,6 @@ -
  • - - - Severities ({{ severities|length }}) - - -
  • @@ -239,16 +232,25 @@ {% for ref in references %}

    • - - - + {% if ref.reference_id %} + + {% else %} + + {% endif %} + + {% if ref.reference_type %} + + {% else %} + + {% endif %} + + - {% for severity in severities %} + {% empty %} - - - {% endfor %} @@ -543,7 +545,6 @@

    No EPSS data available for this vulnerability.

    {% endif %} -
    System Score Found at {{ ref.reference_id }}{{ ref.get_reference_type_display }}{{ ref.url }}
    {{ severity.scoring_system }}{{ severity.value }} - {{ severity.url }} + + There are no known references.
    @@ -613,4 +614,4 @@ } -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/vulnerabilities/templates/vulnerability_package_details.html b/vulnerabilities/templates/vulnerability_package_details.html index 21fb52192..e116e6fa9 100644 --- a/vulnerabilities/templates/vulnerability_package_details.html +++ b/vulnerabilities/templates/vulnerability_package_details.html @@ -85,4 +85,4 @@ } -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index a93a48087..4346ce2e0 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -234,15 +234,6 @@ def get_context_data(self, **kwargs): "published_at": epss_severity.published_at, } - epss_severity = vulnerability.severities.filter(scoring_system="epss").first() - epss_data = None - if epss_severity: - epss_data = { - "percentile": epss_severity.scoring_elements, - "score": epss_severity.value, - "published_at": epss_severity.published_at, - } - context.update( { "vulnerability": vulnerability,