diff --git a/dojo/finding/helper.py b/dojo/finding/helper.py index b390ff17be6..b5c504db4a1 100644 --- a/dojo/finding/helper.py +++ b/dojo/finding/helper.py @@ -1,9 +1,11 @@ import logging from contextlib import suppress from datetime import datetime +from itertools import batched from time import strftime from django.conf import settings +from django.db import transaction from django.db.models.query_utils import Q from django.db.models.signals import post_delete, pre_delete from django.db.utils import IntegrityError @@ -31,6 +33,7 @@ Endpoint, Endpoint_Status, Engagement, + FileUpload, Finding, Finding_Group, JIRA_Instance, @@ -49,7 +52,6 @@ do_false_positive_history, get_current_user, get_object_or_none, - mass_model_updater, to_str_typed, ) @@ -561,7 +563,10 @@ def finding_delete(instance, **kwargs): duplicate_cluster = instance.original_finding.all() if duplicate_cluster: - reconfigure_duplicate_cluster(instance, duplicate_cluster) + if settings.DUPLICATE_CLUSTER_CASCADE_DELETE: + duplicate_cluster.order_by("-id").delete() + else: + reconfigure_duplicate_cluster(instance, duplicate_cluster) else: logger.debug("no duplicate cluster found for finding: %d, so no need to reconfigure", instance.id) @@ -578,20 +583,6 @@ def finding_post_delete(sender, instance, **kwargs): logger.debug("finding post_delete, sender: %s instance: %s", to_str_typed(sender), to_str_typed(instance)) -def reset_duplicate_before_delete(dupe): - dupe.duplicate_finding = None - dupe.duplicate = False - - -def reset_duplicates_before_delete(qs): - mass_model_updater(Finding, qs, reset_duplicate_before_delete, fields=["duplicate", "duplicate_finding"]) - - -def set_new_original(finding, new_original): - if finding.duplicate: - finding.duplicate_finding = new_original - - # can't use model to id here due to the queryset # @dojo_async_task # @app.task @@ -602,85 +593,89 @@ def reconfigure_duplicate_cluster(original, cluster_outside): return if settings.DUPLICATE_CLUSTER_CASCADE_DELETE: - cluster_outside.order_by("-id").delete() - else: - logger.debug("reconfigure_duplicate_cluster: cluster_outside: %s", cluster_outside) - # set new original to first finding in cluster (ordered by id) - new_original = cluster_outside.order_by("id").first() - if new_original: - logger.debug("changing original of duplicate cluster %d to: %s:%s", original.id, new_original.id, new_original.title) - - new_original.duplicate = False - new_original.duplicate_finding = None - new_original.active = original.active - new_original.is_mitigated = original.is_mitigated - new_original.save_no_options() - new_original.found_by.set(original.found_by.all()) - - # if the cluster is size 1, there's only the new original left - if new_original and len(cluster_outside) > 1: - # for find in cluster_outside: - # if find != new_original: - # find.duplicate_finding = new_original - # find.save_no_options() - - mass_model_updater(Finding, cluster_outside, lambda f: set_new_original(f, new_original), fields=["duplicate_finding"]) - - -def prepare_duplicates_for_delete(test=None, engagement=None): - logger.debug("prepare duplicates for delete, test: %s, engagement: %s", test.id if test else None, engagement.id if engagement else None) - if test is None and engagement is None: - logger.warning("nothing to prepare as test and engagement are None") + # Don't delete here — the caller (async_delete_crawl_task or finding_delete) + # handles deletion of outside-scope duplicates efficiently via bulk_delete_findings. + return + logger.debug("reconfigure_duplicate_cluster: cluster_outside: %s", cluster_outside) + # set new original to first finding in cluster (ordered by id) + new_original = cluster_outside.order_by("id").first() + if new_original: + logger.debug("changing original of duplicate cluster %d to: %s:%s", original.id, new_original.id, new_original.title) + + # Use .update() to avoid triggering Finding.save() signals + Finding.objects.filter(id=new_original.id).update( + duplicate=False, + duplicate_finding=None, + active=original.active, + is_mitigated=original.is_mitigated, + ) + new_original.found_by.set(original.found_by.all()) - fix_loop_duplicates() + # Re-point remaining duplicates to the new original in a single query + cluster_outside.exclude(id=new_original.id).update(duplicate_finding=new_original) - # get all originals in the test/engagement - originals = Finding.objects.filter(original_finding__isnull=False) - if engagement: - originals = originals.filter(test__engagement=engagement) - if test: - originals = originals.filter(test=test) - # use distinct to flatten the join result - originals = originals.distinct() +def prepare_duplicates_for_delete(obj): + """ + Prepare duplicate clusters before deleting a Test, Engagement, Product, or Product_Type. - if len(originals) == 0: - logger.debug("no originals found, so no duplicates to prepare for deletion of original") - return + Resets inside-scope duplicate FKs and reconfigures outside-scope clusters + so that cascade_delete won't hit FK violations on the self-referential + duplicate_finding field. + """ + from dojo.utils import FINDING_SCOPE_FILTERS # noqa: PLC0415 circular import - # remove the link to the original from the duplicates inside the cluster so they can be safely deleted by the django framework - total = len(originals) - # logger.debug('originals: %s', [original.id for original in originals]) - for i, original in enumerate(originals): - logger.debug("%d/%d: preparing duplicate cluster for deletion of original: %d", i + 1, total, original.id) - cluster_inside = original.original_finding.all() - if engagement: - cluster_inside = cluster_inside.filter(test__engagement=engagement) + scope_field = FINDING_SCOPE_FILTERS.get(type(obj)) + if scope_field is None: + logger.warning("prepare_duplicates_for_delete: unsupported object type %s", type(obj).__name__) + return - if test: - cluster_inside = cluster_inside.filter(test=test) + logger.debug("prepare_duplicates_for_delete: %s %d", type(obj).__name__, obj.id) - if len(cluster_inside) > 0: - reset_duplicates_before_delete(cluster_inside) + # should not be needed in normal healthy instances. + # but in that case it's a cheap count query and we might as well run it to be safe + fix_loop_duplicates() - # reconfigure duplicates outside test/engagement - cluster_outside = original.original_finding.all() - if engagement: - cluster_outside = cluster_outside.exclude(test__engagement=engagement) + # Build scope as a subquery — never materialized into Python memory + scope_ids_subquery = Finding.objects.filter(**{scope_field: obj}).values_list("id", flat=True) - if test: - cluster_outside = cluster_outside.exclude(test=test) + if not scope_ids_subquery.exists(): + logger.debug("no findings in scope, nothing to prepare") + return - if len(cluster_outside) > 0: - reconfigure_duplicate_cluster(original, cluster_outside) + # Bulk-reset inside-scope duplicates: single UPDATE instead of per-original mass_model_updater. + # Clears the duplicate_finding FK so cascade_delete won't trip over dangling self-references. + inside_reset_count = Finding.objects.filter( + duplicate=True, + duplicate_finding_id__in=scope_ids_subquery, + id__in=scope_ids_subquery, + ).update(duplicate_finding=None, duplicate=False) + logger.debug("bulk-reset %d inside-scope duplicates", inside_reset_count) + + # Reconfigure outside-scope duplicates: still per-original because each cluster + # needs a new original chosen, status copied, and found_by updated. + # Chunked with prefetch_related to bound memory while avoiding N+1 queries. + originals_ids = ( + Finding.objects.filter( + id__in=scope_ids_subquery, + original_finding__in=Finding.objects.exclude(id__in=scope_ids_subquery), + ) + .distinct() + .values_list("id", flat=True) + .iterator(chunk_size=500) + ) - logger.debug("done preparing duplicate cluster for deletion of original: %d", original.id) + for chunk_ids in batched(originals_ids, 500, strict=False): + for original in Finding.objects.filter(id__in=chunk_ids).prefetch_related("original_finding"): + # Inside-scope duplicates were already unlinked by the bulk UPDATE above, + # so original_finding.all() now only contains outside-scope duplicates. + reconfigure_duplicate_cluster(original, original.original_finding.all()) @receiver(pre_delete, sender=Test) def test_pre_delete(sender, instance, **kwargs): logger.debug("test pre_delete, sender: %s instance: %s", to_str_typed(sender), to_str_typed(instance)) - prepare_duplicates_for_delete(test=instance) + prepare_duplicates_for_delete(instance) @receiver(post_delete, sender=Test) @@ -691,7 +686,7 @@ def test_post_delete(sender, instance, **kwargs): @receiver(pre_delete, sender=Engagement) def engagement_pre_delete(sender, instance, **kwargs): logger.debug("engagement pre_delete, sender: %s instance: %s", to_str_typed(sender), to_str_typed(instance)) - prepare_duplicates_for_delete(engagement=instance) + prepare_duplicates_for_delete(instance) @receiver(post_delete, sender=Engagement) @@ -699,6 +694,95 @@ def engagement_post_delete(sender, instance, **kwargs): logger.debug("engagement post_delete, sender: %s instance: %s", to_str_typed(sender), to_str_typed(instance)) +def bulk_clear_finding_m2m(finding_qs): + """ + Bulk-clear M2M through tables for a queryset of findings. + + Must be called BEFORE cascade_delete since M2M through tables + are not discovered by _meta.related_objects. + + Special handling for FileUpload: deletes via ORM so the custom + FileUpload.delete() fires and removes files from disk storage. + Tags are handled via bulk_remove_all_tags to maintain tag counts. + """ + from dojo.tag_utils import bulk_remove_all_tags # noqa: PLC0415 circular import + + finding_ids = finding_qs.values_list("id", flat=True) + + # Collect FileUpload IDs before deleting through table entries + file_ids = list( + Finding.files.through.objects.filter( + finding_id__in=finding_ids, + ).values_list("fileupload_id", flat=True), + ) + + # Collect Note IDs before deleting through table entries + note_ids = list( + Finding.notes.through.objects.filter( + finding_id__in=finding_ids, + ).values_list("notes_id", flat=True), + ) + + # Remove tags with proper count maintenance + bulk_remove_all_tags(Finding, finding_ids) + + # Auto-discover and delete remaining (non-tag) M2M through tables + for m2m_field in Finding._meta.many_to_many: + if hasattr(m2m_field, "tag_options"): + continue + through_model = m2m_field.remote_field.through + # Find the FK column that points to Finding + fk_column = None + for field in through_model._meta.get_fields(): + if hasattr(field, "related_model") and field.related_model is Finding: + fk_column = field.column + break + if fk_column: + count, _ = through_model.objects.filter( + **{f"{fk_column}__in": finding_ids}, + ).delete() + if count: + logger.debug( + "bulk_clear_finding_m2m: deleted %d rows from %s", + count, through_model._meta.db_table, + ) + + # Delete FileUpload objects via ORM so custom delete() removes files from disk + if file_ids: + for file_upload in FileUpload.objects.filter(id__in=file_ids).iterator(): + file_upload.delete() + + # Delete orphaned Notes + if note_ids: + Notes.objects.filter(id__in=note_ids).delete() + + +def bulk_delete_findings(finding_qs, chunk_size=1000): + """ + Delete findings and all related objects efficiently. Including any related object in Dojo-Pro + + Sends the pre_bulk_delete signal, clears M2M through tables (not + discovered by _meta.related_objects), then uses cascade_delete for + all FK relations via raw SQL. + Chunked with per-chunk transaction.atomic() for crash safety. + """ + from dojo.signals import pre_bulk_delete_findings # noqa: PLC0415 circular import + from dojo.utils_cascade_delete import cascade_delete # noqa: PLC0415 circular import + + pre_bulk_delete_findings.send(sender=Finding, finding_qs=finding_qs) + bulk_clear_finding_m2m(finding_qs) + finding_ids = list(finding_qs.values_list("id", flat=True).order_by("id")) + total_chunks = (len(finding_ids) + chunk_size - 1) // chunk_size + for i in range(0, len(finding_ids), chunk_size): + chunk = finding_ids[i:i + chunk_size] + with transaction.atomic(): + cascade_delete(Finding, Finding.objects.filter(id__in=chunk), skip_relations={Finding}) + logger.info( + "bulk_delete_findings: deleted chunk %d/%d (%d findings)", + i // chunk_size + 1, total_chunks, len(chunk), + ) + + def fix_loop_duplicates(): """Due to bugs in the past and even currently when under high parallel load, there can be transitive duplicates.""" """ i.e. A -> B -> C. This can lead to problems when deleting findingns, performing deduplication, etc """ @@ -709,9 +793,10 @@ def fix_loop_duplicates(): loop_count = loop_qs.count() if loop_count > 0: - deduplicationLogger.info(f"Identified {loop_count} Findings with Loops") + deduplicationLogger.warning("fix_loop_duplicates: found %d findings with duplicate loops", loop_count) # Stream IDs only in descending order to avoid loading full Finding rows for find_id in loop_qs.order_by("-id").values_list("id", flat=True).iterator(chunk_size=1000): + deduplicationLogger.warning("fix_loop_duplicates: fixing loop for finding %d", find_id) removeLoop(find_id, 50) new_originals = Finding.objects.filter(duplicate_finding__isnull=True, duplicate=True) @@ -726,6 +811,10 @@ def fix_loop_duplicates(): def removeLoop(finding_id, counter): + # NOTE: This function is recursive and does per-finding DB queries without prefetching. + # It could be optimized to load the duplicate graph as ID pairs in memory and process + # in bulk, but loops are rare (only from past bugs or high parallel load) so the + # current implementation is acceptable. # get latest status finding = Finding.objects.get(id=finding_id) real_original = finding.duplicate_finding diff --git a/dojo/models.py b/dojo/models.py index c2b8c2ee50a..f5b31c789c9 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -1647,7 +1647,7 @@ def is_ci_cd(self): def delete(self, *args, **kwargs): logger.debug("%d engagement delete", self.id) from dojo.finding import helper as finding_helper # noqa: PLC0415 circular import - finding_helper.prepare_duplicates_for_delete(engagement=self) + finding_helper.prepare_duplicates_for_delete(self) super().delete(*args, **kwargs) with suppress(Engagement.DoesNotExist, Product.DoesNotExist): # Suppressing a potential issue created from async delete removing diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index 4bf0fbc651e..c03e5635f87 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -302,7 +302,7 @@ # Initial behaviour in Defect Dojo was to delete all duplicates when an original was deleted # New behaviour is to leave the duplicates in place, but set the oldest of duplicates as new original # Set to True to revert to the old behaviour where all duplicates are deleted - DD_DUPLICATE_CLUSTER_CASCADE_DELETE=(str, False), + DD_DUPLICATE_CLUSTER_CASCADE_DELETE=(bool, False), # Enable Rate Limiting for the login page DD_RATE_LIMITER_ENABLED=(bool, False), # Examples include 5/m 100/h and more https://django-ratelimit.readthedocs.io/en/stable/rates.html#simple-rates diff --git a/dojo/signals.py b/dojo/signals.py new file mode 100644 index 00000000000..351873d34ba --- /dev/null +++ b/dojo/signals.py @@ -0,0 +1,6 @@ +from django.dispatch import Signal + +# Sent before bulk-deleting findings via cascade_delete. +# Receivers can dispatch integrator notifications, collect metrics, etc. +# Provides: finding_qs (QuerySet of findings about to be deleted) +pre_bulk_delete_findings = Signal() diff --git a/dojo/tag_utils.py b/dojo/tag_utils.py index 054dc2b0a08..cf405034be4 100644 --- a/dojo/tag_utils.py +++ b/dojo/tag_utils.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from collections.abc import Iterable from django.conf import settings @@ -8,6 +9,8 @@ from dojo.models import Product # local import to avoid circulars at import time +logger = logging.getLogger(__name__) + def bulk_add_tags_to_instances(tag_or_tags, instances, tag_field_name: str = "tags", batch_size: int | None = None) -> int: """ @@ -161,4 +164,66 @@ def bulk_add_tags_to_instances(tag_or_tags, instances, tag_field_name: str = "ta return total_created -__all__ = ["bulk_add_tags_to_instances"] +def bulk_remove_all_tags(model_class, instance_ids_qs): + """ + Remove all tags from instances identified by the given ID subquery. + + Auto-discovers all TagFields on the model, decrements tag counts correctly, + and deletes through-table rows. + Accepts a QuerySet of IDs (as a subquery) to avoid materializing large ID lists. + + Args: + model_class: The model class (e.g. Finding, Product). + instance_ids_qs: A QuerySet producing instance PKs (used as subquery). + + """ + tag_fields = [ + field for field in model_class._meta.get_fields() + if hasattr(field, "tag_options") + ] + + for tag_field in tag_fields: + + tag_model = tag_field.related_model + through_model = tag_field.remote_field.through + + # Find the FK column that points to the source model + source_field_name = None + target_field_name = None + for field in through_model._meta.get_fields(): + if hasattr(field, "remote_field") and field.remote_field: + if field.remote_field.model == model_class: + source_field_name = field.name + elif field.remote_field.model == tag_model: + target_field_name = field.name + + if not source_field_name or not target_field_name: + continue + + # Get affected tag IDs and their counts before deletion + affected_tags = ( + through_model.objects.filter(**{f"{source_field_name}__in": instance_ids_qs}) + .values(target_field_name) + .annotate(num=models.Count("id")) + ) + + # Decrement tag counts. Tag counts are not used in DefectDojo but we + # maintain them to avoid breaking tagulous's internal bookkeeping. + for entry in affected_tags: + tag_model.objects.filter(pk=entry[target_field_name]).update( + count=models.F("count") - entry["num"], + ) + + # Delete through-table rows + count, _ = through_model.objects.filter( + **{f"{source_field_name}__in": instance_ids_qs}, + ).delete() + + if count: + logger.debug( + "bulk_remove_all_tags: removed %d %s.%s through-table rows", + count, model_class.__name__, tag_field.name, + ) + + +__all__ = ["bulk_add_tags_to_instances", "bulk_remove_all_tags"] diff --git a/dojo/utils.py b/dojo/utils.py index a5d8a13ed81..edfa8156f24 100644 --- a/dojo/utils.py +++ b/dojo/utils.py @@ -6,11 +6,10 @@ import mimetypes import os import pathlib -import random import re -import time from calendar import monthrange from collections.abc import Callable +from contextlib import suppress from datetime import date, datetime, timedelta from functools import cached_property from math import pi, sqrt @@ -21,7 +20,6 @@ import cvss import vobject from amqp.exceptions import ChannelError -from auditlog.models import LogEntry from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cvss import CVSS2, CVSS3, CVSS4 @@ -30,9 +28,8 @@ from django.conf import settings from django.contrib import messages from django.contrib.auth.signals import user_logged_in, user_logged_out, user_login_failed -from django.contrib.contenttypes.models import ContentType from django.core.paginator import Paginator -from django.db import OperationalError +from django.db import transaction from django.db.models import Case, Count, F, IntegerField, Q, Sum, Value, When from django.db.models.query import QuerySet from django.db.models.signals import post_save @@ -72,6 +69,7 @@ Languages, Notifications, Product, + Product_Type, System_Settings, Test, Test_Type, @@ -2028,23 +2026,16 @@ def is_finding_groups_enabled(): return get_system_setting("enable_finding_groups") -# Mapping of object types to their related models for cascading deletes -ASYNC_DELETE_MAPPING = { - "Product_Type": [ - (Endpoint, "product__prod_type__id"), - (Finding, "test__engagement__product__prod_type__id"), - (Test, "engagement__product__prod_type__id"), - (Engagement, "product__prod_type__id"), - (Product, "prod_type__id")], - "Product": [ - (Endpoint, "product__id"), - (Finding, "test__engagement__product__id"), - (Test, "engagement__product__id"), - (Engagement, "product__id")], - "Engagement": [ - (Finding, "test__engagement__id"), - (Test, "engagement__id")], - "Test": [(Finding, "test__id")], +# Supported object types for async cascade deletion +ASYNC_DELETE_SUPPORTED_TYPES = (Product_Type, Product, Engagement, Test) + +# Finding scope filters per model type — used to build the finding queryset +# for bulk deletion before cascade_delete handles the rest. +FINDING_SCOPE_FILTERS = { + Product_Type: "test__engagement__product__prod_type", + Product: "test__engagement__product", + Engagement: "test__engagement", + Test: "test", } @@ -2056,117 +2047,79 @@ def _get_object_name(obj): @app.task -def async_delete_chunk_task(objects, **kwargs): +def async_delete_task(obj, **kwargs): """ - Module-level Celery task to delete a chunk of objects. + Delete an object and all its related objects using the SQL cascade walker. - Accepts **kwargs for _pgh_context injected by dojo_dispatch_task. - Uses PgHistoryTask base class (default) to preserve pghistory context for audit trail. - """ - max_retries = 3 - for obj in objects: - retry_count = 0 - while retry_count < max_retries: - try: - obj.delete() - break # Success, exit retry loop - except OperationalError as e: - error_msg = str(e) - if "deadlock detected" in error_msg.lower(): - retry_count += 1 - if retry_count < max_retries: - # Exponential backoff with jitter - wait_time = (2 ** retry_count) + random.uniform(0, 1) # noqa: S311 - logger.warning( - f"ASYNC_DELETE: Deadlock detected deleting {_get_object_name(obj)} {obj.pk}, " - f"retrying ({retry_count}/{max_retries}) after {wait_time:.2f}s", - ) - time.sleep(wait_time) - # Refresh object from DB before retry - obj.refresh_from_db() - else: - logger.error( - f"ASYNC_DELETE: Deadlock persisted after {max_retries} retries for {_get_object_name(obj)} {obj.pk}: {e}", - ) - raise - else: - # Not a deadlock, re-raise - raise - except AssertionError: - logger.debug("ASYNC_DELETE: object has already been deleted elsewhere. Skipping") - # The id must be None - # The object has already been deleted elsewhere - break - except LogEntry.MultipleObjectsReturned: - # Delete the log entrys first, then delete - LogEntry.objects.filter( - content_type=ContentType.objects.get_for_model(obj.__class__), - object_pk=str(obj.pk), - action=LogEntry.Action.DELETE, - ).delete() - # Now delete the object again (no retry needed for this case) - obj.delete() - break - - -@app.task -def async_delete_crawl_task(obj, model_list, **kwargs): - """ - Module-level Celery task to crawl and delete related objects. + Handles Python-level concerns (duplicates, integrators, M2M, file cleanup, + product grading) explicitly, then uses cascade_delete() for efficient + bottom-up SQL deletion of all FK-related tables. Accepts **kwargs for _pgh_context injected by dojo_dispatch_task. Uses PgHistoryTask base class (default) to preserve pghistory context for audit trail. """ - from dojo.celery_dispatch import dojo_dispatch_task # noqa: PLC0415 circular import - - logger.debug("ASYNC_DELETE: Crawling " + _get_object_name(obj) + ": " + str(obj)) - for model_info in model_list: - task_results = [] - model = model_info[0] - model_query = model_info[1] - filter_dict = {model_query: obj.id} - # Only fetch the IDs since we will make a list of IDs in the following function call - objects_to_delete = model.objects.only("id").filter(**filter_dict).distinct().order_by("id") - logger.debug("ASYNC_DELETE: Deleting " + str(len(objects_to_delete)) + " " + _get_object_name(model) + "s in chunks") - chunk_size = get_setting("ASYNC_OBEJECT_DELETE_CHUNK_SIZE") - chunks = [objects_to_delete[i:i + chunk_size] for i in range(0, len(objects_to_delete), chunk_size)] - logger.debug("ASYNC_DELETE: Split " + _get_object_name(model) + " into " + str(len(chunks)) + " chunks of " + str(chunk_size)) - for chunk in chunks: - logger.debug(f"deleting {len(chunk)} {_get_object_name(model)}") - result = dojo_dispatch_task(async_delete_chunk_task, list(chunk)) - # Collect async task results to wait for them all at once - if hasattr(result, "get"): - task_results.append(result) - # Wait for all chunk deletions to complete (they run in parallel) - for task_result in task_results: - task_result.get(timeout=300) # 5 minute timeout per chunk - # Now delete the main object after all chunks are done - result = dojo_dispatch_task(async_delete_chunk_task, [obj]) - # Wait for final deletion to complete - if hasattr(result, "get"): - result.get(timeout=300) # 5 minute timeout - logger.debug("ASYNC_DELETE: Successfully deleted " + _get_object_name(obj) + ": " + str(obj)) - + from dojo.finding.helper import ( # noqa: PLC0415 circular import + bulk_delete_findings, + prepare_duplicates_for_delete, + ) + from dojo.utils_cascade_delete import cascade_delete # noqa: PLC0415 circular import -@app.task -def async_delete_task(obj, **kwargs): - """ - Module-level Celery task to delete an object and its related objects. + logger.debug("ASYNC_DELETE: Deleting %s: %s", _get_object_name(obj), obj) + if not isinstance(obj, ASYNC_DELETE_SUPPORTED_TYPES): + logger.debug("ASYNC_DELETE: %s async delete not supported. Deleting normally: %s", _get_object_name(obj), obj) + obj.delete() + return - Accepts **kwargs for _pgh_context injected by dojo_dispatch_task. - Uses PgHistoryTask base class (default) to preserve pghistory context for audit trail. - """ - from dojo.celery_dispatch import dojo_dispatch_task # noqa: PLC0415 circular import + obj_name = _get_object_name(obj) + logger.info("ASYNC_DELETE: Starting deletion of %s: %s", obj_name, obj) + + # Capture product reference before deletion for product grading at the end + product = None + with suppress(Product.DoesNotExist, Engagement.DoesNotExist, Test.DoesNotExist): + if isinstance(obj, Engagement): + product = obj.product + elif isinstance(obj, Test): + product = obj.engagement.product + + # Step 1: Determine finding scope + scope_field = FINDING_SCOPE_FILTERS.get(type(obj)) + if scope_field: + finding_qs = Finding.objects.filter(**{scope_field: obj}) + + # Step 2: Prepare duplicate clusters (must happen before any deletion) + # When CASCADE_DELETE=True, reconfigure_duplicate_cluster skips reconfiguration — + # we handle that below by expanding scope to include outside duplicates. + prepare_duplicates_for_delete(obj) + + # Step 3: Delete outside-scope duplicates first — these point to findings + # in the main scope via duplicate_finding FK, so they must be removed before + # the originals to avoid FK violations during chunked deletion. + outside_dupes_qs = ( + Finding.objects.filter(duplicate_finding_id__in=finding_qs.values_list("id", flat=True)) + .exclude(id__in=finding_qs.values_list("id", flat=True)) + ) + chunk_size = get_setting("ASYNC_OBEJECT_DELETE_CHUNK_SIZE") + if outside_dupes_qs.exists(): + logger.info("ASYNC_DELETE: Deleting %d outside-scope duplicates first", outside_dupes_qs.count()) + bulk_delete_findings(outside_dupes_qs, chunk_size=chunk_size) + + # Step 4: Delete the main scope findings + bulk_delete_findings(finding_qs, chunk_size=chunk_size) + + # Step 5: Delete the top-level object and all remaining children (Tests, + # Engagements, Endpoints, etc.) via cascade_delete. Findings are already + # gone, so skip_relations={Finding} avoids walking empty relations. + pk_query = type(obj).objects.filter(pk=obj.pk) + with transaction.atomic(): + cascade_delete(type(obj), pk_query, skip_relations={Finding}) + + # Step 6: Recalculate product grade once (not per-object) + # The custom delete() methods on Finding/Test/Engagement each call + # perform_product_grading — cascade_delete bypasses custom delete(). + if product: + perform_product_grading(product) - logger.debug("ASYNC_DELETE: Deleting " + _get_object_name(obj) + ": " + str(obj)) - model_list = ASYNC_DELETE_MAPPING.get(_get_object_name(obj)) - if model_list: - # The object to be deleted was found in the object list - dojo_dispatch_task(async_delete_crawl_task, obj, model_list) - else: - # The object is not supported in async delete, delete normally - logger.debug("ASYNC_DELETE: " + _get_object_name(obj) + " async delete not supported. Deleteing normally: " + str(obj)) - obj.delete() + logger.info("ASYNC_DELETE: Successfully deleted %s: %s", obj_name, obj) class async_delete: @@ -2182,10 +2135,6 @@ class async_delete: which properly handles user context injection and pghistory context. """ - def __init__(self, *args, **kwargs): - # Keep mapping reference for backwards compatibility - self.mapping = ASYNC_DELETE_MAPPING - def delete(self, obj, **kwargs): """ Entry point to delete an object asynchronously. @@ -2197,18 +2146,10 @@ def delete(self, obj, **kwargs): dojo_dispatch_task(async_delete_task, obj, **kwargs) - # Keep helper methods for backwards compatibility and potential direct use @staticmethod def get_object_name(obj): return _get_object_name(obj) - @staticmethod - def chunk_list(model, full_list): - chunk_size = get_setting("ASYNC_OBEJECT_DELETE_CHUNK_SIZE") - chunk_list = [full_list[i:i + chunk_size] for i in range(0, len(full_list), chunk_size)] - logger.debug("ASYNC_DELETE: Split " + _get_object_name(model) + " into " + str(len(chunk_list)) + " chunks of " + str(chunk_size)) - return chunk_list - @receiver(user_logged_in) def log_user_login(sender, request, user, **kwargs): diff --git a/dojo/utils_cascade_delete.py b/dojo/utils_cascade_delete.py new file mode 100644 index 00000000000..ccbb5eaa810 --- /dev/null +++ b/dojo/utils_cascade_delete.py @@ -0,0 +1,178 @@ +""" +Efficient cascade delete utility for Django models. + +Uses compiled SQL (via SQLDeleteCompiler/SQLUpdateCompiler) to perform cascade +DELETE and SET_NULL operations by walking model._meta.related_objects recursively. +This bypasses Django's Collector and per-object signal overhead. + +Based on: https://dev.to/redhap/efficient-django-delete-cascade-43i5 +""" + +import logging + +from django.db import models, transaction +from django.db.models.sql.compiler import SQLDeleteCompiler + +logger = logging.getLogger(__name__) + + +def get_delete_sql(query): + """Compile a DELETE SQL statement from a QuerySet.""" + return SQLDeleteCompiler( + query.query, transaction.get_connection(), query.db, + ).as_sql() + + +def get_update_sql(query, **updatespec): + """Compile an UPDATE SQL statement from a QuerySet with the given column values.""" + if not query.query.can_filter(): + msg = "Cannot filter this query" + raise ValueError(msg) + query.for_write = True + q = query.query.chain(models.sql.UpdateQuery) + q.add_update_values(updatespec) + q._annotations = None + return q.get_compiler(query.db).as_sql() + + +def execute_compiled_sql(sql, params=None): + """Execute compiled SQL directly via connection.cursor().""" + with transaction.get_connection().cursor() as cur: + cur.execute(sql, params or None) + return cur.rowcount + + +def execute_delete_sql(query): + """Compile and execute a DELETE statement from a QuerySet.""" + return execute_compiled_sql(*get_delete_sql(query)) + + +def execute_update_sql(query, **updatespec): + """Compile and execute an UPDATE statement from a QuerySet.""" + return execute_compiled_sql(*get_update_sql(query, **updatespec)) + + +def cascade_delete(from_model, instance_pk_query, skip_relations=None, base_model=None, level=0): + """ + Recursively walk Django model relations and execute compiled SQL + to perform cascade DELETE / SET_NULL without the Collector. + + Walks from_model._meta.related_objects to discover all FK relations, + recurses into CASCADE children first (bottom-up), then deletes at the + current level. No query execution until recursion unwinds. + + Includes any related object in Dojo-Pro + + Args: + from_model: The model class to delete from. + instance_pk_query: QuerySet selecting the records to delete. + skip_relations: Set of model classes to skip (e.g. self-referential FKs). + base_model: Root model class (set automatically on first call). + level: Recursion depth (for logging only). + + Returns: + Number of records deleted at this level. + + """ + if skip_relations is None: + skip_relations = set() + if base_model is None: + base_model = from_model + + instance_pk_query = instance_pk_query.values_list("pk").order_by() + + logger.debug( + "cascade_delete level %d for %s: checking relations of %s", + level, base_model.__name__, from_model.__name__, + ) + + for relation in from_model._meta.related_objects: + related_model = relation.related_model + if related_model in skip_relations: + logger.debug("cascade_delete: skipping %s", related_model.__name__) + continue + + on_delete = relation.on_delete + if on_delete is None: + logger.debug( + "cascade_delete: no on_delete for %s -> %s, skipping", + from_model.__name__, related_model.__name__, + ) + continue + + on_delete_name = on_delete.__name__ + fk_column = relation.remote_field.column + filterspec = {f"{fk_column}__in": models.Subquery(instance_pk_query)} + + if on_delete_name == "SET_NULL": + count = execute_update_sql( + related_model.objects.filter(**filterspec), + **{fk_column: None}, + ) + logger.debug( + "cascade_delete: SET NULL on %d %s records", + count, related_model.__name__, + ) + + elif on_delete_name == "CASCADE": + related_pk_query = related_model.objects.filter(**filterspec).values_list( + related_model._meta.pk.name, + ) + # Recurse into children first (bottom-up deletion) + cascade_delete( + related_model, related_pk_query, + skip_relations=skip_relations, + base_model=base_model, + level=level + 1, + ) + + elif on_delete_name == "DO_NOTHING": + logger.debug( + "cascade_delete: DO_NOTHING for %s, skipping", + related_model.__name__, + ) + + else: + logger.warning( + "cascade_delete: unhandled on_delete=%s for %s -> %s, skipping", + on_delete_name, from_model.__name__, related_model.__name__, + ) + + # Clear M2M through tables before deleting (not discovered by _meta.related_objects). + # Tag fields are handled via bulk_remove_all_tags to maintain tag counts correctly. + from dojo.tag_utils import bulk_remove_all_tags # noqa: PLC0415 circular import + + bulk_remove_all_tags(from_model, instance_pk_query) + + for m2m_field in from_model._meta.many_to_many: + # Skip tag fields — already handled above + if hasattr(m2m_field, "tag_options"): + continue + through_model = m2m_field.remote_field.through + fk_column = None + for field in through_model._meta.get_fields(): + if hasattr(field, "related_model") and field.related_model is from_model: + fk_column = field.column + break + if fk_column: + filterspec_m2m = {f"{fk_column}__in": models.Subquery(instance_pk_query)} + m2m_count = execute_delete_sql(through_model.objects.filter(**filterspec_m2m)) + if m2m_count: + logger.debug( + "cascade_delete: cleared %d rows from M2M %s", + m2m_count, through_model._meta.db_table, + ) + + # After all children and M2M are deleted, delete records at this level + if level == 0: + del_query = instance_pk_query + else: + filterspec = {f"{from_model._meta.pk.name}__in": models.Subquery(instance_pk_query)} + del_query = from_model.objects.filter(**filterspec) + + count = execute_delete_sql(del_query) + logger.debug( + "cascade_delete level %d: deleted %d %s records", + level, count, from_model.__name__, + ) + return count diff --git a/unittests/test_async_delete.py b/unittests/test_async_delete.py index b8320d24707..1554432ef50 100644 --- a/unittests/test_async_delete.py +++ b/unittests/test_async_delete.py @@ -296,18 +296,3 @@ def test_async_delete_helper_methods(self): "Product", "get_object_name should work with model class", ) - - def test_async_delete_mapping_preserved(self): - """ - Test that the mapping attribute is preserved on async_delete instances. - - This ensures backwards compatibility for code that might access the mapping. - """ - async_del = async_delete() - - # Verify mapping exists and has expected keys - self.assertIsNotNone(async_del.mapping) - self.assertIn("Product", async_del.mapping) - self.assertIn("Product_Type", async_del.mapping) - self.assertIn("Engagement", async_del.mapping) - self.assertIn("Test", async_del.mapping) diff --git a/unittests/test_prepare_duplicates_for_delete.py b/unittests/test_prepare_duplicates_for_delete.py new file mode 100644 index 00000000000..de786c50ad6 --- /dev/null +++ b/unittests/test_prepare_duplicates_for_delete.py @@ -0,0 +1,411 @@ +""" +Tests for prepare_duplicates_for_delete() in dojo.finding.helper. + +These tests verify that duplicate clusters are properly handled before +Test/Engagement deletion: inside-scope duplicates get their FK cleared, +outside-scope duplicates get a new original chosen. +""" + +import logging + +from crum import impersonate +from django.test.utils import override_settings +from django.utils import timezone + +from dojo.finding.helper import prepare_duplicates_for_delete +from dojo.models import Engagement, Finding, Product, Product_Type, Test, Test_Type, User, UserContactInfo + +from .dojo_test_case import DojoTestCase + +logger = logging.getLogger(__name__) + + +@override_settings(DUPLICATE_CLUSTER_CASCADE_DELETE=False) +class TestPrepareDuplicatesForDelete(DojoTestCase): + + """Tests for prepare_duplicates_for_delete().""" + + def setUp(self): + super().setUp() + + self.testuser = User.objects.create( + username="test_prepare_dupes_user", + is_staff=True, + is_superuser=True, + ) + UserContactInfo.objects.create(user=self.testuser, block_execution=True) + + self.system_settings(enable_deduplication=False) + self.system_settings(enable_product_grade=False) + + self.product_type = Product_Type.objects.create(name="Test PT for Prepare Dupes") + self.product = Product.objects.create( + name="Test Product", + description="Test", + prod_type=self.product_type, + ) + self.test_type = Test_Type.objects.get_or_create(name="Manual Test")[0] + + # Engagement 1 with Test 1 and Test 2 + self.engagement1 = Engagement.objects.create( + name="Engagement 1", + product=self.product, + target_start=timezone.now(), + target_end=timezone.now(), + ) + self.test1 = Test.objects.create( + engagement=self.engagement1, + test_type=self.test_type, + target_start=timezone.now(), + target_end=timezone.now(), + ) + self.test2 = Test.objects.create( + engagement=self.engagement1, + test_type=self.test_type, + target_start=timezone.now(), + target_end=timezone.now(), + ) + + # Engagement 2 with Test 3 (for cross-engagement tests) + self.engagement2 = Engagement.objects.create( + name="Engagement 2", + product=self.product, + target_start=timezone.now(), + target_end=timezone.now(), + ) + self.test3 = Test.objects.create( + engagement=self.engagement2, + test_type=self.test_type, + target_start=timezone.now(), + target_end=timezone.now(), + ) + + def _create_finding(self, test, title="Finding"): + return Finding.objects.create( + test=test, + title=title, + severity="High", + description="Test", + mitigation="Test", + impact="Test", + reporter=self.testuser, + ) + + def _make_duplicate(self, duplicate, original): + """Set duplicate relationship directly, bypassing set_duplicate safeguards.""" + duplicate.duplicate = True + duplicate.duplicate_finding = original + duplicate.active = False + super(Finding, duplicate).save(skip_validation=True) + + def test_no_duplicates(self): + """Deleting a test with no duplicate relationships is a no-op.""" + f1 = self._create_finding(self.test1, "F1") + f2 = self._create_finding(self.test1, "F2") + + with impersonate(self.testuser): + prepare_duplicates_for_delete(self.test1) + + f1.refresh_from_db() + f2.refresh_from_db() + self.assertFalse(f1.duplicate) + self.assertFalse(f2.duplicate) + self.assertIsNone(f1.duplicate_finding) + self.assertIsNone(f2.duplicate_finding) + + def test_inside_scope_duplicates_reset(self): + """Duplicates inside the deletion scope have their duplicate FK cleared.""" + original = self._create_finding(self.test1, "Original") + dupe = self._create_finding(self.test1, "Duplicate") + self._make_duplicate(dupe, original) + + with impersonate(self.testuser): + prepare_duplicates_for_delete(self.test1) + + dupe.refresh_from_db() + self.assertIsNone(dupe.duplicate_finding) + self.assertFalse(dupe.duplicate) + + def test_outside_scope_duplicates_get_new_original(self): + """Duplicates outside the deletion scope get a new original.""" + original = self._create_finding(self.test1, "Original") + original.active = True + original.is_mitigated = False + super(Finding, original).save(skip_validation=True) + + outside_dupe = self._create_finding(self.test2, "Outside Dupe") + self._make_duplicate(outside_dupe, original) + + with impersonate(self.testuser): + prepare_duplicates_for_delete(self.test1) + + outside_dupe.refresh_from_db() + # Outside dupe becomes the new original + self.assertFalse(outside_dupe.duplicate) + self.assertIsNone(outside_dupe.duplicate_finding) + # Inherits active/mitigated status from old original + self.assertTrue(outside_dupe.active) + self.assertFalse(outside_dupe.is_mitigated) + + def test_outside_scope_cluster_repointed(self): + """Multiple outside-scope duplicates are re-pointed to the new original.""" + original = self._create_finding(self.test1, "Original") + dupe_b = self._create_finding(self.test2, "Dupe B") + dupe_c = self._create_finding(self.test2, "Dupe C") + dupe_d = self._create_finding(self.test2, "Dupe D") + self._make_duplicate(dupe_b, original) + self._make_duplicate(dupe_c, original) + self._make_duplicate(dupe_d, original) + + with impersonate(self.testuser): + prepare_duplicates_for_delete(self.test1) + + dupe_b.refresh_from_db() + dupe_c.refresh_from_db() + dupe_d.refresh_from_db() + + # Lowest ID becomes new original + new_original = dupe_b + self.assertFalse(new_original.duplicate) + self.assertIsNone(new_original.duplicate_finding) + + # Others re-pointed to new original + self.assertTrue(dupe_c.duplicate) + self.assertEqual(dupe_c.duplicate_finding_id, new_original.id) + self.assertTrue(dupe_d.duplicate) + self.assertEqual(dupe_d.duplicate_finding_id, new_original.id) + + def test_engagement_scope_inside_reset(self): + """Inside-scope reset works at engagement level.""" + original = self._create_finding(self.test1, "Original") + dupe = self._create_finding(self.test2, "Dupe in same engagement") + self._make_duplicate(dupe, original) + + with impersonate(self.testuser): + prepare_duplicates_for_delete(self.engagement1) + + dupe.refresh_from_db() + self.assertIsNone(dupe.duplicate_finding) + self.assertFalse(dupe.duplicate) + + def test_engagement_scope_outside_reconfigure(self): + """Outside-scope reconfiguration works at engagement level.""" + original = self._create_finding(self.test1, "Original in Eng 1") + outside_dupe = self._create_finding(self.test3, "Dupe in Eng 2") + self._make_duplicate(outside_dupe, original) + + with impersonate(self.testuser): + prepare_duplicates_for_delete(self.engagement1) + + outside_dupe.refresh_from_db() + self.assertFalse(outside_dupe.duplicate) + self.assertIsNone(outside_dupe.duplicate_finding) + + def test_mixed_inside_and_outside_duplicates(self): + """Original with duplicates both inside and outside scope.""" + original = self._create_finding(self.test1, "Original") + inside_dupe = self._create_finding(self.test1, "Inside Dupe") + outside_dupe = self._create_finding(self.test2, "Outside Dupe") + self._make_duplicate(inside_dupe, original) + self._make_duplicate(outside_dupe, original) + + with impersonate(self.testuser): + prepare_duplicates_for_delete(self.test1) + + inside_dupe.refresh_from_db() + outside_dupe.refresh_from_db() + + # Inside dupe: FK cleared + self.assertIsNone(inside_dupe.duplicate_finding) + self.assertFalse(inside_dupe.duplicate) + + # Outside dupe: becomes new original + self.assertFalse(outside_dupe.duplicate) + self.assertIsNone(outside_dupe.duplicate_finding) + + @override_settings(DUPLICATE_CLUSTER_CASCADE_DELETE=True) + def test_cascade_delete_skips_outside_reconfigure(self): + """ + When DUPLICATE_CLUSTER_CASCADE_DELETE=True, outside duplicates are left untouched. + + The caller (async_delete_crawl_task) handles deletion of outside-scope + duplicates separately via bulk_delete_findings. + """ + original = self._create_finding(self.test1, "Original") + outside_dupe = self._create_finding(self.test2, "Outside Dupe") + self._make_duplicate(outside_dupe, original) + + with impersonate(self.testuser): + prepare_duplicates_for_delete(self.test1) + + outside_dupe.refresh_from_db() + # Outside dupe is still a duplicate — not reconfigured or deleted + self.assertTrue(outside_dupe.duplicate) + self.assertEqual(outside_dupe.duplicate_finding_id, original.id) + + def test_multiple_originals(self): + """Multiple originals in the same test each get their clusters handled.""" + original_a = self._create_finding(self.test1, "Original A") + original_b = self._create_finding(self.test1, "Original B") + dupe_of_a = self._create_finding(self.test2, "Dupe of A") + dupe_of_b = self._create_finding(self.test2, "Dupe of B") + self._make_duplicate(dupe_of_a, original_a) + self._make_duplicate(dupe_of_b, original_b) + + with impersonate(self.testuser): + prepare_duplicates_for_delete(self.test1) + + dupe_of_a.refresh_from_db() + dupe_of_b.refresh_from_db() + + # Both become new originals + self.assertFalse(dupe_of_a.duplicate) + self.assertIsNone(dupe_of_a.duplicate_finding) + self.assertFalse(dupe_of_b.duplicate) + self.assertIsNone(dupe_of_b.duplicate_finding) + + def test_original_status_copied_to_new_original(self): + """New original inherits active/is_mitigated status from deleted original.""" + original = self._create_finding(self.test1, "Original") + original.active = False + original.is_mitigated = True + super(Finding, original).save(skip_validation=True) + + outside_dupe = self._create_finding(self.test2, "Outside Dupe") + self._make_duplicate(outside_dupe, original) + + with impersonate(self.testuser): + prepare_duplicates_for_delete(self.test1) + + outside_dupe.refresh_from_db() + self.assertFalse(outside_dupe.duplicate) + self.assertFalse(outside_dupe.active) + self.assertTrue(outside_dupe.is_mitigated) + + def test_found_by_copied_to_new_original(self): + """New original inherits found_by from deleted original.""" + original = self._create_finding(self.test1, "Original") + test_type_2 = Test_Type.objects.get_or_create(name="ZAP Scan")[0] + original.found_by.add(self.test_type) + original.found_by.add(test_type_2) + + outside_dupe = self._create_finding(self.test2, "Outside Dupe") + self._make_duplicate(outside_dupe, original) + + with impersonate(self.testuser): + prepare_duplicates_for_delete(self.test1) + + outside_dupe.refresh_from_db() + found_by_ids = set(outside_dupe.found_by.values_list("id", flat=True)) + self.assertIn(self.test_type.id, found_by_ids) + self.assertIn(test_type_2.id, found_by_ids) + + def test_delete_finding_reconfigures_cross_engagement_duplicate(self): + """ + Deleting an original finding makes its cross-engagement duplicate standalone. + + Setup: product with eng A (finding A, original) and eng B (finding B, duplicate of A). + Action: delete finding A. + Expected: finding B becomes a standalone finding (not duplicate, active, no duplicate_finding). + """ + finding_a = self._create_finding(self.test1, "Original A") + finding_a.active = True + finding_a.is_mitigated = False + super(Finding, finding_a).save(skip_validation=True) + + finding_b = self._create_finding(self.test3, "Duplicate B") + self._make_duplicate(finding_b, finding_a) + + # Verify setup + finding_b.refresh_from_db() + self.assertTrue(finding_b.duplicate) + self.assertEqual(finding_b.duplicate_finding_id, finding_a.id) + + # Delete finding A — triggers finding_delete signal -> reconfigure_duplicate_cluster + with impersonate(self.testuser): + finding_a.delete() + + # Finding B should now be standalone + finding_b.refresh_from_db() + self.assertFalse(finding_b.duplicate) + self.assertIsNone(finding_b.duplicate_finding) + self.assertTrue(finding_b.active) + self.assertFalse(finding_b.is_mitigated) + + def test_delete_product_with_cross_engagement_duplicates(self): + """ + Deleting a product with cross-engagement duplicates succeeds without FK violations. + + Setup: product with eng A (finding A, original) and eng B (finding B, duplicate of A). + Action: delete the entire product via async_delete_crawl_task. + Expected: product and all findings are deleted without errors. + """ + from dojo.utils import async_delete # noqa: PLC0415 + + finding_a = self._create_finding(self.test1, "Original A") + finding_a.active = True + finding_a.is_mitigated = False + super(Finding, finding_a).save(skip_validation=True) + + finding_b = self._create_finding(self.test3, "Duplicate B") + self._make_duplicate(finding_b, finding_a) + + product_id = self.product.id + finding_a_id = finding_a.id + finding_b_id = finding_b.id + + with impersonate(self.testuser): + async_del = async_delete() + async_del.delete(self.product) + + # Everything should be gone + self.assertFalse(Product.objects.filter(id=product_id).exists()) + self.assertFalse(Finding.objects.filter(id=finding_a_id).exists()) + self.assertFalse(Finding.objects.filter(id=finding_b_id).exists()) + + def test_delete_product_with_tags(self): + """ + Deleting a product with tags on product and findings succeeds + and correctly decrements tag counts. + """ + from dojo.utils import async_delete # noqa: PLC0415 + + # Add tags to product and findings + self.product.tags = "product-tag, shared-tag" + self.product.save() + + finding_a = self._create_finding(self.test1, "Tagged Finding A") + finding_a.tags = "finding-tag, shared-tag" + super(Finding, finding_a).save(skip_validation=True) + + finding_b = self._create_finding(self.test3, "Tagged Finding B") + finding_b.tags = "finding-tag" + super(Finding, finding_b).save(skip_validation=True) + + product_id = self.product.id + finding_a_id = finding_a.id + finding_b_id = finding_b.id + + # Get tag models to check counts after deletion + # Product and Finding have separate tag models in tagulous + product_tag_model = Product._meta.get_field("tags").related_model + finding_tag_model = Finding._meta.get_field("tags").related_model + product_shared_tag = product_tag_model.objects.get(name="shared-tag") + finding_shared_tag = finding_tag_model.objects.get(name="shared-tag") + + with impersonate(self.testuser): + async_del = async_delete() + async_del.delete(self.product) + + # Everything should be gone + self.assertFalse(Product.objects.filter(id=product_id).exists()) + self.assertFalse(Finding.objects.filter(id=finding_a_id).exists()) + self.assertFalse(Finding.objects.filter(id=finding_b_id).exists()) + + # Tag counts should be decremented to 0 (all referencing objects deleted). + # Tag counts are not used in DefectDojo, but we still verify them to ensure + # our bulk removal method doesn't break tagulous's internal bookkeeping. + product_shared_tag.refresh_from_db() + self.assertEqual(product_shared_tag.count, 0) + finding_shared_tag.refresh_from_db() + self.assertEqual(finding_shared_tag.count, 0)