From 14e17aa8b32847859469094456d98dbeb351bf88 Mon Sep 17 00:00:00 2001 From: sudip-khanal Date: Wed, 22 Apr 2026 22:03:31 +0545 Subject: [PATCH 01/28] feat(event): make event source enum - Add Enum for event sources. - Add source field in place of auto generated source field on event model. - Migrate existing event records to enum type. - Update related serializers and tests. --- api/admin.py | 46 ++----------- api/drf_views.py | 34 +++++++++- api/enums.py | 1 + api/factories/event.py | 2 +- api/filter_set.py | 7 +- api/management/commands/create_events.py | 3 +- api/management/commands/index_and_notify.py | 2 +- api/management/commands/ingest_gdacs.py | 3 +- api/management/commands/ingest_mdb.py | 3 +- api/management/commands/ingest_who.py | 15 +++-- ..._generated_source_event_source_and_more.py | 64 +++++++++++++++++++ api/models.py | 34 ++++++++-- api/serializers.py | 5 +- deployments/tests.py | 2 + 14 files changed, 156 insertions(+), 65 deletions(-) create mode 100644 api/migrations/0231_remove_event_auto_generated_source_event_source_and_more.py diff --git a/api/admin.py b/api/admin.py index 9dab94449..628ac97ff 100644 --- a/api/admin.py +++ b/api/admin.py @@ -26,7 +26,6 @@ import api.models as models from api.admin_classes import RegionRestrictedAdmin -from api.event_sources import SOURCES from api.management.commands.index_and_notify import Command as Notify from lang.admin import TranslationAdmin, TranslationInlineModelAdmin from notifications.models import RecordType, SubscriptionType @@ -158,38 +157,6 @@ def queryset(self, request, queryset): return queryset.filter(is_featured=False) -class EventSourceFilter(admin.SimpleListFilter): - title = _("source") - parameter_name = "event_source" - - def lookups(self, request, model_admin): - return ( - ("input", _("Manual input")), - ("gdacs", _("GDACs scraper")), - ("who", _("WHO scraper")), - ("report_ingest", _("Field report ingest")), - ("report_admin", _("Field report admin")), - ("appeal_admin", _("Appeals admin")), - ("unknown", _("Unknown automated")), - ) - - def queryset(self, request, queryset): - if self.value() == "input": - return queryset.filter(auto_generated=False) - elif self.value() == "gdacs": - return queryset.filter(auto_generated_source=SOURCES["gdacs"]) - elif self.value() == "who": - return queryset.filter(auto_generated_source__startswith="www.who.int") - elif self.value() == "report_ingest": - return queryset.filter(auto_generated_source=SOURCES["report_ingest"]) - elif self.value() == "report_admin": - return queryset.filter(auto_generated_source=SOURCES["report_admin"]) - elif self.value() == "appeal_admin": - return queryset.filter(auto_generated_source=SOURCES["appeal_admin"]) - elif self.value() == "unknown": - return queryset.filter(auto_generated=True).filter(auto_generated_source__isnull=True) - - class DisasterTypeAdmin(CompareVersionAdmin, TranslationAdmin, admin.ModelAdmin): search_fields = ("name",) @@ -246,9 +213,9 @@ def level_updated_at(self, obj): "cc_status", "glide", "auto_generated", - "auto_generated_source", + "source", ) - list_filter = [IsFeaturedFilter, EventSourceFilter] + list_filter = [IsFeaturedFilter, "source"] actions = ["create_field_reports"] search_fields = ( "name", @@ -369,7 +336,7 @@ def changeform_view(self, request, object_id=None, form_url="", extra_context=No self.readonly_fields = ( "appeals", "field_reports", - "auto_generated_source", + "source", "parent_event", "created_at", "updated_at", @@ -378,9 +345,10 @@ def changeform_view(self, request, object_id=None, form_url="", extra_context=No self.readonly_fields = ( "appeals", "field_reports", - "auto_generated_source", + "source", "created_at", "updated_at", + "who_guid", ) # Set severity level from GET parameter @@ -651,7 +619,7 @@ def create_events(self, request, queryset): dtype=getattr(report, "dtype"), disaster_start_date=getattr(report, "created_at"), auto_generated=True, - auto_generated_source=SOURCES["report_admin"], + source=models.Event.EventSource.REPORT_ADMIN, ) if getattr(report, "countries").exists(): for country in report.countries.all(): @@ -760,7 +728,7 @@ def create_events(self, request, queryset): dtype=getattr(appeal, "dtype"), disaster_start_date=getattr(appeal, "start_date"), auto_generated=True, - auto_generated_source=SOURCES["appeal_admin"], + source=models.Event.EventSource.APPEAL_ADMIN, ) if appeal.country is not None: event.countries.add(appeal.country) diff --git a/api/drf_views.py b/api/drf_views.py index f6d7eeea6..31d4448bd 100644 --- a/api/drf_views.py +++ b/api/drf_views.py @@ -751,7 +751,7 @@ def get_queryset(self, *args, **kwargs): if self.action == "response_activity_events": return ( qset.filter(parent_event__isnull=True) - .filter(Q(auto_generated=False) | Q(auto_generated_source="New field report")) + .filter(Q(auto_generated=False) | Q(source=Event.EventSource.NEW_REPORT)) .select_related("dtype") ) return ( @@ -1349,7 +1349,7 @@ class SupportedActivityViewset(viewsets.ReadOnlyModelViewSet): # summary=report.description or "", # disaster_start_date=report.start_date, # auto_generated=True, -# auto_generated_source=SOURCES["new_report"], +# source=Event.EventSource.NEW_REPORT, # visibility=report.visibility, # **{TRANSLATOR_ORIGINAL_LANGUAGE_FIELD_NAME: django_get_language()}, # ) @@ -1555,3 +1555,33 @@ class CountrySupportingPartnerViewSet(viewsets.ModelViewSet): def get_queryset(self): return CountrySupportingPartner.objects.select_related("country") + + +class EmergencyViewset(viewsets.ReadOnlyModelViewSet): + queryset = Event.objects.all() + lookup_field = "id" + serializer_class = DetailEmergencySerializer + filterset_class = EventFilter + + def get_queryset(self): + return ( + super() + .get_queryset() + .select_related( + "dtype", + "parent_event", + ) + .prefetch_related( + "regions", + "countries", + "countries_for_preview", + Prefetch("key_figures", queryset=KeyFigure.objects.all()), + Prefetch("contacts", queryset=EventContact.objects.all()), + ) + .annotate( + latest_field_report_id=Subquery( + FieldReport.objects.filter(event=OuterRef("pk")).order_by("-created_at").values("id")[:1] + ), + appeal_id=Subquery(Appeal.objects.filter(event=OuterRef("pk")).order_by("-created_at").values("id")[:1]), + ) + ) diff --git a/api/enums.py b/api/enums.py index 83e81850e..115855b4d 100644 --- a/api/enums.py +++ b/api/enums.py @@ -20,4 +20,5 @@ "action_category": models.ActionCategory, "profile_org_types": models.Profile.OrgTypes, "supporting_type": models.CountrySupportingPartner.SupportingPartnerType, + "event_source": models.Event.EventSource, } diff --git a/api/factories/event.py b/api/factories/event.py index f3656a2e6..465433b0d 100644 --- a/api/factories/event.py +++ b/api/factories/event.py @@ -72,7 +72,7 @@ def regions(self, create, extracted, **kwargs): previous_update = fuzzy.FuzzyDateTime(datetime.datetime(2008, 1, 1, tzinfo=pytz.utc)) auto_generated = fuzzy.FuzzyChoice([True, False]) - auto_generated_source = fuzzy.FuzzyText(length=50) + source = fuzzy.FuzzyChoice(Event.EventSource.values) is_featured = fuzzy.FuzzyChoice([True, False]) is_featured_region = fuzzy.FuzzyChoice([True, False]) diff --git a/api/filter_set.py b/api/filter_set.py index 625c4da7a..0f0d1ff41 100644 --- a/api/filter_set.py +++ b/api/filter_set.py @@ -2,7 +2,6 @@ from django.contrib.auth.models import User from django.db import models -from api.event_sources import SOURCES from api.models import ( Admin2, Appeal, @@ -183,10 +182,8 @@ class EventFilter(filters.FilterSet): countries__in = ListFilter(field_name="countries__id") regions__in = ListFilter(field_name="regions__id") id = filters.NumberFilter(field_name="id", lookup_expr="exact") - auto_generated_source = filters.ChoiceFilter( - label="Auto generated source choices", - choices=[(v, v) for v in SOURCES.values()], - ) + source = filters.ChoiceFilter(choices=Event.EventSource.choices, lookup_expr="exact") + is_subscribed = filters.BooleanFilter(label="is_subscribed", method="get_is_subcribed_event") class Meta: diff --git a/api/management/commands/create_events.py b/api/management/commands/create_events.py index 7078b772f..df6bef81d 100644 --- a/api/management/commands/create_events.py +++ b/api/management/commands/create_events.py @@ -1,6 +1,5 @@ from django.core.management.base import BaseCommand -from api.event_sources import SOURCES from api.models import Appeal, Event @@ -18,7 +17,7 @@ def handle(self, *args, **options): "dtype": appeal.dtype, "disaster_start_date": appeal.start_date, "auto_generated": True, - "auto_generated_source": SOURCES["appeal_admin"], + "source": Event.EventSource.APPEAL_ADMIN, } event = Event.objects.create(**fields) if appeal.country is not None: diff --git a/api/management/commands/index_and_notify.py b/api/management/commands/index_and_notify.py index d46620ffe..606c85d6e 100644 --- a/api/management/commands/index_and_notify.py +++ b/api/management/commands/index_and_notify.py @@ -1036,7 +1036,7 @@ def handle(self, *args, **options): condR = Q(real_data_update__gte=time_diff) # instead of modified at cond2 = ~Q(previous_update__gte=time_diff_1_day) # negate (~) no previous_update in the last day, so send once a day condF = Q( - auto_generated_source="New field report" + source=Event.EventSource.NEW_REPORT ) # exclude those events that were generated from field reports, to avoid 2x notif. condE = Q(status=CronJobStatus.ERRONEOUS) diff --git a/api/management/commands/ingest_gdacs.py b/api/management/commands/ingest_gdacs.py index c356d1cfc..5a99466d2 100644 --- a/api/management/commands/ingest_gdacs.py +++ b/api/management/commands/ingest_gdacs.py @@ -3,7 +3,6 @@ from dateutil.parser import parse from django.core.management.base import BaseCommand -from api.event_sources import SOURCES from api.logger import logger from api.models import Country, CronJob, CronJobStatus, DisasterType, Event, GDACSEvent @@ -104,7 +103,7 @@ def handle(self, *args, **options): "summary": data["description"], "disaster_start_date": data["publication_date"], "auto_generated": True, - "auto_generated_source": SOURCES["gdacs"], + "source": Event.EventSource.GDACS, "ifrc_severity_level": data["alert_level"], } event = Event.objects.create(**fields) diff --git a/api/management/commands/ingest_mdb.py b/api/management/commands/ingest_mdb.py index 7e8ac772d..b21b78abf 100644 --- a/api/management/commands/ingest_mdb.py +++ b/api/management/commands/ingest_mdb.py @@ -13,7 +13,6 @@ from django.core.management.base import BaseCommand from django.utils import timezone -from api.event_sources import SOURCES from api.fixtures.dtype_map import PK_MAP from api.logger import logger from api.models import ( @@ -249,7 +248,7 @@ def handle(self, *args, **options): "dtype": report_dtype, "disaster_start_date": timezone.now(), "auto_generated": True, - "auto_generated_source": SOURCES["report_ingest"], + "source": Event.EventSource.REPORT_INGEST, } event = Event(**event_record) event.save() diff --git a/api/management/commands/ingest_who.py b/api/management/commands/ingest_who.py index bdc0704c8..55987f2f1 100644 --- a/api/management/commands/ingest_who.py +++ b/api/management/commands/ingest_who.py @@ -11,7 +11,8 @@ class Command(BaseCommand): help = "Add new event (=emergency) entries from WHO API" def handle(self, *args, **options): - guids = [e.auto_generated_source for e in Event.objects.filter(auto_generated_source__startswith="www.who.int")] + # Change this at the top + guids = list(Event.objects.filter(source=Event.EventSource.WHO).values_list("who_guid", flat=True)) logger.info("Querying WHO RSS feed for new emergency data") # get latest @@ -48,10 +49,15 @@ def handle(self, *args, **options): "category": row.pop("category"), "pubDate": row.pop("pubDate"), } - if data["guid"] in guids: - continue if data["guid"] in ["WeDontWantThis", "NeitherThis"]: continue + try: + who_guid = int(data["guid"].strip().split(".")[-1]) + except (ValueError, IndexError): + who_guid = None + + if who_guid is None or who_guid in guids: + continue title = data["title"] # for csr link short = title.replace(" (ex-China)", "") @@ -131,7 +137,8 @@ def handle(self, *args, **options): "summary": summary, "disaster_start_date": date, "auto_generated": True, - "auto_generated_source": data["guid"], + "source": Event.EventSource.WHO, + "who_guid": who_guid, "ifrc_severity_level": alert_level, } # TODO: fields['name'] sometimes exceeds 100 maxlength, so will need some altering if this will be used diff --git a/api/migrations/0231_remove_event_auto_generated_source_event_source_and_more.py b/api/migrations/0231_remove_event_auto_generated_source_event_source_and_more.py new file mode 100644 index 000000000..f6e96e837 --- /dev/null +++ b/api/migrations/0231_remove_event_auto_generated_source_event_source_and_more.py @@ -0,0 +1,64 @@ +# Generated by Django 4.2.29 on 2026-05-07 05:56 + +from django.db import migrations, models + +def migrate_sources(apps, schema_editor): + """ + Populate the source field for Event records: + - Non auto generated events are mapped to MANUAL_INPUT. + - Events that are auto generated and have auto_generated_source are mapped to the corresponding enum value. + - 0 is temporary placeholder for unmapped/unknown auto-generated sources that will be reclassified into proper EventSource via `migrate_event_source` command. + """ + + Model = apps.get_model("api", "Event") + for obj in Model.objects.iterator(): + if not obj.auto_generated: + value = 100 + + else: + src = (obj.auto_generated_source or "").lower() + if "gdacs" in src: + value = 110 + elif "who.int" in src: + value = 120 + guid = src.strip().split(".") + id = guid[-1].strip() + obj.who_guid = int(id) + elif "field report dmis ingest" in src: + value = 130 + elif "field report admin" in src: + value = 140 + elif "appeal" in src: + value = 150 + elif "new field report" in src: + value = 160 + else: + value = 0 + obj.source = value + obj.save(update_fields=["source", "who_guid"]) + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0230_alter_districtgeoms_district'), + ] + + operations = [ + + migrations.AddField( + model_name='event', + name='who_guid', + field=models.IntegerField(blank=True, null=True, verbose_name='Who guid'), + ), + migrations.AddField( + model_name='event', + name='source', + field=models.IntegerField(choices=[(100, 'Manual input'), (110, 'GDACs scraper'), (120, 'WHO scraper'), (130, 'Field report DMIS ingest'), (140, 'Field report admin'), (150, 'Appeal admin'), (160, 'New field report'), (170, 'DREF')], default=100, verbose_name='Event source'), + ), + migrations.RunPython(migrate_sources,reverse_code=migrations.RunPython.noop), + + migrations.RemoveField( + model_name='event', + name='auto_generated_source', + ), + ] diff --git a/api/models.py b/api/models.py index 8bfcffea9..804dfb014 100644 --- a/api/models.py +++ b/api/models.py @@ -767,6 +767,32 @@ def snippet_image_path(instance, filename): class Event(models.Model): """A disaster, which could cover multiple countries""" + class EventSource(models.IntegerChoices): + + Manual_Input = 100, _("Manual input") + """MANUAL_INPUT: Event data manually entered by a user through the event administration interface.""" + + GDACS = 110, _("GDACs scraper") + """GDACS: Event data automatically ingested from the GDACS scraper.""" + + WHO = 120, _("WHO scraper") + """WHO: Event data automatically ingested from the (WHO) scraper.""" + + REPORT_INGEST = 130, _("Field report DMIS ingest") + """REPORT_INGEST: Event data imported through the DMIS field report.""" + + REPORT_ADMIN = 140, _("Field report admin") + """REPORT_ADMIN: Event data created or modified via the field report administration interface.""" + + APPEAL_ADMIN = 150, _("Appeal admin") + """APPEAL_ADMIN: Event data created or managed through the appeal administration interface.""" + + NEW_REPORT = 160, _("New field report") + """NEW_REPORT: Event data originating from newly created field reports.""" + + DREF = 170, _("DREF") + """DREF: Event originating records.""" + name = models.CharField(verbose_name=_("name"), max_length=256) dtype = models.ForeignKey(DisasterType, verbose_name=_("disaster type"), null=True, on_delete=models.SET_NULL) disaster_start_date = models.DateTimeField(verbose_name=_("disaster start date")) @@ -819,15 +845,15 @@ class Event(models.Model): verbose_name=_("ifrc severity level update date"), null=True, blank=True ) glide = models.CharField(verbose_name=_("glide"), max_length=18, blank=True) - + # NOTE: who_guid field Stores WHO source ID extracted from auto_generated_source. + # Example normalization: "www.who.int.247249"->247249 + who_guid = models.IntegerField(verbose_name=("Who guid"), blank=True, null=True) created_at = models.DateTimeField(verbose_name=_("created at"), auto_now_add=True) updated_at = models.DateTimeField(verbose_name=_("updated at"), auto_now=True) previous_update = models.DateTimeField(verbose_name=_("previous update"), null=True, blank=True) auto_generated = models.BooleanField(verbose_name=_("auto generated"), default=False, editable=False) - auto_generated_source = models.CharField( - verbose_name=_("auto generated source"), max_length=50, null=True, blank=True, editable=False - ) + source = models.IntegerField(choices=EventSource.choices, default=EventSource.Manual_Input, verbose_name=_("Event source")) # Meant to give the organization a way of highlighting certain, important events. is_featured = models.BooleanField(default=False, verbose_name=_("is featured on home page")) diff --git a/api/serializers.py b/api/serializers.py index 351749947..035c74a46 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -27,7 +27,6 @@ from per.models import Overview from utils.file_check import validate_file_type -from .event_sources import SOURCES from .models import ( Action, ActionsTaken, @@ -1069,7 +1068,7 @@ class Meta: "name", "slug", "dtype", - "auto_generated_source", + "source", "emergency_response_contact_email", "countries_for_preview", ) @@ -2218,7 +2217,7 @@ def create_event(self, report): summary=report.description or "", disaster_start_date=report.start_date, auto_generated=True, - auto_generated_source=SOURCES["new_report"], + source=Event.EventSources.NEW_REPORT, visibility=report.visibility, **{TRANSLATOR_ORIGINAL_LANGUAGE_FIELD_NAME: django_get_language()}, ) diff --git a/deployments/tests.py b/deployments/tests.py index 456f8fac7..5f56899ef 100644 --- a/deployments/tests.py +++ b/deployments/tests.py @@ -266,6 +266,8 @@ def test_project_update(self): new_country = country.CountryFactory( name="country-2", society_name="society-name-2", + iso="Te", + iso3="frj", ) new_district = district.DistrictFactory( country=new_country, From fdd4e4944fbab7a414fbf9a0a63fd262972bd10c Mon Sep 17 00:00:00 2001 From: sudip-khanal Date: Fri, 24 Apr 2026 15:41:03 +0545 Subject: [PATCH 02/28] feat(api): add new api endpoint for emergency - Add new emergency api for. - Attach latest field report. - Add test case for emergency endpoint. --- api/drf_views.py | 21 +++++++-- api/factories/event.py | 7 +++ api/serializers.py | 52 ++++++++++++++++++++++- api/test_views.py | 96 ++++++++++++++++++++++++++++++++++++++++++ assets | 2 +- main/urls.py | 3 ++ 6 files changed, 176 insertions(+), 5 deletions(-) diff --git a/api/drf_views.py b/api/drf_views.py index 31d4448bd..d6d0a2a51 100644 --- a/api/drf_views.py +++ b/api/drf_views.py @@ -89,6 +89,7 @@ Export, ExternalPartner, FieldReport, + KeyFigure, MainContact, Profile, Region, @@ -122,6 +123,7 @@ CountrySupportingPartnerSerializer, CountryTableauSerializer, DeploymentsByEventSerializer, + DetailEmergencySerializer, DetailEventSerializer, DisasterTypeSerializer, DistrictSerializer, @@ -1557,11 +1559,12 @@ def get_queryset(self): return CountrySupportingPartner.objects.select_related("country") -class EmergencyViewset(viewsets.ReadOnlyModelViewSet): +class EmergencyViewset(ReadOnlyVisibilityViewset): queryset = Event.objects.all() lookup_field = "id" serializer_class = DetailEmergencySerializer filterset_class = EventFilter + visibility_model_class = Event def get_queryset(self): return ( @@ -1579,9 +1582,21 @@ def get_queryset(self): Prefetch("contacts", queryset=EventContact.objects.all()), ) .annotate( + first_field_report_id=Subquery( + FieldReport.objects.filter(event=OuterRef("pk")) + .order_by( + "fr_num", + "updated_at", + ) + .values("id")[:1] + ), latest_field_report_id=Subquery( - FieldReport.objects.filter(event=OuterRef("pk")).order_by("-created_at").values("id")[:1] + FieldReport.objects.filter(event=OuterRef("pk")) + .order_by( + "-fr_num", + "-updated_at", + ) + .values("id")[:1] ), - appeal_id=Subquery(Appeal.objects.filter(event=OuterRef("pk")).order_by("-created_at").values("id")[:1]), ) ) diff --git a/api/factories/event.py b/api/factories/event.py index 465433b0d..78588a825 100644 --- a/api/factories/event.py +++ b/api/factories/event.py @@ -12,6 +12,7 @@ AppealHistory, AppealType, Event, + EventContact, EventFeaturedDocument, EventLink, ) @@ -133,3 +134,9 @@ class AppealHistoryFactory(factory.django.DjangoModelFactory): class Meta: model = AppealHistory + + +class EventContactFactory(factory.django.DjangoModelFactory): + + class Meta: + model = EventContact diff --git a/api/serializers.py b/api/serializers.py index 035c74a46..31dab90f2 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -2217,7 +2217,7 @@ def create_event(self, report): summary=report.description or "", disaster_start_date=report.start_date, auto_generated=True, - source=Event.EventSources.NEW_REPORT, + source=Event.EventSource.NEW_REPORT, visibility=report.visibility, **{TRANSLATOR_ORIGINAL_LANGUAGE_FIELD_NAME: django_get_language()}, ) @@ -2682,3 +2682,53 @@ class CountrySupportingPartnerSerializer(serializers.ModelSerializer): class Meta: model = CountrySupportingPartner fields = "__all__" + + +class DetailEmergencySerializer(serializers.ModelSerializer): + contacts = EventContactSerializer(many=True, read_only=True) + key_figures = KeyFigureSerializer(many=True, read_only=True) + countries = MiniCountrySerializer(many=True, read_only=True) + ifrc_severity_level_display = serializers.CharField(source="get_ifrc_severity_level_display", read_only=True) + visibility_display = serializers.CharField(source="get_visibility_display", read_only=True) + source_display = serializers.CharField(source="get_source_display", read_only=True) + # NOTE: Populated from Queryset using Annotate + first_field_report_id = serializers.IntegerField(read_only=True) + latest_field_report_id = serializers.IntegerField(read_only=True) + + class Meta: + model = Event + fields = ( + "name", + "dtype", + "countries", + "summary", + "disaster_start_date", + "auto_generated", + "source", + "source_display", + "key_figures", + "is_featured", + "is_featured_region", + "hide_attached_field_reports", + "hide_field_report_map", + "id", + "slug", + "ifrc_severity_level", + "ifrc_severity_level_display", + "ifrc_severity_level_update_date", + "parent_event", + "emergency_response_contact_email", + "visibility", + "visibility_display", + "contacts", + "num_injured", + "num_dead", + "num_missing", + "num_affected", + "num_displaced", + "created_at", + "updated_at", + "previous_update", + "first_field_report_id", + "latest_field_report_id", + ) diff --git a/api/test_views.py b/api/test_views.py index 246ab9a2b..4a6e710c8 100644 --- a/api/test_views.py +++ b/api/test_views.py @@ -12,9 +12,11 @@ from django.utils import timezone import api.models as models +from api.factories.disaster_type import DisasterTypeFactory from api.factories.event import ( AppealFactory, AppealType, + EventContactFactory, EventFactory, EventFeaturedDocumentFactory, EventLinkFactory, @@ -1048,3 +1050,97 @@ def test_ifrc_user_sees_all(self): ] ), ) + + +class EmergencyViewTestCase(APITestCase): + def setUp(self): + super().setUp() + self.disaster_type = DisasterTypeFactory.create( + name="Flood", + ) + + self.event1 = EventFactory.create( + dtype=self.disaster_type, + source=models.Event.EventSource.GDACS, + slug="test1", + parent_event=None, + ) + self.contact = EventContactFactory.create( + event=self.event1, + ) + + self.field_report1 = FieldReportFactory.create( + event=self.event1, + created_at=timezone.make_aware(datetime(2024, 1, 1)), + updated_at=timezone.make_aware(datetime(2026, 1, 1)), + fr_num=50, + ) + + self.field_report2 = FieldReportFactory.create( + event=self.event1, + created_at=timezone.make_aware(datetime(2024, 1, 1)), + updated_at=timezone.make_aware(datetime(2025, 1, 1)), + fr_num=20, + ) + + self.event2 = EventFactory.create( + dtype=self.disaster_type, + source=models.Event.EventSource.WHO, + slug="test2", + parent_event=None, + ) + + self.event3 = EventFactory.create( + dtype=self.disaster_type, + source=models.Event.EventSource.APPEAL_ADMIN, + slug="test3", + parent_event=None, + ) + self.appeal2 = AppealFactory.create( + event=self.event3, + dtype=self.disaster_type, + num_beneficiaries=9000, + amount_requested=10000, + amount_funded=1899999, + ) + + self.url = "/api/v2/emergency/" + + def test_get_emergency_list(self): + response = self.client.get(self.url) + self.assert_200(response) + self.assertEqual(response.data["count"], 2) + + def test_retrive_emergency_detail(self): + url = f"/api/v2/emergency/{self.event1.id}/" + response = self.client.get(url) + self.assert_200(response) + self.assertEqual(response.data["id"], self.event1.id) + self.assertEqual(response.data["slug"], self.event1.slug) + self.assertEqual(response.data["name"], self.event1.name) + self.assertEqual(response.data["source"], models.Event.EventSource.GDACS) + + # first field report id check + self.assertEqual(response.data["first_field_report_id"], self.field_report2.id) + # latest check field report + self.assertEqual(response.data["latest_field_report_id"], self.field_report1.id) + + # Filter Tests + def test_filter_by_source(self): + url = f"{self.url}?source=120" + response = self.client.get(url) + self.assert_200(response) + self.assertEqual(response.data["count"], 1) + self.assertEqual(response.data["results"][0]["source"], models.Event.EventSource.WHO) + + def test_filter_by_appeal_source(self): + url = f"{self.url}?source=150" + response = self.client.get(url) + self.assert_200(response) + self.assertEqual(response.data["count"], 1) + self.assertEqual(response.data["results"][0]["source"], models.Event.EventSource.APPEAL_ADMIN) + + def test_filter_by_source_no_match(self): + url = f"{self.url}?source=500" + response = self.client.get(url) + self.assert_400(response) diff --git a/assets b/assets index b386998bb..efbc4e0a2 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit b386998bb3efa9dc7145371c63cb42e4c1ceeab9 +Subproject commit efbc4e0a236f6fc71b9124c9d5f2a68fb5a8e69f diff --git a/main/urls.py b/main/urls.py index ddf055962..2de555a04 100644 --- a/main/urls.py +++ b/main/urls.py @@ -91,6 +91,9 @@ router.register(r"admin2", api_views.Admin2Viewset, basename="admin2") router.register(r"district", api_views.DistrictViewset, basename="district") router.register(r"district_rmd", api_views.DistrictRMDViewset, basename="district_rmd") + +router.register(r"emergency", api_views.EmergencyViewset, basename="emergency") + router.register(r"domainwhitelist", registration_views.DomainWhitelistViewset) router.register(r"eru", deployment_views.ERUViewset, basename="eru") router.register( From fd2b5f3026087993c7851f1a866333fa11ebf1e2 Mon Sep 17 00:00:00 2001 From: sudip-khanal Date: Tue, 28 Apr 2026 10:17:40 +0545 Subject: [PATCH 03/28] feat(api): add related appeal id on emergency api endpoint --- api/drf_views.py | 1 + api/serializers.py | 2 ++ api/test_views.py | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/api/drf_views.py b/api/drf_views.py index d6d0a2a51..3fb9a838e 100644 --- a/api/drf_views.py +++ b/api/drf_views.py @@ -1598,5 +1598,6 @@ def get_queryset(self): ) .values("id")[:1] ), + appeal_id=Subquery(Appeal.objects.filter(event=OuterRef("pk")).order_by("-created_at").values("id")[:1]), ) ) diff --git a/api/serializers.py b/api/serializers.py index 31dab90f2..2546495ce 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -2694,6 +2694,7 @@ class DetailEmergencySerializer(serializers.ModelSerializer): # NOTE: Populated from Queryset using Annotate first_field_report_id = serializers.IntegerField(read_only=True) latest_field_report_id = serializers.IntegerField(read_only=True) + appeal_id = serializers.IntegerField(read_only=True) class Meta: model = Event @@ -2731,4 +2732,5 @@ class Meta: "previous_update", "first_field_report_id", "latest_field_report_id", + "appeal_id", ) diff --git a/api/test_views.py b/api/test_views.py index 4a6e710c8..bf5fb9c56 100644 --- a/api/test_views.py +++ b/api/test_views.py @@ -1109,7 +1109,7 @@ def setUp(self): def test_get_emergency_list(self): response = self.client.get(self.url) self.assert_200(response) - self.assertEqual(response.data["count"], 2) + self.assertEqual(response.data["count"], 3) def test_retrive_emergency_detail(self): url = f"/api/v2/emergency/{self.event1.id}/" @@ -1126,7 +1126,7 @@ def test_retrive_emergency_detail(self): self.assertEqual(response.data["latest_field_report_id"], self.field_report1.id) # Filter Tests - def test_filter_by_source(self): + def test_filter_by_who_source(self): url = f"{self.url}?source=120" response = self.client.get(url) self.assert_200(response) From ec8485a9200b5ea477de42bd105458d12c91bf2c Mon Sep 17 00:00:00 2001 From: sudip-khanal Date: Tue, 12 May 2026 15:06:25 +0545 Subject: [PATCH 04/28] fix(test): fix emergency api test case --- ..._generated_source_event_source_and_more.py | 64 ------------------- ..._generated_source_event_source_and_more.py | 27 ++++++++ api/test_views.py | 8 +-- 3 files changed, 31 insertions(+), 68 deletions(-) delete mode 100644 api/migrations/0231_remove_event_auto_generated_source_event_source_and_more.py create mode 100644 api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py diff --git a/api/migrations/0231_remove_event_auto_generated_source_event_source_and_more.py b/api/migrations/0231_remove_event_auto_generated_source_event_source_and_more.py deleted file mode 100644 index f6e96e837..000000000 --- a/api/migrations/0231_remove_event_auto_generated_source_event_source_and_more.py +++ /dev/null @@ -1,64 +0,0 @@ -# Generated by Django 4.2.29 on 2026-05-07 05:56 - -from django.db import migrations, models - -def migrate_sources(apps, schema_editor): - """ - Populate the source field for Event records: - - Non auto generated events are mapped to MANUAL_INPUT. - - Events that are auto generated and have auto_generated_source are mapped to the corresponding enum value. - - 0 is temporary placeholder for unmapped/unknown auto-generated sources that will be reclassified into proper EventSource via `migrate_event_source` command. - """ - - Model = apps.get_model("api", "Event") - for obj in Model.objects.iterator(): - if not obj.auto_generated: - value = 100 - - else: - src = (obj.auto_generated_source or "").lower() - if "gdacs" in src: - value = 110 - elif "who.int" in src: - value = 120 - guid = src.strip().split(".") - id = guid[-1].strip() - obj.who_guid = int(id) - elif "field report dmis ingest" in src: - value = 130 - elif "field report admin" in src: - value = 140 - elif "appeal" in src: - value = 150 - elif "new field report" in src: - value = 160 - else: - value = 0 - obj.source = value - obj.save(update_fields=["source", "who_guid"]) - -class Migration(migrations.Migration): - - dependencies = [ - ('api', '0230_alter_districtgeoms_district'), - ] - - operations = [ - - migrations.AddField( - model_name='event', - name='who_guid', - field=models.IntegerField(blank=True, null=True, verbose_name='Who guid'), - ), - migrations.AddField( - model_name='event', - name='source', - field=models.IntegerField(choices=[(100, 'Manual input'), (110, 'GDACs scraper'), (120, 'WHO scraper'), (130, 'Field report DMIS ingest'), (140, 'Field report admin'), (150, 'Appeal admin'), (160, 'New field report'), (170, 'DREF')], default=100, verbose_name='Event source'), - ), - migrations.RunPython(migrate_sources,reverse_code=migrations.RunPython.noop), - - migrations.RemoveField( - model_name='event', - name='auto_generated_source', - ), - ] diff --git a/api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py b/api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py new file mode 100644 index 000000000..157b8dce9 --- /dev/null +++ b/api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.29 on 2026-05-12 08:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0231_alter_export_export_type'), + ] + + operations = [ + migrations.RemoveField( + model_name='event', + name='auto_generated_source', + ), + migrations.AddField( + model_name='event', + name='source', + field=models.IntegerField(choices=[(100, 'Manual input'), (110, 'GDACs scraper'), (120, 'WHO scraper'), (130, 'Field report DMIS ingest'), (140, 'Field report admin'), (150, 'Appeal admin'), (160, 'New field report'), (170, 'DREF')], default=100, verbose_name='Event source'), + ), + migrations.AddField( + model_name='event', + name='who_guid', + field=models.IntegerField(blank=True, null=True, verbose_name='Who guid'), + ), + ] diff --git a/api/test_views.py b/api/test_views.py index bf5fb9c56..992852ba5 100644 --- a/api/test_views.py +++ b/api/test_views.py @@ -1071,15 +1071,15 @@ def setUp(self): self.field_report1 = FieldReportFactory.create( event=self.event1, - created_at=timezone.make_aware(datetime(2024, 1, 1)), - updated_at=timezone.make_aware(datetime(2026, 1, 1)), + created_at=datetime.datetime(2024, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc), + updated_at=datetime.datetime(2026, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc), fr_num=50, ) self.field_report2 = FieldReportFactory.create( event=self.event1, - created_at=timezone.make_aware(datetime(2024, 1, 1)), - updated_at=timezone.make_aware(datetime(2025, 1, 1)), + created_at=datetime.datetime(2024, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc), + updated_at=datetime.datetime(2025, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc), fr_num=20, ) From 87735caeb0d009a78d58c550dddc70a307f25b72 Mon Sep 17 00:00:00 2001 From: sudip-khanal Date: Wed, 29 Apr 2026 16:59:21 +0545 Subject: [PATCH 05/28] feat(api): add script to update event source --- .../commands/migrate_event_source.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 api/management/commands/migrate_event_source.py diff --git a/api/management/commands/migrate_event_source.py b/api/management/commands/migrate_event_source.py new file mode 100644 index 000000000..62cc0989f --- /dev/null +++ b/api/management/commands/migrate_event_source.py @@ -0,0 +1,28 @@ +from django.core.management.base import BaseCommand + +from api.models import Event + + +class Command(BaseCommand): + help = "Update event sources based on name prefix and auto-generated status" + + def handle(self, *args, **options): + queryset = Event.objects.filter(auto_generated=True, source=0) + self.stdout.write( + self.style.NOTICE( + f"{queryset.count()} events will be assigned a source (GDACS or Manual Input) based on name prefix." + ) + ) + to_update = [] + for event in queryset.iterator(): + if event.name.startswith("GDACS"): + event.source = Event.EventSource.GDACS + self.stdout.write(self.style.NOTICE(f"Updating {event.name} source to {Event.EventSource.GDACS.label}")) + else: + event.source = Event.EventSource.Manual_Input + self.stdout.write(self.style.NOTICE(f"Updating {event.name} source to {Event.EventSource.Manual_Input.label}")) + event.auto_generated = False + to_update.append(event) + + updated_event = Event.objects.bulk_update(to_update, ["source", "auto_generated"]) + self.stdout.write(self.style.SUCCESS(f"Total {updated_event} Events source have been updated successfully.")) From 27765a55a12315cc2a3a8037eaf05a1a4e95c91a Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 13 May 2026 15:33:12 +0545 Subject: [PATCH 06/28] feat(api): update on emergency endpoint - Add stage field for operational timeframe - Update test cases - Use RetrieveModelMixin instead of ReadOnlyVisibilityViewset - Add enums for stage --- api/drf_views.py | 156 +++++++++++++++++++++++++++++++++++++-------- api/serializers.py | 153 +++++++++++++++++++++++++++++++++++++++++--- api/test_views.py | 46 ++----------- 3 files changed, 279 insertions(+), 76 deletions(-) diff --git a/api/drf_views.py b/api/drf_views.py index 3fb9a838e..63d713b55 100644 --- a/api/drf_views.py +++ b/api/drf_views.py @@ -6,6 +6,7 @@ Avg, Case, Count, + Exists, ExpressionWrapper, F, OuterRef, @@ -13,6 +14,7 @@ Q, Subquery, Sum, + Value, When, ) from django.db.models.fields import IntegerField @@ -57,8 +59,9 @@ ) from country_plan.models import CountryPlan from databank.serializers import CountryOverviewSerializer -from deployments.models import ERU, Personnel +from deployments.models import ERU, EmergencyProject, Personnel from deployments.serializers import ListDeployedERUByEventSerializer +from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate from main.enums import GlobalEnumSerializer, get_enum_values from main.filters import NullsLastOrderingFilter from main.permissions import DenyGuestUserMutationPermission, DenyGuestUserPermission @@ -73,6 +76,7 @@ Appeal, AppealDocument, AppealHistory, + AppealStatus, AppealType, Country, CountryKeyDocument, @@ -86,6 +90,7 @@ EventContact, EventFeaturedDocument, EventSeverityLevelHistory, + EventStage, Export, ExternalPartner, FieldReport, @@ -1559,45 +1564,142 @@ def get_queryset(self): return CountrySupportingPartner.objects.select_related("country") -class EmergencyViewset(ReadOnlyVisibilityViewset): +class EmergencyViewset( + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, + ReadOnlyVisibilityViewsetMixin, +): queryset = Event.objects.all() lookup_field = "id" serializer_class = DetailEmergencySerializer filterset_class = EventFilter - visibility_model_class = Event def get_queryset(self): + today = timezone.now().date().strftime("%Y-%m-%d") + + # We might need to use joins intead of doing with appeal code and dref appeal code + _dref_code = Subquery( + Appeal.objects.filter( + event=OuterRef(OuterRef("pk")), + status=AppealStatus.ACTIVE, + atype=AppealType.DREF, + ) + .order_by("-start_date") + .values("code")[:1] + ) + return ( super() .get_queryset() - .select_related( - "dtype", - "parent_event", + .annotate( + # Aggregated Values + response_activity_count=Subquery( + EmergencyProject.objects.filter(event=OuterRef("pk")) + .values("event") + .annotate(count=Count("id")) + .values("count")[:1], + output_field=IntegerField(), + ), + active_deployments_count=Count( + "personneldeployment__personnel", + filter=Q( + personneldeployment__personnel__type=Personnel.TypeChoices.RR, + personneldeployment__personnel__start_date__date__lte=today, + personneldeployment__personnel__end_date__date__gte=today, + personneldeployment__personnel__is_active=True, + ), + distinct=True, + ), + surge_alerts_count=Count("surgealert", distinct=True), + # stages + stage=Case( + When( + Exists(Appeal.objects.filter(event=OuterRef("pk"), status=AppealStatus.ACTIVE, atype=AppealType.APPEAL)), + then=Value(EventStage.EMERGENCY_APPEAL), + ), + When( + Exists(Appeal.objects.filter(event=OuterRef("pk"), status=AppealStatus.ACTIVE, atype=AppealType.DREF)) + & Exists(DrefFinalReport.objects.filter(appeal_code=_dref_code, status=Dref.Status.APPROVED)), + then=Value(EventStage.DREF_FINAL_REPORT), + ), + When( + Exists(Appeal.objects.filter(event=OuterRef("pk"), status=AppealStatus.ACTIVE, atype=AppealType.DREF)) + & Exists(DrefOperationalUpdate.objects.filter(appeal_code=_dref_code, status=Dref.Status.APPROVED)), + then=Value(EventStage.DREF_OPERATIONAL_UPDATE), + ), + When( + Exists(Appeal.objects.filter(event=OuterRef("pk"), status=AppealStatus.ACTIVE, atype=AppealType.DREF)), + then=Value(EventStage.DREF), + ), + When( + Exists(FieldReport.objects.filter(event=OuterRef("pk"))), + then=Value(EventStage.FIELD_REPORT), + ), + default=Value(None), + output_field=IntegerField(null=True), + ), + ) + .annotate( + # Passing values for the current stage's instance, to avoid extra queries in the serializer. + stage_field_report_id=Case( + When( + stage=EventStage.FIELD_REPORT, + then=Subquery(FieldReport.objects.filter(event=OuterRef("pk")).order_by("-created_at").values("id")[:1]), + ), + default=Value(None), + output_field=IntegerField(null=True), + ), + stage_appeal_id=Case( + When( + stage=EventStage.EMERGENCY_APPEAL, + then=Subquery( + Appeal.objects.filter(event=OuterRef("pk"), status=AppealStatus.ACTIVE, atype=AppealType.APPEAL) + .order_by("-start_date") + .values("id")[:1] + ), + ), + default=Value(None), + output_field=IntegerField(null=True), + ), + stage_dref_id=Case( + When( + stage=EventStage.DREF, + then=Subquery(Dref.objects.filter(appeal_code=_dref_code, status=Dref.Status.APPROVED).values("id")[:1]), + ), + default=Value(None), + output_field=IntegerField(null=True), + ), + stage_ops_update_id=Case( + When( + stage=EventStage.DREF_OPERATIONAL_UPDATE, + then=Subquery( + DrefOperationalUpdate.objects.filter(appeal_code=_dref_code, status=Dref.Status.APPROVED) + .order_by("-created_at") + .values("id")[:1] + ), + ), + default=Value(None), + output_field=IntegerField(null=True), + ), + stage_final_report_id=Case( + When( + stage=EventStage.DREF_FINAL_REPORT, + then=Subquery( + DrefFinalReport.objects.filter(appeal_code=_dref_code, status=Dref.Status.APPROVED) + .order_by("-created_at") + .values("id")[:1] + ), + ), + default=Value(None), + output_field=IntegerField(null=True), + ), ) + .select_related("dtype") .prefetch_related( "regions", - "countries", - "countries_for_preview", + Prefetch("countries", queryset=Country.objects.select_related("region")), + Prefetch("countries_for_preview", queryset=Country.objects.select_related("region")), Prefetch("key_figures", queryset=KeyFigure.objects.all()), Prefetch("contacts", queryset=EventContact.objects.all()), ) - .annotate( - first_field_report_id=Subquery( - FieldReport.objects.filter(event=OuterRef("pk")) - .order_by( - "fr_num", - "updated_at", - ) - .values("id")[:1] - ), - latest_field_report_id=Subquery( - FieldReport.objects.filter(event=OuterRef("pk")) - .order_by( - "-fr_num", - "-updated_at", - ) - .values("id")[:1] - ), - appeal_id=Subquery(Appeal.objects.filter(event=OuterRef("pk")).order_by("-created_at").values("id")[:1]), - ) ) diff --git a/api/serializers.py b/api/serializers.py index 2546495ce..3a2f04e86 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -5,6 +5,7 @@ from django.conf import settings from django.contrib.auth.models import Permission, User from django.db import models, transaction +from django.db.models.query import Prefetch from django.utils import timezone from django.utils.translation import get_language as django_get_language from drf_spectacular.utils import extend_schema_field @@ -14,7 +15,12 @@ from api.tasks import generate_export_pdf from api.utils import CountryValidator, RegionValidator, generate_eap_export_url from deployments.models import EmergencyProject, Personnel, PersonnelDeployment -from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate +from dref.models import ( + Dref, + DrefFinalReport, + DrefOperationalUpdate, + PlannedIntervention, +) from eap.models import EAPRegistration, FullEAP, SimplifiedEAP from lang.models import String from lang.serializers import ModelSerializer @@ -52,6 +58,7 @@ EventFeaturedDocument, EventLink, EventSeverityLevelHistory, + EventStage, Export, ExternalPartner, FieldReport, @@ -2691,10 +2698,21 @@ class DetailEmergencySerializer(serializers.ModelSerializer): ifrc_severity_level_display = serializers.CharField(source="get_ifrc_severity_level_display", read_only=True) visibility_display = serializers.CharField(source="get_visibility_display", read_only=True) source_display = serializers.CharField(source="get_source_display", read_only=True) + # NOTE: Populated from Queryset using Annotate - first_field_report_id = serializers.IntegerField(read_only=True) - latest_field_report_id = serializers.IntegerField(read_only=True) - appeal_id = serializers.IntegerField(read_only=True) + + # Aggregated values + response_activity_count = serializers.IntegerField(read_only=True) + active_deployments_count = serializers.IntegerField(read_only=True) + surge_alerts_count = serializers.IntegerField(read_only=True) + + # Stages + stage = serializers.IntegerField(read_only=True) + stage_display = serializers.SerializerMethodField() + + field_report = serializers.SerializerMethodField() + appeal = serializers.SerializerMethodField() + dref = serializers.SerializerMethodField() class Meta: model = Event @@ -2730,7 +2748,126 @@ class Meta: "created_at", "updated_at", "previous_update", - "first_field_report_id", - "latest_field_report_id", - "appeal_id", - ) + # Aggregated values + "response_activity_count", + "active_deployments_count", + "surge_alerts_count", + # Stages + "stage", + "stage_display", + "field_report", + "appeal", + "dref", + ) + + def _get_stage_instance(self, event): + if hasattr(self, "_stage_instance_cache"): + return self._stage_instance_cache + + stage = getattr(event, "stage") + instance = None + + if stage == EventStage.FIELD_REPORT and event.stage_field_report_id: + print("ayo ta", event.stage_field_report_id) + instance = ( + FieldReport.objects.select_related("dtype", "event") + .prefetch_related( + Prefetch("countries", queryset=Country.objects.select_related("region")), + Prefetch("districts", queryset=District.objects.select_related("country")), + Prefetch( + "actions_taken", + queryset=ActionsTaken.objects.prefetch_related("actions"), + ), + "contacts", + ) + .get(pk=event.stage_field_report_id) + ) + + elif stage == EventStage.EMERGENCY_APPEAL and event.stage_appeal_id: + instance = Appeal.objects.select_related( + "country", + "country__region", + "dtype", + "region", + ).get(pk=event.stage_appeal_id) + + elif stage == EventStage.DREF and event.stage_dref_id: + instance = ( + Dref.objects.select_related( + "country", + "country__region", + "disaster_type", + "national_society", + ) + .prefetch_related( + Prefetch("district", queryset=District.objects.select_related("country")), + Prefetch( + "planned_interventions", + queryset=PlannedIntervention.objects.prefetch_related("indicators"), + ), + ) + .get(pk=event.stage_dref_id) + ) + + elif stage == EventStage.DREF_OPERATIONAL_UPDATE and event.stage_ops_update_id: + instance = ( + DrefOperationalUpdate.objects.select_related("country", "country__region", "disaster_type", "national_society") + .prefetch_related( + Prefetch("district", queryset=District.objects.select_related("country")), + Prefetch( + "planned_interventions", + queryset=PlannedIntervention.objects.prefetch_related("indicators"), + ), + ) + .get(pk=event.stage_ops_update_id) + ) + + elif stage == EventStage.DREF_FINAL_REPORT and event.stage_final_report_id: + instance = ( + DrefFinalReport.objects.select_related("country", "country__region", "disaster_type", "national_society") + .prefetch_related( + Prefetch("district", queryset=District.objects.select_related("country")), + Prefetch( + "planned_interventions", + queryset=PlannedIntervention.objects.prefetch_related("indicators"), + ), + ) + .get(pk=event.stage_final_report_id) + ) + + self._stage_instance_cache = instance + return instance + + def get_stage_display(self, event): + stage = getattr(event, "stage", None) + return EventStage(stage).label if stage is not None else None + + def get_field_report(self, event): + if getattr(event, "stage", None) != EventStage.FIELD_REPORT: + return None + instance = self._get_stage_instance(event) + # TODO(susilnem): Create new serializer for the EmergencyPage + return DetailFieldReportSerializer(instance, context=self.context).data if instance else None + + def get_appeal(self, event): + if getattr(event, "stage", None) != EventStage.EMERGENCY_APPEAL: + return None + instance = self._get_stage_instance(event) + # TODO(susilnem): Create new serializer for the EmergencyPage + return RelatedAppealSerializer(instance, context=self.context).data if instance else None + + def get_dref(self, event): + if getattr(event, "stage", None) not in ( + EventStage.DREF, + EventStage.DREF_OPERATIONAL_UPDATE, + EventStage.DREF_FINAL_REPORT, + ): + return None + instance = self._get_stage_instance(event) + if not instance: + return None + + # TODO(susilnem): Create new serializer for the EmergencyPage + from dref.serializers import MiniDrefSerializer + + return MiniDrefSerializer(instance, context=self.context).data diff --git a/api/test_views.py b/api/test_views.py index 992852ba5..31aba14cb 100644 --- a/api/test_views.py +++ b/api/test_views.py @@ -22,7 +22,7 @@ EventLinkFactory, ) from api.factories.field_report import FieldReportFactory -from api.models import Profile, VisibilityChoices +from api.models import EventStage, Profile, VisibilityChoices from deployments.factories.user import UserFactory from dref.models import DrefFile from main.test_case import APITestCase @@ -1069,20 +1069,12 @@ def setUp(self): event=self.event1, ) - self.field_report1 = FieldReportFactory.create( + self.field_report = FieldReportFactory.create( event=self.event1, created_at=datetime.datetime(2024, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc), updated_at=datetime.datetime(2026, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc), fr_num=50, ) - - self.field_report2 = FieldReportFactory.create( - event=self.event1, - created_at=datetime.datetime(2024, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc), - updated_at=datetime.datetime(2025, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc), - fr_num=20, - ) - self.event2 = EventFactory.create( dtype=self.disaster_type, source=models.Event.EventSource.WHO, @@ -1104,13 +1096,6 @@ def setUp(self): amount_funded=1899999, ) - self.url = "/api/v2/emergency/" - - def test_get_emergency_list(self): - response = self.client.get(self.url) - self.assert_200(response) - self.assertEqual(response.data["count"], 3) - def test_retrive_emergency_detail(self): url = f"/api/v2/emergency/{self.event1.id}/" response = self.client.get(url) @@ -1120,27 +1105,6 @@ def test_retrive_emergency_detail(self): self.assertEqual(response.data["name"], self.event1.name) self.assertEqual(response.data["source"], models.Event.EventSource.GDACS) - # first field report id check - self.assertEqual(response.data["first_field_report_id"], self.field_report2.id) - # latest check field report - self.assertEqual(response.data["latest_field_report_id"], self.field_report1.id) - - # Filter Tests - def test_filter_by_who_source(self): - url = f"{self.url}?source=120" - response = self.client.get(url) - self.assert_200(response) - self.assertEqual(response.data["count"], 1) - self.assertEqual(response.data["results"][0]["source"], models.Event.EventSource.WHO) - - def test_filter_by_appeal_source(self): - url = f"{self.url}?source=150" - response = self.client.get(url) - self.assert_200(response) - self.assertEqual(response.data["count"], 1) - self.assertEqual(response.data["results"][0]["source"], models.Event.EventSource.APPEAL_ADMIN) - - def test_filter_by_source_no_match(self): - url = f"{self.url}?source=500" - response = self.client.get(url) - self.assert_400(response) + # Stage check + self.assertEqual(response.data["stage"], EventStage.FIELD_REPORT, response.data) + self.assertEqual(response.data["field_report"]["id"], self.field_report.id, response.data) From 4b7133744e74eb099f633504bf0620f5d3644e3a Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 13 May 2026 15:35:09 +0545 Subject: [PATCH 07/28] feat(dref): replace field report with event and update dref approve logic - add new field event instead of field report - update on serializer - update on dref approve logic - create event if not passed on dref approve --- api/models.py | 9 +++++ dref/admin.py | 4 +- dref/filter_set.py | 1 + ...089_remove_dref_field_report_dref_event.py | 24 ++++++++++++ dref/models.py | 14 ++++--- dref/serializers.py | 10 ++++- dref/utils.py | 17 ++++++++ dref/views.py | 39 ++++++++++++++++--- 8 files changed, 104 insertions(+), 14 deletions(-) create mode 100644 dref/migrations/0089_remove_dref_field_report_dref_event.py diff --git a/api/models.py b/api/models.py index 804dfb014..b8d0c549f 100644 --- a/api/models.py +++ b/api/models.py @@ -761,6 +761,15 @@ def snippet_image_path(instance, filename): return "emergencies/%s/%s" % (instance.id, filename) +# NOTE: Stage for the emergency timeline +class EventStage(models.IntegerChoices): + EMERGENCY_APPEAL = 1, _("Emergency Appeal") + DREF = 2, _("DREF") + DREF_OPERATIONAL_UPDATE = 3, _("DREF Operational Update") + DREF_FINAL_REPORT = 4, _("DREF Final Report") + FIELD_REPORT = 5, _("Field Report") + + # NOTE: If ever in future we need to create an api to update the event table # we also need to make sure to add appropriate signal to create ifrc severity level event history @reversion.register() diff --git a/dref/admin.py b/dref/admin.py index 7b38c4393..02e84fa7e 100644 --- a/dref/admin.py +++ b/dref/admin.py @@ -149,7 +149,7 @@ class DrefAdmin(CompareVersionAdmin, TranslationAdmin, admin.ModelAdmin): "images", "cover_image", "users", - "field_report", + "event", "supporting_document", "national_society_actions", "needs_identified", @@ -171,7 +171,7 @@ def get_queryset(self, request): "event_map", "cover_image", "country", - "field_report", + "event", "supporting_document", ) .prefetch_related("planned_interventions", "needs_identified", "national_society_actions", "users", "risk_security") diff --git a/dref/filter_set.py b/dref/filter_set.py index 2e27ebf18..cc41bad90 100644 --- a/dref/filter_set.py +++ b/dref/filter_set.py @@ -40,6 +40,7 @@ class BaseDrefFilterSet(filters.FilterSet): queryset=DisasterType.objects.all(), ) appeal_code = filters.CharFilter(field_name="appeal_code", lookup_expr="icontains") + event = filters.NumberFilter(field_name="event", lookup_expr="exact") class CompletedDrefOperationsFilterSet(BaseDrefFilterSet): diff --git a/dref/migrations/0089_remove_dref_field_report_dref_event.py b/dref/migrations/0089_remove_dref_field_report_dref_event.py new file mode 100644 index 000000000..d4c5afe5d --- /dev/null +++ b/dref/migrations/0089_remove_dref_field_report_dref_event.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.30 on 2026-05-13 08:20 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0232_remove_event_auto_generated_source_event_source_and_more'), + ('dref', '0088_remove_identifiedneed_title_ar_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='dref', + name='field_report', + ), + migrations.AddField( + model_name='dref', + name='event', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='event_dref', to='api.event', verbose_name='event'), + ), + ] diff --git a/dref/models.py b/dref/models.py index dc8ae4228..23895e2bf 100644 --- a/dref/models.py +++ b/dref/models.py @@ -11,7 +11,7 @@ from django.utils.translation import gettext_lazy as _ from pdf2image import convert_from_bytes -from api.models import Country, DisasterType, District, FieldReport +from api.models import Country, DisasterType, District, Event from deployments.models import Sector from main.fields import SecureFileField @@ -295,13 +295,13 @@ class Status(models.IntegerChoices): related_name="modified_by_dref", ) users = models.ManyToManyField(settings.AUTH_USER_MODEL, verbose_name=_("users"), blank=True, related_name="user_dref") - field_report = models.ForeignKey( - FieldReport, - verbose_name=_("field report"), + event = models.ForeignKey[Event]( + Event, + verbose_name=_("event"), on_delete=models.SET_NULL, null=True, blank=True, - related_name="field_report_dref", + related_name="event_dref", ) title = models.CharField(verbose_name=_("title"), max_length=255) title_prefix = models.CharField(verbose_name=_("title prefix"), max_length=255, null=True, blank=True) @@ -745,6 +745,10 @@ class Status(models.IntegerChoices): blank=True, ) + # TYPING + id: int + pk: int + class Meta: verbose_name = _("dref") verbose_name_plural = _("drefs") diff --git a/dref/serializers.py b/dref/serializers.py index c8f9e9c9d..6cc840373 100644 --- a/dref/serializers.py +++ b/dref/serializers.py @@ -11,7 +11,7 @@ from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from api.models import Appeal +from api.models import Appeal, Event from api.serializers import ( DisasterTypeSerializer, MiniCountrySerializer, @@ -365,6 +365,14 @@ class Meta: ] +class DrefApproveSerializer(serializers.Serializer): + event = serializers.PrimaryKeyRelatedField( + queryset=Event.objects.all(), + required=False, + allow_null=True, + ) + + class DrefSerializer(NestedUpdateMixin, NestedCreateMixin, ModelSerializer): SUB_TOTAL_COST = 75000 SURGE_DEPLOYMENT_COST = 10000 diff --git a/dref/utils.py b/dref/utils.py index 70db6822c..71ab31aef 100644 --- a/dref/utils.py +++ b/dref/utils.py @@ -2,6 +2,7 @@ from django.contrib.postgres.aggregates import ArrayAgg from django.db import models +from api.models import Event from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate @@ -53,3 +54,19 @@ def get_dref_users(): ) ) return dref_users_list + + +def create_event_from_dref(dref: Dref) -> Event: + event = Event.objects.create( + name=dref.title, + dtype=dref.disaster_type, + summary=dref.event_description or "", + disaster_start_date=dref.event_date or dref.hazard_date, + glide=dref.glide_code or "", + auto_generated=True, + source=Event.EventSource.DREF, + ) + + event.countries.add(dref.country) + event.districts.add(*dref.district.all()) + return event diff --git a/dref/views.py b/dref/views.py index 7975881db..6d95435c5 100644 --- a/dref/views.py +++ b/dref/views.py @@ -38,6 +38,7 @@ AddDrefUserSerializer, CompletedDrefOperationsSerializer, Dref3Serializer, + DrefApproveSerializer, DrefFileInputSerializer, DrefFileSerializer, DrefFinalReport3Serializer, @@ -50,6 +51,7 @@ MiniDrefSerializer, ) from dref.tasks import process_dref_translation +from dref.utils import create_event_from_dref from main.permissions import DenyGuestUserPermission logger = logging.getLogger(__name__) @@ -90,23 +92,48 @@ def get_queryset(self): ) return filter_dref_queryset_by_user_access(user, queryset) - @extend_schema(request=None, responses=DrefSerializer) + @extend_schema( + request=DrefApproveSerializer, + responses=DrefSerializer, + ) @action( detail=True, url_path="approve", methods=["post"], - permission_classes=[permissions.IsAuthenticated, ApproveDrefPermission, DenyGuestUserPermission], + permission_classes=[ + permissions.IsAuthenticated, + ApproveDrefPermission, + DenyGuestUserPermission, + ], ) def get_approved(self, request, pk=None, version=None): - dref = self.get_object() + dref: Dref = self.get_object() + if dref.status == Dref.Status.APPROVED: raise serializers.ValidationError(gettext("This Dref has already been approved.")) + if dref.status != Dref.Status.FINALIZED: raise serializers.ValidationError(gettext("Must be finalized before it can be approved")) + + serializer = DrefApproveSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + event = serializer.validated_data.get("event", None) + + if dref.event and event and dref.event != event: + raise serializers.ValidationError({"event": gettext("This Dref is already attached to an event.")}) + + # NOTE: If the Dref is not attached to an event, + # attaching it to the provided event or create a new one if not provided. + if not dref.event: + if event: + dref.event = event + else: + dref.event = create_event_from_dref(dref) dref.status = Dref.Status.APPROVED - dref.save(update_fields=["status"]) - serializer = DrefSerializer(dref, context={"request": request}) - return response.Response(serializer.data) + dref.save(update_fields=["event", "status"]) + + return response.Response(DrefSerializer(dref, context={"request": request}).data) @extend_schema(request=None, responses=DrefSerializer) @action( From bad2082a3f47fb11902d1ed69e3c3b9e39951a79 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 13 May 2026 15:38:19 +0545 Subject: [PATCH 08/28] feat(dref): add script to migrate events from appeal based on appeal code --- .../management/commands/migrate_dref_event.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 dref/management/commands/migrate_dref_event.py diff --git a/dref/management/commands/migrate_dref_event.py b/dref/management/commands/migrate_dref_event.py new file mode 100644 index 000000000..74b98d748 --- /dev/null +++ b/dref/management/commands/migrate_dref_event.py @@ -0,0 +1,107 @@ +from django.core.management.base import BaseCommand +from django.db.models import Case, IntegerField, Value, When + +from api.models import Appeal, AppealType +from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate + + +class Command(BaseCommand): + help = "Migrate related Event from Appeal / Dref Final / Operational Update to Dref" + + def handle(self, *args, **options): + self.stdout.write(self.style.NOTICE("Starting migration of events to Dref...")) + + appeal_event_map = dict( + Appeal.objects.filter( + atype=AppealType.DREF, + event__isnull=False, + ) + .exclude(code__isnull=True) + .values_list("code", "event_id") + ) + + final_report_map = dict( + DrefFinalReport.objects.exclude(appeal_code__isnull=True) + .annotate( + approval_priority=Case( + When(status=Dref.Status.APPROVED, then=Value(0)), + default=Value(1), + output_field=IntegerField(), + ) + ) + .order_by("dref_id", "approval_priority") + .distinct("dref_id") + .values_list("dref_id", "appeal_code") + ) + + operational_update_map = dict( + DrefOperationalUpdate.objects.exclude(appeal_code__isnull=True) + .annotate( + approval_priority=Case( + When(status=Dref.Status.APPROVED, then=Value(0)), + default=Value(1), + output_field=IntegerField(), + ) + ) + .order_by("dref_id", "approval_priority", "-created_at") + .distinct("dref_id") + .values_list("dref_id", "appeal_code") + ) + + dref_queryset = Dref.objects.only("id", "title", "appeal_code", "event_id") + self.stdout.write(self.style.NOTICE("\nMigrating Dref Event Mapping")) + + drefs_to_update = [] + for dref in dref_queryset.iterator(): + self.stdout.write(self.style.NOTICE(f"\nDref: {dref.title} (id={dref.id})")) + + appeal_code = None + matched_with = None + + self.stdout.write(self.style.NOTICE("\tChecking Final Report")) + if final_report_map.get(dref.id): + appeal_code = final_report_map[dref.id] + matched_with = "DREF FINAL REPORT" + self.stdout.write(self.style.SUCCESS(f"\t\tFound appeal_code from Final Report ({appeal_code})")) + else: + self.stdout.write(self.style.WARNING("\t\tNo appeal_code in Final Report")) + + if not appeal_code: + self.stdout.write(self.style.NOTICE("\tChecking Operational Update")) + if operational_update_map.get(dref.id): + appeal_code = operational_update_map[dref.id] + matched_with = "DREF OPERATIONAL UPDATE" + self.stdout.write(self.style.SUCCESS(f"\t\tFound appeal_code from Operational Update ({appeal_code})")) + else: + self.stdout.write(self.style.WARNING("\t\tNo appeal_code in Operational Update")) + + if not appeal_code: + self.stdout.write(self.style.NOTICE("\tChecking Dref")) + if dref.appeal_code: + appeal_code = dref.appeal_code + matched_with = "DREF" + self.stdout.write(self.style.SUCCESS(f"\t\tFound appeal_code from Dref ({appeal_code})")) + else: + self.stdout.write(self.style.WARNING("\t\tNo appeal_code in Dref")) + + if not appeal_code: + self.stdout.write(self.style.WARNING("\tNo appeal_code found, skipping...")) + continue + + new_event_id = appeal_event_map.get(appeal_code) + if not new_event_id: + self.stdout.write(self.style.WARNING(f"\tNo matching Appeal found for appeal_code={appeal_code}, skipping...")) + continue + + if dref.event_id and dref.event_id != new_event_id: + self.stdout.write( + self.style.WARNING(f"\tConflict: existing_event_id={dref.event_id}, new_event_id={new_event_id} (skipped)") + ) + continue + + dref.event_id = new_event_id + drefs_to_update.append(dref) + self.stdout.write(self.style.SUCCESS(f"\tUpdating event_id={new_event_id} using {matched_with}")) + + Dref.objects.bulk_update(drefs_to_update, ["event_id"]) + self.stdout.write(self.style.SUCCESS(f"\nSuccessfully updated {len(drefs_to_update)} Dref records\n")) From e454514c655e841ce9157f41377ab8db7771b73e Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Thu, 14 May 2026 12:09:04 +0545 Subject: [PATCH 09/28] feat(event): add latest field report on mini event endpoint --- api/drf_views.py | 11 ++++++++--- api/serializers.py | 4 ++++ main/urls.py | 1 + 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/api/drf_views.py b/api/drf_views.py index 63d713b55..49921bf1a 100644 --- a/api/drf_views.py +++ b/api/drf_views.py @@ -59,7 +59,7 @@ ) from country_plan.models import CountryPlan from databank.serializers import CountryOverviewSerializer -from deployments.models import ERU, EmergencyProject, Personnel +from deployments.models import ERU, Personnel from deployments.serializers import ListDeployedERUByEventSerializer from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate from main.enums import GlobalEnumSerializer, get_enum_values @@ -754,7 +754,8 @@ def get_queryset(self, *args, **kwargs): qset = super().get_queryset() if self.action == "mini_events": # return Event.objects.filter(parent_event__isnull=True).select_related('dtype') - return qset.filter(parent_event__isnull=True).select_related("dtype") + return qset.filter(parent_event__isnull=True).select_related("dtype").prefetch_related("countries_for_preview") + if self.action == "response_activity_events": return ( qset.filter(parent_event__isnull=True) @@ -874,7 +875,11 @@ def retrieve(self, request, pk=None, *args, **kwargs): ) @action(methods=["get"], detail=False, url_path="mini") def mini_events(self, request): - queryset = self.filter_queryset(self.get_queryset()) + queryset = self.filter_queryset(self.get_queryset()).annotate( + latest_field_report_id=Subquery( + FieldReport.objects.filter(event=OuterRef("pk")).order_by("-updated_at").values("id")[:1] + ) + ) serializer = ListMiniEventSerializer(queryset, many=True) page = self.paginate_queryset(queryset) if page is not None: diff --git a/api/serializers.py b/api/serializers.py index 3a2f04e86..c6b914f19 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -1067,6 +1067,8 @@ class Meta: class ListMiniEventSerializer(ModelSerializer): dtype = DisasterTypeSerializer(required=False) countries_for_preview = MiniCountrySerializer(many=True, read_only=True) + latest_field_report_id = serializers.IntegerField(read_only=True) + source_display = serializers.CharField(source="get_source_display", read_only=True) class Meta: model = Event @@ -1076,8 +1078,10 @@ class Meta: "slug", "dtype", "source", + "source_display", "emergency_response_contact_email", "countries_for_preview", + "latest_field_report_id", ) diff --git a/main/urls.py b/main/urls.py index 2de555a04..0fcb30b25 100644 --- a/main/urls.py +++ b/main/urls.py @@ -272,6 +272,7 @@ url(r"^api/v2/per-options/", per_views.PerOptionsView.as_view()), url(r"^api/v2/export-per/(?P\d+)/", per_views.ExportPerView.as_view()), url(r"^api/v2/local-units-options/", local_units_views.LocalUnitOptionsView.as_view()), + # NOTE: This should be removed as DefaultRouter should cover this. url(r"^api/v2/event/(?P\d+)", api_views.EventViewset.as_view({"get": "retrieve"})), url(r"^api/v2/event/(?P[-\w]+)", api_views.EventViewset.as_view({"get": "retrieve"}, lookup_field="slug")), url(r"^api/v2/delegation-office/(?P\d+)", DelegationOfficeDetailAPIView.as_view()), From 593d4d023522c2a745cb4e767a312aab9bb7b7e8 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Fri, 15 May 2026 16:50:09 +0545 Subject: [PATCH 10/28] feat(emergency): add staging logic for emergency page - priority based (Appeal, Drefs, Dref_type Appeal, Field report) - Add test cases --- api/drf_views.py | 147 ++++--- api/factories/event.py | 1 + .../commands/migrate_event_source.py | 4 +- api/models.py | 7 +- api/serializers.py | 171 ++++++-- api/test_views.py | 368 +++++++++++++++++- .../0095_alter_emergencyproject_event.py | 20 + deployments/models.py | 5 +- deployments/tests.py | 21 +- dref/management/commands/migrate_event.py | 111 ++++++ dref/models.py | 2 + dref/serializers.py | 202 ++++++++++ dref/utils.py | 2 +- 13 files changed, 952 insertions(+), 109 deletions(-) create mode 100644 deployments/migrations/0095_alter_emergencyproject_event.py create mode 100644 dref/management/commands/migrate_event.py diff --git a/api/drf_views.py b/api/drf_views.py index 49921bf1a..4e5bbbf2c 100644 --- a/api/drf_views.py +++ b/api/drf_views.py @@ -1580,30 +1580,48 @@ class EmergencyViewset( filterset_class = EventFilter def get_queryset(self): - today = timezone.now().date().strftime("%Y-%m-%d") + today = timezone.now().date() - # We might need to use joins intead of doing with appeal code and dref appeal code - _dref_code = Subquery( - Appeal.objects.filter( - event=OuterRef(OuterRef("pk")), - status=AppealStatus.ACTIVE, - atype=AppealType.DREF, - ) - .order_by("-start_date") - .values("code")[:1] + active_appeal_qs = Appeal.objects.filter( + event=OuterRef("pk"), + status=AppealStatus.ACTIVE, + ) + + active_dref_appeal_qs = active_appeal_qs.filter( + atype=AppealType.DREF, + ) + + active_emergency_appeal_qs = active_appeal_qs.filter( + atype=AppealType.APPEAL, ) + field_report_qs = FieldReport.objects.filter(event=OuterRef("pk")) + + latest_appeal_qs = active_emergency_appeal_qs.order_by("-start_date") + + approved_dref_qs = Dref.objects.filter( + event=OuterRef("pk"), + status=Dref.Status.APPROVED, + ) + + approved_ops_update_qs = DrefOperationalUpdate.objects.filter( + dref__event=OuterRef("pk"), + status=Dref.Status.APPROVED, + ).order_by("-created_at") + + approved_final_report_qs = DrefFinalReport.objects.filter( + dref__event=OuterRef("pk"), + status=Dref.Status.APPROVED, + ).order_by("-created_at") + return ( super() .get_queryset() .annotate( # Aggregated Values - response_activity_count=Subquery( - EmergencyProject.objects.filter(event=OuterRef("pk")) - .values("event") - .annotate(count=Count("id")) - .values("count")[:1], - output_field=IntegerField(), + response_activity_count=Count( + "emergency_projects", + distinct=True, ), active_deployments_count=Count( "personneldeployment__personnel", @@ -1615,26 +1633,35 @@ def get_queryset(self): ), distinct=True, ), - surge_alerts_count=Count("surgealert", distinct=True), - # stages + surge_alerts_count=Count( + "surgealert", + distinct=True, + ), + # Stage stage=Case( When( - Exists(Appeal.objects.filter(event=OuterRef("pk"), status=AppealStatus.ACTIVE, atype=AppealType.APPEAL)), + Exists(active_emergency_appeal_qs), then=Value(EventStage.EMERGENCY_APPEAL), ), When( - Exists(Appeal.objects.filter(event=OuterRef("pk"), status=AppealStatus.ACTIVE, atype=AppealType.DREF)) - & Exists(DrefFinalReport.objects.filter(appeal_code=_dref_code, status=Dref.Status.APPROVED)), + Exists(approved_final_report_qs), then=Value(EventStage.DREF_FINAL_REPORT), ), When( - Exists(Appeal.objects.filter(event=OuterRef("pk"), status=AppealStatus.ACTIVE, atype=AppealType.DREF)) - & Exists(DrefOperationalUpdate.objects.filter(appeal_code=_dref_code, status=Dref.Status.APPROVED)), + Exists(approved_ops_update_qs), then=Value(EventStage.DREF_OPERATIONAL_UPDATE), ), When( - Exists(Appeal.objects.filter(event=OuterRef("pk"), status=AppealStatus.ACTIVE, atype=AppealType.DREF)), - then=Value(EventStage.DREF), + Exists(approved_dref_qs), + then=Value(EventStage.DREF_APPLICATION), + ), + # If there is an active appeal of DREF type, but no approved DREF yet, + # we consider the emergency to be in the Emergency Appeal stage. + # Reaches here only if no approved DREF/ops-update/final-report exists + # So an active appeal type DREF appeal with no approved DREF = Emergency Appeal + When( + Exists(active_dref_appeal_qs), + then=Value(EventStage.EMERGENCY_APPEAL), ), When( Exists(FieldReport.objects.filter(event=OuterRef("pk"))), @@ -1645,31 +1672,36 @@ def get_queryset(self): ), ) .annotate( - # Passing values for the current stage's instance, to avoid extra queries in the serializer. - stage_field_report_id=Case( + # Passing values for the current stage's instance, + # to avoid extra queries in serializer. + stage_appeal_id=Case( When( - stage=EventStage.FIELD_REPORT, - then=Subquery(FieldReport.objects.filter(event=OuterRef("pk")).order_by("-created_at").values("id")[:1]), + stage=EventStage.EMERGENCY_APPEAL, + then=Subquery(latest_appeal_qs.values("id")[:1]), ), default=Value(None), output_field=IntegerField(null=True), ), - stage_appeal_id=Case( + stage_dref_id=Case( When( - stage=EventStage.EMERGENCY_APPEAL, - then=Subquery( - Appeal.objects.filter(event=OuterRef("pk"), status=AppealStatus.ACTIVE, atype=AppealType.APPEAL) - .order_by("-start_date") - .values("id")[:1] - ), + stage__in=[ + EventStage.DREF_APPLICATION, + EventStage.DREF_OPERATIONAL_UPDATE, + EventStage.DREF_FINAL_REPORT, + ], + then=Subquery(approved_dref_qs.values("id")[:1]), ), default=Value(None), output_field=IntegerField(null=True), ), - stage_dref_id=Case( + stage_final_report_id=Case( When( - stage=EventStage.DREF, - then=Subquery(Dref.objects.filter(appeal_code=_dref_code, status=Dref.Status.APPROVED).values("id")[:1]), + stage=EventStage.DREF_FINAL_REPORT, + then=Subquery( + DrefFinalReport.objects.filter( + dref__id=OuterRef("stage_dref_id"), status=Dref.Status.APPROVED + ).values("id")[:1] + ), ), default=Value(None), output_field=IntegerField(null=True), @@ -1678,33 +1710,40 @@ def get_queryset(self): When( stage=EventStage.DREF_OPERATIONAL_UPDATE, then=Subquery( - DrefOperationalUpdate.objects.filter(appeal_code=_dref_code, status=Dref.Status.APPROVED) - .order_by("-created_at") - .values("id")[:1] + DrefOperationalUpdate.objects.filter( + dref__id=OuterRef("stage_dref_id"), status=Dref.Status.APPROVED + ).values("id")[:1] ), ), default=Value(None), output_field=IntegerField(null=True), ), - stage_final_report_id=Case( + stage_field_report_id=Case( When( - stage=EventStage.DREF_FINAL_REPORT, - then=Subquery( - DrefFinalReport.objects.filter(appeal_code=_dref_code, status=Dref.Status.APPROVED) - .order_by("-created_at") - .values("id")[:1] - ), + stage=EventStage.FIELD_REPORT, + then=Subquery(field_report_qs.order_by("-updated_at", "-fr_num").values("id")[:1]), ), default=Value(None), output_field=IntegerField(null=True), ), + first_field_report_created_at=Subquery(field_report_qs.order_by("created_at", "fr_num").values("created_at")[:1]), + latest_field_report_created_at=Subquery( + field_report_qs.order_by("-created_at", "-fr_num").values("created_at")[:1] + ), ) .select_related("dtype") .prefetch_related( - "regions", - Prefetch("countries", queryset=Country.objects.select_related("region")), - Prefetch("countries_for_preview", queryset=Country.objects.select_related("region")), - Prefetch("key_figures", queryset=KeyFigure.objects.all()), - Prefetch("contacts", queryset=EventContact.objects.all()), + Prefetch( + "countries", + queryset=Country.objects.select_related("region"), + ), + Prefetch( + "key_figures", + queryset=KeyFigure.objects.all(), + ), + Prefetch( + "contacts", + queryset=EventContact.objects.all(), + ), ) ) diff --git a/api/factories/event.py b/api/factories/event.py index 78588a825..c868ff5de 100644 --- a/api/factories/event.py +++ b/api/factories/event.py @@ -26,6 +26,7 @@ class Meta: name = fuzzy.FuzzyText(length=50) slug = fuzzy.FuzzyText(length=50) dtype = factory.SubFactory(DisasterTypeFactory) + source = fuzzy.FuzzyChoice(Event.EventSource) @factory.post_generation def districts(self, create, extracted, **kwargs): diff --git a/api/management/commands/migrate_event_source.py b/api/management/commands/migrate_event_source.py index 62cc0989f..89af9d8e8 100644 --- a/api/management/commands/migrate_event_source.py +++ b/api/management/commands/migrate_event_source.py @@ -19,8 +19,8 @@ def handle(self, *args, **options): event.source = Event.EventSource.GDACS self.stdout.write(self.style.NOTICE(f"Updating {event.name} source to {Event.EventSource.GDACS.label}")) else: - event.source = Event.EventSource.Manual_Input - self.stdout.write(self.style.NOTICE(f"Updating {event.name} source to {Event.EventSource.Manual_Input.label}")) + event.source = Event.EventSource.MANUAL_INPUT + self.stdout.write(self.style.NOTICE(f"Updating {event.name} source to {Event.EventSource.MANUAL_INPUT.label}")) event.auto_generated = False to_update.append(event) diff --git a/api/models.py b/api/models.py index b8d0c549f..4649f7d0d 100644 --- a/api/models.py +++ b/api/models.py @@ -764,7 +764,7 @@ def snippet_image_path(instance, filename): # NOTE: Stage for the emergency timeline class EventStage(models.IntegerChoices): EMERGENCY_APPEAL = 1, _("Emergency Appeal") - DREF = 2, _("DREF") + DREF_APPLICATION = 2, _("DREF Application") DREF_OPERATIONAL_UPDATE = 3, _("DREF Operational Update") DREF_FINAL_REPORT = 4, _("DREF Final Report") FIELD_REPORT = 5, _("Field Report") @@ -778,7 +778,8 @@ class Event(models.Model): class EventSource(models.IntegerChoices): - Manual_Input = 100, _("Manual input") + # TODO(Susilnem): use upper case + MANUAL_INPUT = 100, _("Manual input") """MANUAL_INPUT: Event data manually entered by a user through the event administration interface.""" GDACS = 110, _("GDACs scraper") @@ -862,7 +863,7 @@ class EventSource(models.IntegerChoices): previous_update = models.DateTimeField(verbose_name=_("previous update"), null=True, blank=True) auto_generated = models.BooleanField(verbose_name=_("auto generated"), default=False, editable=False) - source = models.IntegerField(choices=EventSource.choices, default=EventSource.Manual_Input, verbose_name=_("Event source")) + source = models.IntegerField(choices=EventSource.choices, default=EventSource.MANUAL_INPUT, verbose_name=_("Event source")) # Meant to give the organization a way of highlighting certain, important events. is_featured = models.BooleanField(default=False, verbose_name=_("is featured on home page")) diff --git a/api/serializers.py b/api/serializers.py index c6b914f19..8bf80cafe 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -4,7 +4,9 @@ from django.conf import settings from django.contrib.auth.models import Permission, User +from django.contrib.gis.db.models import OuterRef, Subquery, Value from django.db import models, transaction +from django.db.models.fields import IntegerField from django.db.models.query import Prefetch from django.utils import timezone from django.utils.translation import get_language as django_get_language @@ -20,6 +22,7 @@ DrefFinalReport, DrefOperationalUpdate, PlannedIntervention, + ProposedAction, ) from eap.models import EAPRegistration, FullEAP, SimplifiedEAP from lang.models import String @@ -2137,6 +2140,82 @@ class Meta: fields = "__all__" +# NOTE: Using this specific serializer for emergency page schema +# Contains information from latest field report mostly and some fields from first field report +class EmergencyFieldReportSerializer( + serializers.ModelSerializer, +): + actions_taken = ActionsTakenSerializer(many=True) + status_display = serializers.CharField(source="get_status_display", read_only=True) + dtype = DisasterTypeSerializer(read_only=True) + dref_display = serializers.CharField(source="get_dref_display", read_only=True) + visibility_display = serializers.CharField(source="get_visibility_display", read_only=True) + contacts = FieldReportContactSerializer(many=True) + countries = MiniCountrySerializer(many=True) + districts = MiniDistrictSerializer(many=True) + + first_fr_ns_request_assistance = serializers.BooleanField(read_only=True) + first_fr_request_assistance = serializers.BooleanField(read_only=True) + + class Meta: + model = FieldReport + fields = ( + "id", + "summary", + "countries", + "districts", + "status", + "status_display", + "visibility", + "visibility_display", + "dref", + "dref_display", + "description", + "dtype", + "contacts", + # Key Figures + # IFRC figures + "num_injured", + "num_dead", + "num_missing", + "num_affected", + "num_displaced", + "epi_num_dead", + "num_assisted", + "num_localstaff", + "num_volunteers", + "num_expats_delegates", + "num_highest_risk", + "num_potentially_affected", + # Government figures + "gov_num_injured", + "gov_num_dead", + "gov_num_missing", + "gov_num_affected", + "gov_num_displaced", + "gov_num_assisted", + "gov_num_highest_risk", + "gov_num_potentially_affected", + # Other figures + "other_num_injured", + "other_num_dead", + "other_num_missing", + "other_num_affected", + "other_num_displaced", + "other_num_assisted", + "other_num_highest_risk", + "other_num_potentially_affected", + "start_date", + "report_date", + "created_at", + "updated_at", + "actions_taken", + # First field report related fields, annotate + "first_fr_ns_request_assistance", + "first_fr_request_assistance", + ) + + class DetailFieldReportSerializer(FieldReportEnumDisplayMixin, ModelSerializer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -2696,6 +2775,8 @@ class Meta: class DetailEmergencySerializer(serializers.ModelSerializer): + from dref.serializers import EmergencyDrefSerializer + contacts = EventContactSerializer(many=True, read_only=True) key_figures = KeyFigureSerializer(many=True, read_only=True) countries = MiniCountrySerializer(many=True, read_only=True) @@ -2718,6 +2799,10 @@ class DetailEmergencySerializer(serializers.ModelSerializer): appeal = serializers.SerializerMethodField() dref = serializers.SerializerMethodField() + # Operational timeframe + first_field_report_created_at = serializers.DateTimeField(read_only=True) + latest_field_report_created_at = serializers.DateTimeField(read_only=True) + class Meta: model = Event fields = ( @@ -2762,6 +2847,8 @@ class Meta: "field_report", "appeal", "dref", + "first_field_report_created_at", + "latest_field_report_created_at", ) def _get_stage_instance(self, event): @@ -2772,9 +2859,18 @@ def _get_stage_instance(self, event): instance = None if stage == EventStage.FIELD_REPORT and event.stage_field_report_id: - print("ayo ta", event.stage_field_report_id) + _first_field_report_queryset = FieldReport.objects.filter(event_id=OuterRef("event_id")).order_by( + "created_at", "fr_num" + ) instance = ( - FieldReport.objects.select_related("dtype", "event") + FieldReport.objects.select_related( + "dtype", + "event", + ) + .annotate( + first_fr_ns_request_assistance=Subquery(_first_field_report_queryset.values("ns_request_assistance")[:1]), + first_fr_request_assistance=Subquery(_first_field_report_queryset.values("request_assistance")[:1]), + ) .prefetch_related( Prefetch("countries", queryset=Country.objects.select_related("region")), Prefetch("districts", queryset=District.objects.select_related("country")), @@ -2788,57 +2884,46 @@ def _get_stage_instance(self, event): ) elif stage == EventStage.EMERGENCY_APPEAL and event.stage_appeal_id: - instance = Appeal.objects.select_related( - "country", - "country__region", - "dtype", - "region", - ).get(pk=event.stage_appeal_id) - - elif stage == EventStage.DREF and event.stage_dref_id: + instance = Appeal.objects.get(pk=event.stage_appeal_id) + elif ( + stage + in [ + EventStage.DREF_APPLICATION, + EventStage.DREF_OPERATIONAL_UPDATE, + EventStage.DREF_FINAL_REPORT, + ] + and event.stage_dref_id + ): instance = ( Dref.objects.select_related( "country", - "country__region", "disaster_type", - "national_society", + "cover_image", + "cover_image__created_by", ) - .prefetch_related( - Prefetch("district", queryset=District.objects.select_related("country")), - Prefetch( - "planned_interventions", - queryset=PlannedIntervention.objects.prefetch_related("indicators"), + .annotate( + operational_update_id=Value( + event.stage_ops_update_id, + output_field=IntegerField(null=True), + ), + final_report_id=Value( + event.stage_final_report_id, + output_field=IntegerField(null=True), ), ) - .get(pk=event.stage_dref_id) - ) - - elif stage == EventStage.DREF_OPERATIONAL_UPDATE and event.stage_ops_update_id: - instance = ( - DrefOperationalUpdate.objects.select_related("country", "country__region", "disaster_type", "national_society") .prefetch_related( Prefetch("district", queryset=District.objects.select_related("country")), Prefetch( "planned_interventions", queryset=PlannedIntervention.objects.prefetch_related("indicators"), ), - ) - .get(pk=event.stage_ops_update_id) - ) - - elif stage == EventStage.DREF_FINAL_REPORT and event.stage_final_report_id: - instance = ( - DrefFinalReport.objects.select_related("country", "country__region", "disaster_type", "national_society") - .prefetch_related( - Prefetch("district", queryset=District.objects.select_related("country")), Prefetch( - "planned_interventions", - queryset=PlannedIntervention.objects.prefetch_related("indicators"), + "proposed_action", + queryset=ProposedAction.objects.prefetch_related("activities"), ), ) - .get(pk=event.stage_final_report_id) + .get(pk=event.stage_dref_id) ) - self._stage_instance_cache = instance return instance @@ -2846,23 +2931,24 @@ def get_stage_display(self, event): stage = getattr(event, "stage", None) return EventStage(stage).label if stage is not None else None + @extend_schema_field(EmergencyFieldReportSerializer()) def get_field_report(self, event): if getattr(event, "stage", None) != EventStage.FIELD_REPORT: return None instance = self._get_stage_instance(event) - # TODO(susilnem): Create new serializer for the EmergencyPage - return DetailFieldReportSerializer(instance, context=self.context).data if instance else None + return EmergencyFieldReportSerializer(instance, context=self.context).data if instance else None + @extend_schema_field(RelatedAppealSerializer()) def get_appeal(self, event): if getattr(event, "stage", None) != EventStage.EMERGENCY_APPEAL: return None instance = self._get_stage_instance(event) - # TODO(susilnem): Create new serializer for the EmergencyPage return RelatedAppealSerializer(instance, context=self.context).data if instance else None + @extend_schema_field(EmergencyDrefSerializer()) def get_dref(self, event): if getattr(event, "stage", None) not in ( - EventStage.DREF, + EventStage.DREF_APPLICATION, EventStage.DREF_OPERATIONAL_UPDATE, EventStage.DREF_FINAL_REPORT, ): @@ -2871,7 +2957,6 @@ def get_dref(self, event): if not instance: return None - # TODO(susilnem): Create new serializer for the EmergencyPage - from dref.serializers import MiniDrefSerializer + from dref.serializers import EmergencyDrefSerializer - return MiniDrefSerializer(instance, context=self.context).data + return EmergencyDrefSerializer(instance, context=self.context).data diff --git a/api/test_views.py b/api/test_views.py index 31aba14cb..b76e0a75f 100644 --- a/api/test_views.py +++ b/api/test_views.py @@ -10,8 +10,10 @@ from django.test import override_settings from django.urls import reverse from django.utils import timezone +from django.utils.dateparse import parse_datetime import api.models as models +from api.factories.country import CountryFactory from api.factories.disaster_type import DisasterTypeFactory from api.factories.event import ( AppealFactory, @@ -22,9 +24,14 @@ EventLinkFactory, ) from api.factories.field_report import FieldReportFactory -from api.models import EventStage, Profile, VisibilityChoices +from api.models import AppealStatus, EventStage, Profile, VisibilityChoices from deployments.factories.user import UserFactory -from dref.models import DrefFile +from dref.factories.dref import ( + DrefFactory, + DrefFinalReportFactory, + DrefOperationalUpdateFactory, +) +from dref.models import Dref, DrefFile from main.test_case import APITestCase from per.factories import OpsLearningFactory @@ -1108,3 +1115,360 @@ def test_retrive_emergency_detail(self): # Stage check self.assertEqual(response.data["stage"], EventStage.FIELD_REPORT, response.data) self.assertEqual(response.data["field_report"]["id"], self.field_report.id, response.data) + + +class EmergencyStageTestCase(APITestCase): + """ + Tests for the stage annotation and stage-specific nested serializer + fields (field_report / appeal / dref) on GET /api/v2/emergency// + """ + + def setUp(self): + super().setUp() + self.disaster_type = DisasterTypeFactory.create(name="Flood") + self.country = CountryFactory.create(name="country1", iso3="DEP", iso="DE") + + def _url(self, event): + return f"/api/v2/emergency/{event.id}/" + + def _get(self, event): + return self.client.get(self._url(event)) + + def _approved_dref(self, event): + return DrefFactory.create( + event=event, + status=Dref.Status.APPROVED, + disaster_type=self.disaster_type, + country=self.country, + ) + + def _approved_ops_update(self, dref): + return DrefOperationalUpdateFactory.create( + dref=dref, + status=Dref.Status.APPROVED, + disaster_type=self.disaster_type, + country=self.country, + ) + + def _approved_final_report(self, dref): + return DrefFinalReportFactory.create( + dref=dref, + status=Dref.Status.APPROVED, + disaster_type=self.disaster_type, + country=self.country, + ) + + def _active_emergency_appeal(self, event): + return AppealFactory.create( + event=event, + dtype=self.disaster_type, + status=AppealStatus.ACTIVE, + atype=AppealType.APPEAL, + ) + + def test_stage_is_none(self): + event = EventFactory.create( + dtype=self.disaster_type, + source=models.Event.EventSource.MANUAL_INPUT, + ) + + data = self._get(event).data + + self.assertIsNone(data["stage"]) + self.assertIsNone(data["stage_display"]) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["appeal"]) + self.assertIsNone(data["dref"]) + + def test_stage_field_report(self): + event = EventFactory.create(dtype=self.disaster_type, source=models.Event.EventSource.MANUAL_INPUT) + field_report = FieldReportFactory.create(event=event) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.FIELD_REPORT) + self.assertEqual(data["stage_display"], EventStage(EventStage.FIELD_REPORT).label) + self.assertEqual(data["field_report"]["id"], field_report.id) + self.assertIsNone(data["appeal"]) + self.assertIsNone(data["dref"]) + + def test_stage_field_report_returns_latest_by_updated_at_and_fr_num(self): + """ + When multiple field reports exist, the one ordered by + (-updated_at, -fr_num) should be returned. + """ + event = EventFactory.create(dtype=self.disaster_type) + first_fr = FieldReportFactory.create( + event=event, + dtype=self.disaster_type, + created_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2024, 6, 1, tzinfo=datetime.UTC), + fr_num=1, + ) + + latest_fr = FieldReportFactory.create( + event=event, + dtype=self.disaster_type, + created_at=datetime.datetime(2025, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + fr_num=2, + ) + + data = self._get(event).data + + self.assertIsNotNone(data["field_report"], data) + self.assertEqual(data["field_report"]["id"], latest_fr.id) + self.assertEqual(data["stage"], EventStage.FIELD_REPORT) + self.assertEqual(parse_datetime(data["first_field_report_created_at"]), first_fr.created_at) + self.assertEqual(parse_datetime(data["latest_field_report_created_at"]), latest_fr.created_at) + + def test_stage_dref_application(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + DrefOperationalUpdateFactory.create( + dref=dref, + status=Dref.Status.DRAFT, + disaster_type=self.disaster_type, + country=self.country, + ) + + data = self._get(event).data + + self.assertEqual( + data["stage"], + EventStage.DREF_APPLICATION.value, + ) + + self.assertEqual( + data["stage_display"], + EventStage(EventStage.DREF_APPLICATION).label, + ) + + self.assertEqual(data["dref"]["id"], dref.id) + self.assertIsNone(data["dref"]["operational_update_details"]) + self.assertIsNone(data["dref"]["final_report_details"]) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["appeal"]) + + # After Final report create + dref_final_report = DrefFinalReportFactory.create( + dref=dref, + status=Dref.Status.APPROVED, + disaster_type=self.disaster_type, + country=self.country, + ) + data = self._get(event).data + + self.assertEqual( + data["stage"], + EventStage.DREF_FINAL_REPORT.value, + ) + self.assertEqual( + data["stage_display"], + EventStage(EventStage.DREF_FINAL_REPORT).label, + ) + self.assertEqual(data["dref"]["id"], dref.id) + self.assertIsNone(data["dref"]["operational_update_details"]) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["appeal"]) + self.assertIsNotNone(data["dref"]["final_report_details"]) + self.assertEqual(data["dref"]["final_report_details"]["id"], dref_final_report.id) + + def test_stage_dref_operational_update(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + ops_update = self._approved_ops_update(dref) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_OPERATIONAL_UPDATE) + self.assertEqual(data["dref"]["id"], dref.id) + self.assertEqual(data["dref"]["operational_update_details"]["id"], ops_update.id) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["appeal"]) + + def test_stage_dref_final_report(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + final_report = self._approved_final_report(dref) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_FINAL_REPORT) + self.assertEqual(data["stage_display"], EventStage(EventStage.DREF_FINAL_REPORT).label) + self.assertEqual(data["dref"]["id"], dref.id) + self.assertEqual(data["dref"]["final_report_details"]["id"], final_report.id) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["appeal"]) + + def test_stage_emergency_appeal(self): + event = EventFactory.create( + dtype=self.disaster_type, + source=models.Event.EventSource.MANUAL_INPUT, + ) + appeal = self._active_emergency_appeal(event) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + self.assertEqual(data["stage_display"], EventStage(EventStage.EMERGENCY_APPEAL).label) + self.assertEqual(data["appeal"]["id"], appeal.id) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["dref"]) + + # If Field report is created + first_fr = FieldReportFactory.create( + event=event, + dtype=self.disaster_type, + created_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2024, 6, 1, tzinfo=datetime.UTC), + fr_num=1, + ) + latest_fr = FieldReportFactory.create( + event=event, + dtype=self.disaster_type, + created_at=datetime.datetime(2025, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + fr_num=2, + ) + + data = self._get(event).data + # Emergency appeal stage should take priority over field report stage + # BUT Fields Report dates for latest and first should be present + self.assertEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + + self.assertEqual(parse_datetime(data["first_field_report_created_at"]), first_fr.created_at) + self.assertEqual(parse_datetime(data["latest_field_report_created_at"]), latest_fr.created_at) + + def test_inactive_appeal_does_not_trigger_emergency_appeal_stage(self): + """An appeal that is not ACTIVE should not resolve to EMERGENCY_APPEAL.""" + event = EventFactory.create(dtype=self.disaster_type) + AppealFactory.create( + event=event, + dtype=self.disaster_type, + status=AppealStatus.CLOSED, + atype=AppealType.APPEAL, + ) + + data = self._get(event).data + self.assertNotEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + + def test_emergency_appeal_takes_priority_over_entire_dref_chain(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + self._approved_ops_update(dref) + self._approved_final_report(dref) + self._active_emergency_appeal(event) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + + def test_dref_final_report_takes_priority_over_ops_update_and_application(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + self._approved_ops_update(dref) + self._approved_final_report(dref) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_FINAL_REPORT) + + def test_dref_ops_update_takes_priority_over_dref_application(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + self._approved_ops_update(dref) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_OPERATIONAL_UPDATE) + + # Create DRAFT Final report, stage should still be DREF_OPERATIONAL_UPDATE until Final report is APPROVED + DrefFinalReportFactory.create( + dref=dref, + status=Dref.Status.DRAFT, + disaster_type=self.disaster_type, + country=self.country, + ) + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.DREF_OPERATIONAL_UPDATE) + self.assertIsNone(data["appeal"]) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["dref"]["final_report_details"]) + + def test_dref_application_takes_priority_over_field_report(self): + event = EventFactory.create(dtype=self.disaster_type) + FieldReportFactory.create(event=event) + dref = self._approved_dref(event) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_APPLICATION) + + # Create DRAFT ops update, stage should still be DREF Application until ops update is APPROVED + DrefOperationalUpdateFactory.create( + dref=dref, + status=Dref.Status.DRAFT, + disaster_type=self.disaster_type, + country=self.country, + ) + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.DREF_APPLICATION) + self.assertIsNone(data["appeal"]) + self.assertIsNone(data["field_report"]) + + # Create DRAFT Final report, stage should still be DREF Aplication until Final report is APPROVED + DrefFinalReportFactory.create( + dref=dref, + status=Dref.Status.DRAFT, + disaster_type=self.disaster_type, + country=self.country, + ) + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_APPLICATION) + self.assertIsNone(data["appeal"]) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["dref"]["final_report_details"]) + + def fallback_to_appeal_for_no_approved_dref(self): + """ + If no approved DREF application exists, but an ACTIVE appeal dref type exists, stage should resolve to EMERGENCY_APPEAL + and not DREF. + """ + event = EventFactory.create(dtype=self.disaster_type) + AppealFactory.create( + event=event, + dtype=self.disaster_type, + status=AppealStatus.ACTIVE, + atype=AppealType.APPEAL, + ) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + self.assertIsNone(data["dref"]) + self.assertIsNone(data["field_report"]) + self.assertIsNotNone(data["appeal"]) + + # If Field report is created + first_fr = FieldReportFactory.create( + event=event, + dtype=self.disaster_type, + created_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2024, 6, 1, tzinfo=datetime.UTC), + fr_num=1, + ) + latest_fr = FieldReportFactory.create( + event=event, + dtype=self.disaster_type, + created_at=datetime.datetime(2025, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + fr_num=2, + ) + + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + + self.assertEqual(parse_datetime(data["first_field_report_created_at"]), first_fr.created_at) + self.assertEqual(parse_datetime(data["latest_field_report_created_at"]), latest_fr.created_at) diff --git a/deployments/migrations/0095_alter_emergencyproject_event.py b/deployments/migrations/0095_alter_emergencyproject_event.py new file mode 100644 index 000000000..7595f257b --- /dev/null +++ b/deployments/migrations/0095_alter_emergencyproject_event.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.30 on 2026-05-14 05:06 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0232_remove_event_auto_generated_source_event_source_and_more'), + ('deployments', '0094_erureadinesstype_ns_contribution'), + ] + + operations = [ + migrations.AlterField( + model_name='emergencyproject', + name='event', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='emergency_projects', to='api.event', verbose_name='Event'), + ), + ] diff --git a/deployments/models.py b/deployments/models.py index ee3ef795b..0fa470b46 100644 --- a/deployments/models.py +++ b/deployments/models.py @@ -817,7 +817,10 @@ class ActivityStatus(models.TextChoices): related_name="+", ) event = models.ForeignKey( - Event, verbose_name=_("Event"), on_delete=models.CASCADE, related_name="+" + Event, + verbose_name=_("Event"), + on_delete=models.CASCADE, + related_name="emergency_projects", ) # this is the current operation districts = models.ManyToManyField( District, diff --git a/deployments/tests.py b/deployments/tests.py index 5f56899ef..af709512d 100644 --- a/deployments/tests.py +++ b/deployments/tests.py @@ -687,9 +687,24 @@ def setUp(self): disaster_start_date=datetime.datetime(2025, 3, 1), ) - self.country1 = country.CountryFactory(name="Test Country1") - self.country2 = country.CountryFactory(name="Test Country2") - self.country3 = country.CountryFactory(name="Test Country3") + self.country1 = country.CountryFactory.create( + name="Test Country1", + iso3="Ts1", + iso="T1", + society_name="Test Society", + ) + self.country2 = country.CountryFactory.create( + name="Test Country2", + iso3="TS2", + iso="T2", + society_name="Test Society2", + ) + self.country3 = country.CountryFactory.create( + name="Test Country3", + iso3="TS3", + iso="T3", + society_name="Test Society3", + ) self.eru_owner = ERUOwnerFactory( national_society_country=self.country1, diff --git a/dref/management/commands/migrate_event.py b/dref/management/commands/migrate_event.py new file mode 100644 index 000000000..576aa401b --- /dev/null +++ b/dref/management/commands/migrate_event.py @@ -0,0 +1,111 @@ +from django.core.management.base import BaseCommand + +from api.models import Appeal, AppealType +from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate + + +class Command(BaseCommand): + help = "Migrate related Event from Appeal / Dref Final / Operational Update to Dref" + + def handle(self, *args, **options): + self.stdout.write(self.style.NOTICE("Starting migration of events to Dref...")) + + appeal_map = dict( + Appeal.objects.filter( + atype=AppealType.DREF, + event__isnull=False, + ) + .exclude(code__isnull=True) + .values_list("code", "id") + ) + + appeal_event_map = dict( + Appeal.objects.filter( + atype=AppealType.DREF, + event__isnull=False, + ) + .exclude(code__isnull=True) + .values_list("code", "event_id") + ) + + final_report_map = dict(DrefFinalReport.objects.exclude(appeal_code__isnull=True).values_list("appeal_code", "id")) + + operational_update_map = dict( + DrefOperationalUpdate.objects.exclude(appeal_code__isnull=True).values_list("appeal_code", "id") + ) + + dref_queryset = Dref.objects.only("id", "title", "appeal_code", "event_id") + + self.stdout.write(self.style.NOTICE(f"Found {dref_queryset.count()} Dref records")) + self.stdout.write(self.style.NOTICE("\nMigrating Dref Event Mapping")) + + drefs_to_update = [] + + for dref in dref_queryset.iterator(): + code = dref.appeal_code + + self.stdout.write(self.style.NOTICE(f"\nDref: {dref.title} (id={dref.id})")) + + new_event_id = None + matched_with = None + + appeal_id = None + final_id = None + op_id = None + + self.stdout.write(self.style.NOTICE("\tChecking Appeal")) + + if code: + appeal_id = appeal_map.get(code) + new_event_id = appeal_event_map.get(code) + + if new_event_id: + matched_with = f"APPEAL (id:{appeal_id})" + self.stdout.write(self.style.SUCCESS(f"\t\tMatched with Appeal (id:{appeal_id})")) + else: + self.stdout.write(self.style.WARNING("\t\tNo match in Appeal")) + + if not new_event_id: + self.stdout.write(self.style.NOTICE("\tChecking Final Report")) + + final_id = final_report_map.get(code) + + if final_id: + new_event_id = appeal_event_map.get(code) + if new_event_id: + matched_with = f"DREF FINAL REPORT (id:{final_id})" + self.stdout.write(self.style.SUCCESS(f"\t\tMatched with Final Report (id:{final_id})")) + else: + self.stdout.write(self.style.WARNING("\t\tNo match in Final Report")) + + if not new_event_id: + self.stdout.write(self.style.NOTICE("\tChecking Operational Update")) + + op_id = operational_update_map.get(code) + + if op_id: + new_event_id = appeal_event_map.get(code) + if new_event_id: + matched_with = f"DREF OPERATIONAL UPDATE (id:{op_id})" + self.stdout.write(self.style.SUCCESS(f"\t\tMatched with Operational Update (id:{op_id})")) + else: + self.stdout.write(self.style.WARNING("\t\tNo match in Operational Update")) + + if not new_event_id: + self.stdout.write(self.style.WARNING("\tNo matching event found")) + continue + + if dref.event_id and dref.event_id != new_event_id: + self.stdout.write( + self.style.WARNING(f"\tConflict: existing_event_id={dref.event_id}, new_event_id={new_event_id} (skipped)") + ) + continue + + dref.event_id = new_event_id + drefs_to_update.append(dref) + + self.stdout.write(self.style.SUCCESS(f"\tUpdating event using {matched_with}")) + + Dref.objects.bulk_update(drefs_to_update, ["event_id"]) + + self.stdout.write(self.style.SUCCESS(f"\nSuccessfully updated {len(drefs_to_update)} Dref records\n")) diff --git a/dref/models.py b/dref/models.py index 23895e2bf..4be868395 100644 --- a/dref/models.py +++ b/dref/models.py @@ -1,5 +1,6 @@ import copy import os +from typing import Optional import reversion from django.conf import settings @@ -748,6 +749,7 @@ class Status(models.IntegerChoices): # TYPING id: int pk: int + event_id: Optional[int] class Meta: verbose_name = _("dref") diff --git a/dref/serializers.py b/dref/serializers.py index 6cc840373..3b44bb05e 100644 --- a/dref/serializers.py +++ b/dref/serializers.py @@ -5,6 +5,7 @@ from django.conf import settings from django.contrib.auth.models import User from django.db import models, transaction +from django.db.models.query import Prefetch from django.utils import timezone from django.utils.translation import get_language, gettext from django.utils.translation import gettext_lazy as _ @@ -247,6 +248,7 @@ class Meta: "operational_update_details", "final_report_details", "country", + "event", "country_details", "has_ops_update", "has_final_report", @@ -2248,3 +2250,203 @@ class Meta(BaseDref3Serializer.Meta): class DrefFinalReport3Serializer(BaseDref3Serializer): class Meta(BaseDref3Serializer.Meta): model = DrefFinalReport + + +# NOTE: This serializer is only used for the emergency page in GO, +# which has a very specific and limited use case. +# It is not intended to be a general-purpose serializer for DREF objects, +# and as such, it does not include all fields or functionality of the other serializers. +# It is designed to provide the necessary data for the emergency page in a format that is easy to consume and display. +class EmergencyDrefFinalReportSerializer(serializers.ModelSerializer): + status_display = serializers.CharField(source="get_status_display", read_only=True) + country_details = MiniCountrySerializer(source="country", read_only=True) + district_details = MiniDistrictSerializer(source="district", read_only=True, many=True) + planned_interventions = PlannedInterventionSerializer(many=True, read_only=True) + cover_image_file = DrefFileSerializer(source="cover_image", read_only=True) + proposed_action = ProposedActionSerializer(many=True, required=False) + disaster_type_details = DisasterTypeSerializer(source="disaster_type", read_only=True) + + class Meta: + model = DrefFinalReport + fields = ( + "id", + "title", + "disaster_type_details", + "cover_image_file", + "status", + "status_display", + "event_scope", + "event_description", + "event_date", + "appeal_code", + "glide_code", + "country_details", + "district_details", + "planned_interventions", + "proposed_action", + # Timeframe of operation + "operation_start_date", + "operation_end_date", + "total_dref_allocation", + "government_requested_assistance", + "number_of_people_targeted", + "number_of_people_affected", + "total_targeted_population", + "estimated_number_of_affected_male", + "estimated_number_of_affected_female", + "estimated_number_of_affected_girls_under_18", + "estimated_number_of_affected_boys_under_18", + ) + + +class EmergencyDrefOperationalUpdateSerializer(serializers.ModelSerializer): + status_display = serializers.CharField(source="get_status_display", read_only=True) + country_details = MiniCountrySerializer(source="country", read_only=True) + district_details = MiniDistrictSerializer(source="district", read_only=True, many=True) + planned_interventions = PlannedInterventionSerializer(many=True, read_only=True) + cover_image_file = DrefFileSerializer(source="cover_image", required=False, allow_null=True) + disaster_type_details = DisasterTypeSerializer(source="disaster_type", read_only=True) + + class Meta: + model = DrefOperationalUpdate + fields = ( + "id", + "title", + "disaster_type_details", + "cover_image_file", + "status", + "status_display", + "event_scope", + "event_description", + "event_date", + "appeal_code", + "glide_code", + "country_details", + "district_details", + "planned_interventions", + # Timeframe of operation + "new_operational_start_date", + "new_operational_end_date", + "total_dref_allocation", + "government_requested_assistance", + "number_of_people_targeted", + "number_of_people_affected", + "total_targeted_population", + "estimated_number_of_affected_male", + "estimated_number_of_affected_female", + "estimated_number_of_affected_girls_under_18", + "estimated_number_of_affected_boys_under_18", + ) + + +class EmergencyDrefSerializer(serializers.ModelSerializer): + status_display = serializers.CharField(source="get_status_display", read_only=True) + country_details = MiniCountrySerializer(source="country", read_only=True) + district_details = MiniDistrictSerializer(source="district", read_only=True, many=True) + planned_interventions = PlannedInterventionSerializer(many=True, read_only=True) + proposed_action = ProposedActionSerializer(many=True, read_only=True) + cover_image_file = DrefFileSerializer(source="cover_image", required=False, allow_null=True) + disaster_type_details = DisasterTypeSerializer(source="disaster_type", read_only=True) + + # Dref operational update + operational_update_details = serializers.SerializerMethodField() + # Dref Final report + final_report_details = serializers.SerializerMethodField() + + class Meta: + model = Dref + fields = ( + "id", + "title", + "disaster_type_details", + "cover_image_file", + "event_description", + "event_scope", + "status", + "status_display", + "appeal_code", + "glide_code", + "country_details", + "district_details", + "planned_interventions", + "proposed_action", + "emergency_appeal_planned", + "government_requested_assistance", + "did_ns_request_fund", + # Related drefs + "operational_update_details", + "final_report_details", + # Key Figures + "num_affected", + "num_assisted", + "hazard_date_and_location", + "amount_requested", + "total_targeted_population", + "estimated_number_of_affected_male", + "estimated_number_of_affected_female", + "estimated_number_of_affected_girls_under_18", + "estimated_number_of_affected_boys_under_18", + # Operational timeframe date + "hazard_date", + "end_date", + # For Response Type + "event_date", + ) + + @extend_schema_field(EmergencyDrefOperationalUpdateSerializer()) + def get_operational_update_details(self, obj): + if not obj.operational_update_id: + return None + + instance = ( + DrefOperationalUpdate.objects.select_related( + "country", + "disaster_type", + "cover_image", + "cover_image__created_by", + ) + .prefetch_related( + "district", + Prefetch( + "planned_interventions", + queryset=PlannedIntervention.objects.prefetch_related("indicators"), + ), + ) + .get(pk=obj.operational_update_id) + ) + + return EmergencyDrefOperationalUpdateSerializer( + instance, + context=self.context, + ).data + + @extend_schema_field(EmergencyDrefFinalReportSerializer()) + def get_final_report_details(self, obj): + if not obj.final_report_id: + return None + + instance = ( + DrefFinalReport.objects.select_related( + "country", + "disaster_type", + "cover_image", + "cover_image__created_by", + ) + .prefetch_related( + "district", + Prefetch( + "planned_interventions", + queryset=PlannedIntervention.objects.prefetch_related("indicators"), + ), + Prefetch( + "proposed_action", + queryset=ProposedAction.objects.prefetch_related("activities"), + ), + ) + .get(pk=obj.final_report_id) + ) + + return EmergencyDrefFinalReportSerializer( + instance, + context=self.context, + ).data diff --git a/dref/utils.py b/dref/utils.py index 71ab31aef..15fd74efe 100644 --- a/dref/utils.py +++ b/dref/utils.py @@ -60,7 +60,7 @@ def create_event_from_dref(dref: Dref) -> Event: event = Event.objects.create( name=dref.title, dtype=dref.disaster_type, - summary=dref.event_description or "", + summary=dref.event_description or dref.event_scope or "", disaster_start_date=dref.event_date or dref.hazard_date, glide=dref.glide_code or "", auto_generated=True, From 45987b5edd6760f92618fdceeb67551d78c6ea72 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Thu, 21 May 2026 14:00:06 +0545 Subject: [PATCH 11/28] feat(emergency): Add new stage for appeal dref type - update on stage logic generation - rename enums of the EventSource - update test cases - add source field from auto generated source on event - generate new migrations for the emergency, dref and deployments --- api/admin.py | 2 +- api/drf_views.py | 26 +++++- api/enums.py | 1 + api/management/commands/index_and_notify.py | 2 +- api/management/commands/ingest_mdb.py | 2 +- .../commands/migrate_event_source.py | 28 ------ ..._generated_source_event_source_and_more.py | 92 ++++++++++++++++--- api/models.py | 14 +-- api/serializers.py | 41 +++++++-- api/test_views.py | 8 +- .../0095_alter_emergencyproject_event.py | 2 +- ...089_remove_dref_field_report_dref_event.py | 2 +- 12 files changed, 152 insertions(+), 68 deletions(-) delete mode 100644 api/management/commands/migrate_event_source.py diff --git a/api/admin.py b/api/admin.py index 628ac97ff..01352ed76 100644 --- a/api/admin.py +++ b/api/admin.py @@ -619,7 +619,7 @@ def create_events(self, request, queryset): dtype=getattr(report, "dtype"), disaster_start_date=getattr(report, "created_at"), auto_generated=True, - source=models.Event.EventSource.REPORT_ADMIN, + source=models.Event.EventSource.FIELD_REPORT_ADMIN, ) if getattr(report, "countries").exists(): for country in report.countries.all(): diff --git a/api/drf_views.py b/api/drf_views.py index 4e5bbbf2c..57d39d3c2 100644 --- a/api/drf_views.py +++ b/api/drf_views.py @@ -89,6 +89,7 @@ Event, EventContact, EventFeaturedDocument, + EventLink, EventSeverityLevelHistory, EventStage, Export, @@ -759,7 +760,7 @@ def get_queryset(self, *args, **kwargs): if self.action == "response_activity_events": return ( qset.filter(parent_event__isnull=True) - .filter(Q(auto_generated=False) | Q(source=Event.EventSource.NEW_REPORT)) + .filter(Q(auto_generated=False) | Q(source=Event.EventSource.NEW_FIELD_REPORT)) .select_related("dtype") ) return ( @@ -1361,7 +1362,7 @@ class SupportedActivityViewset(viewsets.ReadOnlyModelViewSet): # summary=report.description or "", # disaster_start_date=report.start_date, # auto_generated=True, -# source=Event.EventSource.NEW_REPORT, +# source=Event.EventSource.NEW_FIELD_REPORT, # visibility=report.visibility, # **{TRANSLATOR_ORIGINAL_LANGUAGE_FIELD_NAME: django_get_language()}, # ) @@ -1658,10 +1659,10 @@ def get_queryset(self): # If there is an active appeal of DREF type, but no approved DREF yet, # we consider the emergency to be in the Emergency Appeal stage. # Reaches here only if no approved DREF/ops-update/final-report exists - # So an active appeal type DREF appeal with no approved DREF = Emergency Appeal + # So an active appeal type DREF appeal with no approved DREF = DREF_APPEAL_ONLY stage. When( Exists(active_dref_appeal_qs), - then=Value(EventStage.EMERGENCY_APPEAL), + then=Value(EventStage.DREF_APPEAL_ONLY), ), When( Exists(FieldReport.objects.filter(event=OuterRef("pk"))), @@ -1676,7 +1677,10 @@ def get_queryset(self): # to avoid extra queries in serializer. stage_appeal_id=Case( When( - stage=EventStage.EMERGENCY_APPEAL, + stage__in=[ + EventStage.EMERGENCY_APPEAL, + EventStage.DREF_APPEAL_ONLY, + ], then=Subquery(latest_appeal_qs.values("id")[:1]), ), default=Value(None), @@ -1737,6 +1741,10 @@ def get_queryset(self): "countries", queryset=Country.objects.select_related("region"), ), + Prefetch( + "districts", + queryset=District.objects.select_related("country"), + ), Prefetch( "key_figures", queryset=KeyFigure.objects.all(), @@ -1745,5 +1753,13 @@ def get_queryset(self): "contacts", queryset=EventContact.objects.all(), ), + Prefetch( + "links", + queryset=EventLink.objects.all(), + ), + Prefetch( + "featured_documents", + queryset=EventFeaturedDocument.objects.order_by("-id"), + ), ) ) diff --git a/api/enums.py b/api/enums.py index 115855b4d..6d9b843c2 100644 --- a/api/enums.py +++ b/api/enums.py @@ -21,4 +21,5 @@ "profile_org_types": models.Profile.OrgTypes, "supporting_type": models.CountrySupportingPartner.SupportingPartnerType, "event_source": models.Event.EventSource, + "emergency_stage": models.EventStage, } diff --git a/api/management/commands/index_and_notify.py b/api/management/commands/index_and_notify.py index 606c85d6e..0156bb792 100644 --- a/api/management/commands/index_and_notify.py +++ b/api/management/commands/index_and_notify.py @@ -1036,7 +1036,7 @@ def handle(self, *args, **options): condR = Q(real_data_update__gte=time_diff) # instead of modified at cond2 = ~Q(previous_update__gte=time_diff_1_day) # negate (~) no previous_update in the last day, so send once a day condF = Q( - source=Event.EventSource.NEW_REPORT + source=Event.EventSource.NEW_FIELD_REPORT ) # exclude those events that were generated from field reports, to avoid 2x notif. condE = Q(status=CronJobStatus.ERRONEOUS) diff --git a/api/management/commands/ingest_mdb.py b/api/management/commands/ingest_mdb.py index b21b78abf..0784016f1 100644 --- a/api/management/commands/ingest_mdb.py +++ b/api/management/commands/ingest_mdb.py @@ -248,7 +248,7 @@ def handle(self, *args, **options): "dtype": report_dtype, "disaster_start_date": timezone.now(), "auto_generated": True, - "source": Event.EventSource.REPORT_INGEST, + "source": Event.EventSource.FIELD_REPORT_DMIS_INGEST, } event = Event(**event_record) event.save() diff --git a/api/management/commands/migrate_event_source.py b/api/management/commands/migrate_event_source.py deleted file mode 100644 index 89af9d8e8..000000000 --- a/api/management/commands/migrate_event_source.py +++ /dev/null @@ -1,28 +0,0 @@ -from django.core.management.base import BaseCommand - -from api.models import Event - - -class Command(BaseCommand): - help = "Update event sources based on name prefix and auto-generated status" - - def handle(self, *args, **options): - queryset = Event.objects.filter(auto_generated=True, source=0) - self.stdout.write( - self.style.NOTICE( - f"{queryset.count()} events will be assigned a source (GDACS or Manual Input) based on name prefix." - ) - ) - to_update = [] - for event in queryset.iterator(): - if event.name.startswith("GDACS"): - event.source = Event.EventSource.GDACS - self.stdout.write(self.style.NOTICE(f"Updating {event.name} source to {Event.EventSource.GDACS.label}")) - else: - event.source = Event.EventSource.MANUAL_INPUT - self.stdout.write(self.style.NOTICE(f"Updating {event.name} source to {Event.EventSource.MANUAL_INPUT.label}")) - event.auto_generated = False - to_update.append(event) - - updated_event = Event.objects.bulk_update(to_update, ["source", "auto_generated"]) - self.stdout.write(self.style.SUCCESS(f"Total {updated_event} Events source have been updated successfully.")) diff --git a/api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py b/api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py index 157b8dce9..5820ceb2f 100644 --- a/api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py +++ b/api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py @@ -1,27 +1,93 @@ -# Generated by Django 4.2.29 on 2026-05-12 08:28 +# Generated by Django 4.2.30 on 2026-05-21 04:49 from django.db import migrations, models -class Migration(migrations.Migration): +def migrate_sources(apps, _): + """ + Populate the source field for Event records: + - Non auto generated events are mapped to MANUAL_INPUT. + - Events that are auto generated and have auto_generated_source are mapped to the corresponding enum value. + - Events that are auto generated but do not have auto_generated_source are mapped to MANUAL_INPUT. + - For WHO, the who_guid field is populated with the id extracted from the auto_generated_source field. + """ + + Event = apps.get_model("api", "Event") + + for obj in Event.objects.iterator(): + update_fields = ["source"] + + if not obj.auto_generated: + source = 100 # MANUAL_INPUT + + else: + raw_source = (obj.auto_generated_source or "").lower() + if "gdacs" in raw_source: + source = 110 # GDACS + + elif "who.int" in raw_source: + source = 120 + + parts = raw_source.strip().split(".") + who_id = parts[-1].strip() + try: + obj.who_guid = int(who_id) + update_fields.append("who_guid") + except (ValueError, TypeError): + pass + elif "field report dmis ingest" in raw_source: + source = 130 # FIELD_REPORT_DMIS_INGEST + elif "field report admin" in raw_source: + source = 140 # FIELD_REPORT_ADMIN + elif "appeal" in raw_source: + source = 150 # APPEAL_ADMIN + elif "new field report" in raw_source: + source = 160 # NEW_FIELD_REPORT + else: + # Using title for backward compatibility, as some events have auto_generated_source empty but title containing "GDACS" + if obj.title and obj.title.lower().startswith("gdacs"): + source = 110 # GDACS + else: + source = 100 # MANUAL_INPUT + obj.auto_generated = False + update_fields.append("auto_generated") + obj.source = source + obj.save(update_fields=update_fields) + + +class Migration(migrations.Migration): dependencies = [ - ('api', '0231_alter_export_export_type'), + ("api", "0231_alter_export_export_type"), ] operations = [ - migrations.RemoveField( - model_name='event', - name='auto_generated_source', - ), migrations.AddField( - model_name='event', - name='source', - field=models.IntegerField(choices=[(100, 'Manual input'), (110, 'GDACs scraper'), (120, 'WHO scraper'), (130, 'Field report DMIS ingest'), (140, 'Field report admin'), (150, 'Appeal admin'), (160, 'New field report'), (170, 'DREF')], default=100, verbose_name='Event source'), + model_name="event", + name="source", + field=models.IntegerField( + choices=[ + (100, "Manual input"), + (110, "GDACs scraper"), + (120, "WHO scraper"), + (130, "Field report DMIS ingest"), + (140, "Field report admin"), + (150, "Appeal admin"), + (160, "New field report"), + (170, "DREF"), + ], + default=100, + verbose_name="Event source", + ), ), migrations.AddField( - model_name='event', - name='who_guid', - field=models.IntegerField(blank=True, null=True, verbose_name='Who guid'), + model_name="event", + name="who_guid", + field=models.IntegerField(blank=True, null=True, verbose_name="Who guid"), + ), + migrations.RunPython(migrate_sources, reverse_code=migrations.RunPython.noop), + migrations.RemoveField( + model_name="event", + name="auto_generated_source", ), ] diff --git a/api/models.py b/api/models.py index 4649f7d0d..d8facc360 100644 --- a/api/models.py +++ b/api/models.py @@ -768,6 +768,7 @@ class EventStage(models.IntegerChoices): DREF_OPERATIONAL_UPDATE = 3, _("DREF Operational Update") DREF_FINAL_REPORT = 4, _("DREF Final Report") FIELD_REPORT = 5, _("Field Report") + DREF_APPEAL_ONLY = 6, _("DREF") # NOTE: If ever in future we need to create an api to update the event table @@ -778,7 +779,6 @@ class Event(models.Model): class EventSource(models.IntegerChoices): - # TODO(Susilnem): use upper case MANUAL_INPUT = 100, _("Manual input") """MANUAL_INPUT: Event data manually entered by a user through the event administration interface.""" @@ -788,17 +788,17 @@ class EventSource(models.IntegerChoices): WHO = 120, _("WHO scraper") """WHO: Event data automatically ingested from the (WHO) scraper.""" - REPORT_INGEST = 130, _("Field report DMIS ingest") - """REPORT_INGEST: Event data imported through the DMIS field report.""" + FIELD_REPORT_DMIS_INGEST = 130, _("Field report DMIS ingest") + """FIELD_REPORT_DMIS_INGEST: Event data imported through the DMIS field report.""" - REPORT_ADMIN = 140, _("Field report admin") - """REPORT_ADMIN: Event data created or modified via the field report administration interface.""" + FIELD_REPORT_ADMIN = 140, _("Field report admin") + """FIELD_REPORT_ADMIN: Event data created or modified via the field report administration interface.""" APPEAL_ADMIN = 150, _("Appeal admin") """APPEAL_ADMIN: Event data created or managed through the appeal administration interface.""" - NEW_REPORT = 160, _("New field report") - """NEW_REPORT: Event data originating from newly created field reports.""" + NEW_FIELD_REPORT = 160, _("New field report") + """NEW_FIELD_REPORT: Event data originating from newly created field reports.""" DREF = 170, _("DREF") """DREF: Event originating records.""" diff --git a/api/serializers.py b/api/serializers.py index 8bf80cafe..b2870ccff 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -2147,9 +2147,12 @@ class EmergencyFieldReportSerializer( ): actions_taken = ActionsTakenSerializer(many=True) status_display = serializers.CharField(source="get_status_display", read_only=True) + appeal_display = serializers.CharField(source="get_appeal_display", read_only=True) + bulletin_display = serializers.CharField(source="get_bulletin_display", read_only=True) dtype = DisasterTypeSerializer(read_only=True) dref_display = serializers.CharField(source="get_dref_display", read_only=True) visibility_display = serializers.CharField(source="get_visibility_display", read_only=True) + imminent_dref_display = serializers.CharField(source="get_imminent_dref_display", read_only=True) contacts = FieldReportContactSerializer(many=True) countries = MiniCountrySerializer(many=True) districts = MiniDistrictSerializer(many=True) @@ -2166,6 +2169,12 @@ class Meta: "districts", "status", "status_display", + "appeal", + "appeal_display", + "bulletin", + "bulletin_display", + "imminent_dref", + "imminent_dref_display", "visibility", "visibility_display", "dref", @@ -2307,7 +2316,7 @@ def create_event(self, report): summary=report.description or "", disaster_start_date=report.start_date, auto_generated=True, - source=Event.EventSource.NEW_REPORT, + source=Event.EventSource.NEW_FIELD_REPORT, visibility=report.visibility, **{TRANSLATOR_ORIGINAL_LANGUAGE_FIELD_NAME: django_get_language()}, ) @@ -2774,7 +2783,7 @@ class Meta: fields = "__all__" -class DetailEmergencySerializer(serializers.ModelSerializer): +class DetailEmergencySerializer(ModelSerializer): from dref.serializers import EmergencyDrefSerializer contacts = EventContactSerializer(many=True, read_only=True) @@ -2783,6 +2792,9 @@ class DetailEmergencySerializer(serializers.ModelSerializer): ifrc_severity_level_display = serializers.CharField(source="get_ifrc_severity_level_display", read_only=True) visibility_display = serializers.CharField(source="get_visibility_display", read_only=True) source_display = serializers.CharField(source="get_source_display", read_only=True) + links = EventLinkSerializer(many=True, read_only=True) + districts = MiniDistrictSerializer(many=True) + featured_documents = EventFeaturedDocumentSerializer(many=True, read_only=True) # NOTE: Populated from Queryset using Annotate @@ -2806,8 +2818,11 @@ class DetailEmergencySerializer(serializers.ModelSerializer): class Meta: model = Event fields = ( + "id", + "slug", "name", "dtype", + "glide", "countries", "summary", "disaster_start_date", @@ -2817,10 +2832,11 @@ class Meta: "key_figures", "is_featured", "is_featured_region", + "tab_one_title", + "tab_two_title", + "tab_three_title", "hide_attached_field_reports", "hide_field_report_map", - "id", - "slug", "ifrc_severity_level", "ifrc_severity_level_display", "ifrc_severity_level_update_date", @@ -2829,6 +2845,9 @@ class Meta: "visibility", "visibility_display", "contacts", + "links", + "districts", + "featured_documents", "num_injured", "num_dead", "num_missing", @@ -2883,7 +2902,14 @@ def _get_stage_instance(self, event): .get(pk=event.stage_field_report_id) ) - elif stage == EventStage.EMERGENCY_APPEAL and event.stage_appeal_id: + elif ( + stage + in [ + EventStage.EMERGENCY_APPEAL, + EventStage.DREF_APPEAL_ONLY, + ] + and event.stage_appeal_id + ): instance = Appeal.objects.get(pk=event.stage_appeal_id) elif ( stage @@ -2940,7 +2966,10 @@ def get_field_report(self, event): @extend_schema_field(RelatedAppealSerializer()) def get_appeal(self, event): - if getattr(event, "stage", None) != EventStage.EMERGENCY_APPEAL: + if getattr(event, "stage", None) not in ( + EventStage.EMERGENCY_APPEAL, + EventStage.DREF_APPEAL_ONLY, + ): return None instance = self._get_stage_instance(event) return RelatedAppealSerializer(instance, context=self.context).data if instance else None diff --git a/api/test_views.py b/api/test_views.py index b76e0a75f..aadf5c565 100644 --- a/api/test_views.py +++ b/api/test_views.py @@ -1431,9 +1431,9 @@ def test_dref_application_takes_priority_over_field_report(self): self.assertIsNone(data["field_report"]) self.assertIsNone(data["dref"]["final_report_details"]) - def fallback_to_appeal_for_no_approved_dref(self): + def fallback_to_dref_appeal_for_no_approved_dref(self): """ - If no approved DREF application exists, but an ACTIVE appeal dref type exists, stage should resolve to EMERGENCY_APPEAL + If no approved DREF application exists, but an ACTIVE appeal dref type exists, stage should resolve to DREF_APPEAL_ONLY and not DREF. """ event = EventFactory.create(dtype=self.disaster_type) @@ -1441,7 +1441,7 @@ def fallback_to_appeal_for_no_approved_dref(self): event=event, dtype=self.disaster_type, status=AppealStatus.ACTIVE, - atype=AppealType.APPEAL, + atype=AppealType.DREF, ) data = self._get(event).data @@ -1468,7 +1468,7 @@ def fallback_to_appeal_for_no_approved_dref(self): ) data = self._get(event).data - self.assertEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + self.assertEqual(data["stage"], EventStage.DREF_APPEAL_ONLY) self.assertEqual(parse_datetime(data["first_field_report_created_at"]), first_fr.created_at) self.assertEqual(parse_datetime(data["latest_field_report_created_at"]), latest_fr.created_at) diff --git a/deployments/migrations/0095_alter_emergencyproject_event.py b/deployments/migrations/0095_alter_emergencyproject_event.py index 7595f257b..632195669 100644 --- a/deployments/migrations/0095_alter_emergencyproject_event.py +++ b/deployments/migrations/0095_alter_emergencyproject_event.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.30 on 2026-05-14 05:06 +# Generated by Django 4.2.30 on 2026-05-21 04:49 from django.db import migrations, models import django.db.models.deletion diff --git a/dref/migrations/0089_remove_dref_field_report_dref_event.py b/dref/migrations/0089_remove_dref_field_report_dref_event.py index d4c5afe5d..ad25ecffb 100644 --- a/dref/migrations/0089_remove_dref_field_report_dref_event.py +++ b/dref/migrations/0089_remove_dref_field_report_dref_event.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.30 on 2026-05-13 08:20 +# Generated by Django 4.2.30 on 2026-05-21 04:49 from django.db import migrations, models import django.db.models.deletion From 3305f644072f2f462955323bf71aef448ef018a2 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Fri, 22 May 2026 14:14:24 +0545 Subject: [PATCH 12/28] feat(emergency): add previous data on field report and dref ops update --- api/drf_views.py | 40 ++++--- ..._generated_source_event_source_and_more.py | 2 +- api/serializers.py | 66 +++++++++-- api/test_views.py | 64 +++++++++- dref/management/commands/migrate_event.py | 111 ------------------ dref/serializers.py | 53 +++++++++ 6 files changed, 198 insertions(+), 138 deletions(-) delete mode 100644 dref/management/commands/migrate_event.py diff --git a/api/drf_views.py b/api/drf_views.py index 57d39d3c2..e2b0a8ce8 100644 --- a/api/drf_views.py +++ b/api/drf_views.py @@ -1583,23 +1583,31 @@ class EmergencyViewset( def get_queryset(self): today = timezone.now().date() - active_appeal_qs = Appeal.objects.filter( - event=OuterRef("pk"), - status=AppealStatus.ACTIVE, + appeal_priority_qs = ( + Appeal.objects.filter( + event=OuterRef("pk"), + status__in=[AppealStatus.ACTIVE, AppealStatus.CLOSED], + ) + .annotate( + priority=Case( + When(status=AppealStatus.ACTIVE, then=Value(1)), + When(status=AppealStatus.CLOSED, then=Value(2)), + output_field=IntegerField(), + ) + ) + .order_by("priority", "-start_date") ) - active_dref_appeal_qs = active_appeal_qs.filter( + active_dref_appeal_qs = appeal_priority_qs.filter( atype=AppealType.DREF, ) - active_emergency_appeal_qs = active_appeal_qs.filter( + active_emergency_appeal_qs = appeal_priority_qs.filter( atype=AppealType.APPEAL, ) field_report_qs = FieldReport.objects.filter(event=OuterRef("pk")) - latest_appeal_qs = active_emergency_appeal_qs.order_by("-start_date") - approved_dref_qs = Dref.objects.filter( event=OuterRef("pk"), status=Dref.Status.APPROVED, @@ -1657,7 +1665,7 @@ def get_queryset(self): then=Value(EventStage.DREF_APPLICATION), ), # If there is an active appeal of DREF type, but no approved DREF yet, - # we consider the emergency to be in the Emergency Appeal stage. + # we consider the emergency to be in the Dref Appeal only stage. # Reaches here only if no approved DREF/ops-update/final-report exists # So an active appeal type DREF appeal with no approved DREF = DREF_APPEAL_ONLY stage. When( @@ -1677,11 +1685,12 @@ def get_queryset(self): # to avoid extra queries in serializer. stage_appeal_id=Case( When( - stage__in=[ - EventStage.EMERGENCY_APPEAL, - EventStage.DREF_APPEAL_ONLY, - ], - then=Subquery(latest_appeal_qs.values("id")[:1]), + stage=EventStage.EMERGENCY_APPEAL, + then=Subquery(active_emergency_appeal_qs.values("id")[:1]), + ), + When( + stage=EventStage.DREF_APPEAL_ONLY, + then=Subquery(active_dref_appeal_qs.values("id")[:1]), ), default=Value(None), output_field=IntegerField(null=True), @@ -1712,7 +1721,10 @@ def get_queryset(self): ), stage_ops_update_id=Case( When( - stage=EventStage.DREF_OPERATIONAL_UPDATE, + stage__in=[ + EventStage.DREF_OPERATIONAL_UPDATE, + EventStage.DREF_FINAL_REPORT, + ], then=Subquery( DrefOperationalUpdate.objects.filter( dref__id=OuterRef("stage_dref_id"), status=Dref.Status.APPROVED diff --git a/api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py b/api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py index 5820ceb2f..c81006a44 100644 --- a/api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py +++ b/api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py @@ -46,7 +46,7 @@ def migrate_sources(apps, _): source = 160 # NEW_FIELD_REPORT else: # Using title for backward compatibility, as some events have auto_generated_source empty but title containing "GDACS" - if obj.title and obj.title.lower().startswith("gdacs"): + if obj.name and obj.name.lower().startswith("gdacs"): source = 110 # GDACS else: source = 100 # MANUAL_INPUT diff --git a/api/serializers.py b/api/serializers.py index b2870ccff..980ca40a1 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -2142,6 +2142,22 @@ class Meta: # NOTE: Using this specific serializer for emergency page schema # Contains information from latest field report mostly and some fields from first field report +class EmergencyPreviousFieldReportSerializer( + serializers.ModelSerializer, +): + class Meta: + model = FieldReport + fields = ( + "id", + "event_id", + "fr_num", + "start_date", + "report_date", + "created_at", + "updated_at", + ) + + class EmergencyFieldReportSerializer( serializers.ModelSerializer, ): @@ -2783,6 +2799,17 @@ class Meta: fields = "__all__" +class TimelineEmergencyFieldReportSerializer(serializers.ModelSerializer): + class Meta: + model = FieldReport + fields = ( + "id", + "report_date", + "fr_num", + "start_date", + ) + + class DetailEmergencySerializer(ModelSerializer): from dref.serializers import EmergencyDrefSerializer @@ -2815,6 +2842,8 @@ class DetailEmergencySerializer(ModelSerializer): first_field_report_created_at = serializers.DateTimeField(read_only=True) latest_field_report_created_at = serializers.DateTimeField(read_only=True) + timeline_field_reports = serializers.SerializerMethodField() + class Meta: model = Event fields = ( @@ -2868,15 +2897,29 @@ class Meta: "dref", "first_field_report_created_at", "latest_field_report_created_at", + "timeline_field_reports", ) - def _get_stage_instance(self, event): - if hasattr(self, "_stage_instance_cache"): - return self._stage_instance_cache + @extend_schema_field(TimelineEmergencyFieldReportSerializer(many=True)) + def get_timeline_field_reports(self, event): + field_reports = ( + FieldReport.objects.filter(event=event) + .order_by( + "-updated_at", + "-fr_num", + ) + .values( + "id", + "report_date", + "fr_num", + "start_date", + ) + ) + serializer = TimelineEmergencyFieldReportSerializer(field_reports, many=True) + return serializer.data + def _get_stage_instance(self, event): stage = getattr(event, "stage") - instance = None - if stage == EventStage.FIELD_REPORT and event.stage_field_report_id: _first_field_report_queryset = FieldReport.objects.filter(event_id=OuterRef("event_id")).order_by( "created_at", "fr_num" @@ -2891,16 +2934,17 @@ def _get_stage_instance(self, event): first_fr_request_assistance=Subquery(_first_field_report_queryset.values("request_assistance")[:1]), ) .prefetch_related( - Prefetch("countries", queryset=Country.objects.select_related("region")), - Prefetch("districts", queryset=District.objects.select_related("country")), + "contacts", + "countries", + "districts", Prefetch( "actions_taken", queryset=ActionsTaken.objects.prefetch_related("actions"), ), - "contacts", ) .get(pk=event.stage_field_report_id) ) + return instance elif ( stage @@ -2911,6 +2955,7 @@ def _get_stage_instance(self, event): and event.stage_appeal_id ): instance = Appeal.objects.get(pk=event.stage_appeal_id) + return instance elif ( stage in [ @@ -2938,7 +2983,7 @@ def _get_stage_instance(self, event): ), ) .prefetch_related( - Prefetch("district", queryset=District.objects.select_related("country")), + "district", Prefetch( "planned_interventions", queryset=PlannedIntervention.objects.prefetch_related("indicators"), @@ -2950,8 +2995,7 @@ def _get_stage_instance(self, event): ) .get(pk=event.stage_dref_id) ) - self._stage_instance_cache = instance - return instance + return instance def get_stage_display(self, event): stage = getattr(event, "stage", None) diff --git a/api/test_views.py b/api/test_views.py index aadf5c565..11d9e62b4 100644 --- a/api/test_views.py +++ b/api/test_views.py @@ -1346,13 +1346,31 @@ def test_inactive_appeal_does_not_trigger_emergency_appeal_stage(self): AppealFactory.create( event=event, dtype=self.disaster_type, - status=AppealStatus.CLOSED, + status=AppealStatus.ARCHIVED, atype=AppealType.APPEAL, ) data = self._get(event).data self.assertNotEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + # CLOSED one also considered but ACTIVE takes over + AppealFactory.create( + event=event, + dtype=self.disaster_type, + status=AppealStatus.CLOSED, + atype=AppealType.APPEAL, + ) + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + self.assertEqual(data["appeal"]["status"], AppealStatus.CLOSED) + + self._active_emergency_appeal(event) + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + + # Active should take priority over closed and archived + self.assertEqual(data["appeal"]["status"], AppealStatus.ACTIVE) + def test_emergency_appeal_takes_priority_over_entire_dref_chain(self): event = EventFactory.create(dtype=self.disaster_type) dref = self._approved_dref(event) @@ -1472,3 +1490,47 @@ def fallback_to_dref_appeal_for_no_approved_dref(self): self.assertEqual(parse_datetime(data["first_field_report_created_at"]), first_fr.created_at) self.assertEqual(parse_datetime(data["latest_field_report_created_at"]), latest_fr.created_at) + + def test_field_report_with_timeline_field_reports(self): + event = EventFactory.create(dtype=self.disaster_type) + FieldReportFactory.create_batch( + 3, + event=event, + dtype=self.disaster_type, + ) + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.FIELD_REPORT, data) + self.assertIsNotNone(data["field_report"], data) + self.assertEqual(len(data["timeline_field_reports"]), 3) + + def test_dref_operational_update_with_timeline_ops_updates(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + DrefOperationalUpdateFactory.create_batch( + 3, + dref=dref, + status=Dref.Status.APPROVED, + disaster_type=self.disaster_type, + country=self.country, + ) + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.DREF_OPERATIONAL_UPDATE, data) + + dref_data = data.get("dref") + self.assertIsNotNone(dref_data, data) + + self.assertEqual(len(dref_data["timeline_operational_updates"]), 3) + + # NOTE: if Final report is created, stage should be DREF_FINAL_REPORT and previous operational updates should also show + DrefFinalReportFactory.create( + dref=dref, + status=Dref.Status.APPROVED, + disaster_type=self.disaster_type, + country=self.country, + ) + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.DREF_FINAL_REPORT) + dref_data = data.get("dref") + self.assertIsNotNone(dref_data) + self.assertIsNotNone(dref_data["final_report_details"]) + self.assertEqual(len(dref_data["timeline_operational_updates"]), 3) diff --git a/dref/management/commands/migrate_event.py b/dref/management/commands/migrate_event.py deleted file mode 100644 index 576aa401b..000000000 --- a/dref/management/commands/migrate_event.py +++ /dev/null @@ -1,111 +0,0 @@ -from django.core.management.base import BaseCommand - -from api.models import Appeal, AppealType -from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate - - -class Command(BaseCommand): - help = "Migrate related Event from Appeal / Dref Final / Operational Update to Dref" - - def handle(self, *args, **options): - self.stdout.write(self.style.NOTICE("Starting migration of events to Dref...")) - - appeal_map = dict( - Appeal.objects.filter( - atype=AppealType.DREF, - event__isnull=False, - ) - .exclude(code__isnull=True) - .values_list("code", "id") - ) - - appeal_event_map = dict( - Appeal.objects.filter( - atype=AppealType.DREF, - event__isnull=False, - ) - .exclude(code__isnull=True) - .values_list("code", "event_id") - ) - - final_report_map = dict(DrefFinalReport.objects.exclude(appeal_code__isnull=True).values_list("appeal_code", "id")) - - operational_update_map = dict( - DrefOperationalUpdate.objects.exclude(appeal_code__isnull=True).values_list("appeal_code", "id") - ) - - dref_queryset = Dref.objects.only("id", "title", "appeal_code", "event_id") - - self.stdout.write(self.style.NOTICE(f"Found {dref_queryset.count()} Dref records")) - self.stdout.write(self.style.NOTICE("\nMigrating Dref Event Mapping")) - - drefs_to_update = [] - - for dref in dref_queryset.iterator(): - code = dref.appeal_code - - self.stdout.write(self.style.NOTICE(f"\nDref: {dref.title} (id={dref.id})")) - - new_event_id = None - matched_with = None - - appeal_id = None - final_id = None - op_id = None - - self.stdout.write(self.style.NOTICE("\tChecking Appeal")) - - if code: - appeal_id = appeal_map.get(code) - new_event_id = appeal_event_map.get(code) - - if new_event_id: - matched_with = f"APPEAL (id:{appeal_id})" - self.stdout.write(self.style.SUCCESS(f"\t\tMatched with Appeal (id:{appeal_id})")) - else: - self.stdout.write(self.style.WARNING("\t\tNo match in Appeal")) - - if not new_event_id: - self.stdout.write(self.style.NOTICE("\tChecking Final Report")) - - final_id = final_report_map.get(code) - - if final_id: - new_event_id = appeal_event_map.get(code) - if new_event_id: - matched_with = f"DREF FINAL REPORT (id:{final_id})" - self.stdout.write(self.style.SUCCESS(f"\t\tMatched with Final Report (id:{final_id})")) - else: - self.stdout.write(self.style.WARNING("\t\tNo match in Final Report")) - - if not new_event_id: - self.stdout.write(self.style.NOTICE("\tChecking Operational Update")) - - op_id = operational_update_map.get(code) - - if op_id: - new_event_id = appeal_event_map.get(code) - if new_event_id: - matched_with = f"DREF OPERATIONAL UPDATE (id:{op_id})" - self.stdout.write(self.style.SUCCESS(f"\t\tMatched with Operational Update (id:{op_id})")) - else: - self.stdout.write(self.style.WARNING("\t\tNo match in Operational Update")) - - if not new_event_id: - self.stdout.write(self.style.WARNING("\tNo matching event found")) - continue - - if dref.event_id and dref.event_id != new_event_id: - self.stdout.write( - self.style.WARNING(f"\tConflict: existing_event_id={dref.event_id}, new_event_id={new_event_id} (skipped)") - ) - continue - - dref.event_id = new_event_id - drefs_to_update.append(dref) - - self.stdout.write(self.style.SUCCESS(f"\tUpdating event using {matched_with}")) - - Dref.objects.bulk_update(drefs_to_update, ["event_id"]) - - self.stdout.write(self.style.SUCCESS(f"\nSuccessfully updated {len(drefs_to_update)} Dref records\n")) diff --git a/dref/serializers.py b/dref/serializers.py index 3b44bb05e..2e2bb49f9 100644 --- a/dref/serializers.py +++ b/dref/serializers.py @@ -2289,6 +2289,7 @@ class Meta: "operation_end_date", "total_dref_allocation", "government_requested_assistance", + "num_assisted", "number_of_people_targeted", "number_of_people_affected", "total_targeted_population", @@ -2296,6 +2297,30 @@ class Meta: "estimated_number_of_affected_female", "estimated_number_of_affected_girls_under_18", "estimated_number_of_affected_boys_under_18", + "assisted_num_of_boys_under_18", + "assisted_num_of_girls_under_18", + "assisted_num_of_men", + "assisted_num_of_women", + "women", + "men", + "boys", + "people_assisted", + "date_of_approval", + "created_at", + "modified_at", + ) + + +class TimelineEmergencyDrefOperationalUpdateSerializer(serializers.ModelSerializer): + class Meta: + model = DrefOperationalUpdate + fields = ( + "id", + "summary_of_change", + "date_of_approval", + "operational_update_number", + "total_targeted_population", + "total_dref_allocation", ) @@ -2318,6 +2343,7 @@ class Meta: "status_display", "event_scope", "event_description", + "operational_update_number", "event_date", "appeal_code", "glide_code", @@ -2329,6 +2355,10 @@ class Meta: "new_operational_end_date", "total_dref_allocation", "government_requested_assistance", + "women", + "men", + "boys", + "people_assisted", "number_of_people_targeted", "number_of_people_affected", "total_targeted_population", @@ -2347,17 +2377,23 @@ class EmergencyDrefSerializer(serializers.ModelSerializer): proposed_action = ProposedActionSerializer(many=True, read_only=True) cover_image_file = DrefFileSerializer(source="cover_image", required=False, allow_null=True) disaster_type_details = DisasterTypeSerializer(source="disaster_type", read_only=True) + type_of_dref_display = serializers.CharField(source="get_type_of_dref_display", read_only=True) # Dref operational update operational_update_details = serializers.SerializerMethodField() # Dref Final report final_report_details = serializers.SerializerMethodField() + # Timeline of operational updates + timeline_operational_updates = serializers.SerializerMethodField() + class Meta: model = Dref fields = ( "id", "title", + "type_of_dref", + "type_of_dref_display", "disaster_type_details", "cover_image_file", "event_description", @@ -2379,19 +2415,36 @@ class Meta: # Key Figures "num_affected", "num_assisted", + "women", + "men", + "boys", "hazard_date_and_location", "amount_requested", + "total_cost", "total_targeted_population", "estimated_number_of_affected_male", "estimated_number_of_affected_female", "estimated_number_of_affected_girls_under_18", "estimated_number_of_affected_boys_under_18", + "people_assisted", # Operational timeframe date "hazard_date", "end_date", + "total_cost", + "date_of_approval", # For Response Type "event_date", + "timeline_operational_updates", + ) + + @extend_schema_field(TimelineEmergencyDrefOperationalUpdateSerializer(many=True)) + def get_timeline_operational_updates(self, obj): + ops_updates = DrefOperationalUpdate.objects.filter(dref=obj).order_by("operational_update_number") + serializer = TimelineEmergencyDrefOperationalUpdateSerializer( + ops_updates, + many=True, ) + return serializer.data @extend_schema_field(EmergencyDrefOperationalUpdateSerializer()) def get_operational_update_details(self, obj): From ad175d557d1183415e3b63cc649476cb064bf85f Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Mon, 25 May 2026 18:08:53 +0545 Subject: [PATCH 13/28] feat(emergency): Squash migration --- .../0095_alter_emergencyproject_event.py | 20 -------------- .../0096_alter_emergencyproject_event.py | 27 +++++++++++++++++++ ...089_remove_dref_field_report_dref_event.py | 24 ++++++++++------- 3 files changed, 42 insertions(+), 29 deletions(-) delete mode 100644 deployments/migrations/0095_alter_emergencyproject_event.py create mode 100644 deployments/migrations/0096_alter_emergencyproject_event.py diff --git a/deployments/migrations/0095_alter_emergencyproject_event.py b/deployments/migrations/0095_alter_emergencyproject_event.py deleted file mode 100644 index 632195669..000000000 --- a/deployments/migrations/0095_alter_emergencyproject_event.py +++ /dev/null @@ -1,20 +0,0 @@ -# Generated by Django 4.2.30 on 2026-05-21 04:49 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('api', '0232_remove_event_auto_generated_source_event_source_and_more'), - ('deployments', '0094_erureadinesstype_ns_contribution'), - ] - - operations = [ - migrations.AlterField( - model_name='emergencyproject', - name='event', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='emergency_projects', to='api.event', verbose_name='Event'), - ), - ] diff --git a/deployments/migrations/0096_alter_emergencyproject_event.py b/deployments/migrations/0096_alter_emergencyproject_event.py new file mode 100644 index 000000000..f699854dd --- /dev/null +++ b/deployments/migrations/0096_alter_emergencyproject_event.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.30 on 2026-05-25 12:01 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0232_remove_event_auto_generated_source_event_source_and_more"), + ( + "deployments", + "0095_molnix_appraisal_appraised_person_and_event_participation_constraint", + ), + ] + + operations = [ + migrations.AlterField( + model_name="emergencyproject", + name="event", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="emergency_projects", + to="api.event", + verbose_name="Event", + ), + ), + ] diff --git a/dref/migrations/0089_remove_dref_field_report_dref_event.py b/dref/migrations/0089_remove_dref_field_report_dref_event.py index ad25ecffb..db46fffe2 100644 --- a/dref/migrations/0089_remove_dref_field_report_dref_event.py +++ b/dref/migrations/0089_remove_dref_field_report_dref_event.py @@ -1,24 +1,30 @@ -# Generated by Django 4.2.30 on 2026-05-21 04:49 +# Generated by Django 4.2.30 on 2026-05-25 12:01 from django.db import migrations, models import django.db.models.deletion class Migration(migrations.Migration): - dependencies = [ - ('api', '0232_remove_event_auto_generated_source_event_source_and_more'), - ('dref', '0088_remove_identifiedneed_title_ar_and_more'), + ("api", "0232_remove_event_auto_generated_source_event_source_and_more"), + ("dref", "0088_remove_identifiedneed_title_ar_and_more"), ] operations = [ migrations.RemoveField( - model_name='dref', - name='field_report', + model_name="dref", + name="field_report", ), migrations.AddField( - model_name='dref', - name='event', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='event_dref', to='api.event', verbose_name='event'), + model_name="dref", + name="event", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="event_dref", + to="api.event", + verbose_name="event", + ), ), ] From 5cd9596580c3d33b873b6d092914c8a1d05e846b Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Thu, 28 May 2026 13:12:00 +0545 Subject: [PATCH 14/28] feat(dref): trigger event translation for approved dref --- api/models.py | 5 +++++ dref/utils.py | 1 + dref/views.py | 7 ++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/api/models.py b/api/models.py index d8facc360..b58bab95b 100644 --- a/api/models.py +++ b/api/models.py @@ -886,6 +886,11 @@ class EventSource(models.IntegerChoices): verbose_name=_("emergency response contact email"), null=True, blank=True, max_length=255 ) + # TYPING + + id: int + pk: int + class Meta: ordering = ( "-disaster_start_date", diff --git a/dref/utils.py b/dref/utils.py index 15fd74efe..5b5150760 100644 --- a/dref/utils.py +++ b/dref/utils.py @@ -69,4 +69,5 @@ def create_event_from_dref(dref: Dref) -> Event: event.countries.add(dref.country) event.districts.add(*dref.district.all()) + event.regions.add(dref.country.region) return event diff --git a/dref/views.py b/dref/views.py index 6d95435c5..0c0a47881 100644 --- a/dref/views.py +++ b/dref/views.py @@ -52,6 +52,7 @@ ) from dref.tasks import process_dref_translation from dref.utils import create_event_from_dref +from lang.serializers import TranslatedModelSerializerMixin from main.permissions import DenyGuestUserPermission logger = logging.getLogger(__name__) @@ -129,7 +130,11 @@ def get_approved(self, request, pk=None, version=None): if event: dref.event = event else: - dref.event = create_event_from_dref(dref) + event = create_event_from_dref(dref) + dref.event = event + # Translate the emergency instance + TranslatedModelSerializerMixin.trigger_field_translation(event) + dref.status = Dref.Status.APPROVED dref.save(update_fields=["event", "status"]) From 2265b0452d7725fca8260b9e1f02e44a04c3ef2d Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Mon, 1 Jun 2026 14:57:58 +0545 Subject: [PATCH 15/28] feat(dref): add test cases for dref approved event translation --- dref/test_views.py | 76 +++++++++++++++++++++++++++++++++++++++++++++- dref/utils.py | 9 ++++-- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/dref/test_views.py b/dref/test_views.py index 3c4a221d0..a0071523d 100644 --- a/dref/test_views.py +++ b/dref/test_views.py @@ -10,7 +10,7 @@ from django.core import management from rest_framework import status -from api.models import Country, DisasterType, District, Region, RegionName +from api.models import Country, DisasterType, District, Event, Region, RegionName from api.utils import get_model_name from deployments.factories.project import SectorFactory from deployments.factories.user import UserFactory @@ -30,6 +30,7 @@ ProposedAction, ) from dref.tasks import send_dref_email +from lang.serializers import TranslatedModelSerializerMixin from main.test_case import APITestCase @@ -2664,6 +2665,79 @@ def test_dref_final_report_finalize(self, mock_translation): self.assert_200(response) self.assertEqual(response.data["status"], Dref.Status.FINALIZED) + @patch.object( + TranslatedModelSerializerMixin, + "trigger_field_translation", + ) + def test_create_event_from_dref(self, mock_trigger_translation): + region = Region.objects.create(name=RegionName.ASIA_PACIFIC) + country = Country.objects.create(name="Test countrynpl", region=region) + district = District.objects.create(name="test district", country=country) + disaster_type = DisasterType.objects.create(name="test disaster") + dref = DrefFactory.create( + title="Test Title", + disaster_type=disaster_type, + event_description="Test event description", + event_date="2021-10-10", + glide_code="GLIDE123", + created_by=self.user, + country=country, + status=Dref.Status.FINALIZED, + type_of_dref=Dref.DrefType.ASSESSMENT, + ) + dref.district.set([district]) + url = f"/api/v2/dref/{dref.id}/approve/" + self.authenticate(self.root_user) + response = self.client.post(url) + self.assert_200(response) + + dref.refresh_from_db() + + dref_event_id = response.data["event"] + event_instance = Event.objects.get(id=dref_event_id) + + # Translation triggered + mock_trigger_translation.assert_called_once() + + translated_event = mock_trigger_translation.call_args.args[0] + + self.assertEqual( + translated_event.pk, + event_instance.pk, + ) + + self.assertEqual( + { + event_instance.name, + event_instance.dtype.id, + event_instance.summary, + event_instance.disaster_start_date.date(), + event_instance.glide, + event_instance.source, + }, + { + dref.title, + dref.disaster_type.id, + dref.event_description, + dref.event_date, + dref.glide_code, + Event.EventSource.DREF, + }, + ) + self.assertEqual( + list(event_instance.regions.values_list("id", flat=True)), + [dref.country.region.id], + ) + self.assertEqual( + list(event_instance.countries.values_list("id", flat=True)), + [dref.country.id], + ) + self.assertEqual( + list(event_instance.districts.values_list("id", flat=True)), + [district.id], + ) + self.assertTrue(event_instance.auto_generated) + User = get_user_model() diff --git a/dref/utils.py b/dref/utils.py index 5b5150760..d7e04ccbb 100644 --- a/dref/utils.py +++ b/dref/utils.py @@ -67,7 +67,12 @@ def create_event_from_dref(dref: Dref) -> Event: source=Event.EventSource.DREF, ) - event.countries.add(dref.country) + country = getattr(dref, "country", None) + if country: + event.countries.add(dref.country) + event.districts.add(*dref.district.all()) - event.regions.add(dref.country.region) + region = getattr(country, "region", None) + if region: + event.regions.add(region) return event From dd41e926c87abb79b656866c473d64224f7de7c6 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Mon, 15 Jun 2026 16:55:19 +0545 Subject: [PATCH 16/28] fix(test): use datetime for filling the date fields --- api/test_views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/test_views.py b/api/test_views.py index 11d9e62b4..6f215aedc 100644 --- a/api/test_views.py +++ b/api/test_views.py @@ -1,3 +1,4 @@ +import datetime import os import re import uuid From bb955bdf19ed9c650fa46d3fffd4cc8f3225da56 Mon Sep 17 00:00:00 2001 From: frozenhelium Date: Thu, 25 Jun 2026 00:42:37 +0545 Subject: [PATCH 17/28] fix(emergency): fill DREF data gaps on emergency pages - Set event severity from DREF disaster category on creation - Expose DREF contacts on the emergency payload - Expose imminent targeted population - Prefetch operational updates in the completed DREF operations endpoint - Filter timeline operational updates to approved and expose modified_at - Record date_of_approval when an operational update is approved --- assets | 2 +- dref/serializers.py | 72 ++++++++++++++++++++++++++++++++++++++++++++- dref/utils.py | 11 ++++++- dref/views.py | 20 +++++++++++-- 4 files changed, 100 insertions(+), 5 deletions(-) diff --git a/assets b/assets index a393d4ffb..780b7a6c0 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit a393d4ffbdf8f81b10ed899a688a37757b74df9b +Subproject commit 780b7a6c0efc7b6e353015308c701be2d7c2c419 diff --git a/dref/serializers.py b/dref/serializers.py index 2e2bb49f9..9a75ec5bd 100644 --- a/dref/serializers.py +++ b/dref/serializers.py @@ -2252,6 +2252,48 @@ class Meta(BaseDref3Serializer.Meta): model = DrefFinalReport +# Flatten the DREF's scalar contact fields into an api.EventContact-shaped list +# for the emergency page. Integrity contact is intentionally excluded. +DREF_EMERGENCY_CONTACT_GROUPS = ( + ("national_society_contact", "National Society Contact"), + ("ifrc_appeal_manager", "IFRC Appeal Manager"), + ("ifrc_project_manager", "IFRC Project Manager"), + ("ifrc_emergency", "IFRC Emergency"), + ("media_contact", "Media Contact"), +) + + +def get_dref_emergency_contacts(obj): + contacts = [] + for prefix, ctype in DREF_EMERGENCY_CONTACT_GROUPS: + name = getattr(obj, f"{prefix}_name", None) + title = getattr(obj, f"{prefix}_title", None) + email = getattr(obj, f"{prefix}_email", None) + phone = getattr(obj, f"{prefix}_phone_number", None) + if not any([name, title, email, phone]): + continue + contacts.append( + { + "id": f"{obj.id}-{prefix}", + "ctype": ctype, + "name": name or "", + "title": title or "", + "email": email or "", + "phone": phone or "", + } + ) + return contacts + + +class EmergencyDrefContactSerializer(serializers.Serializer): + id = serializers.CharField() + ctype = serializers.CharField() + name = serializers.CharField() + title = serializers.CharField() + email = serializers.CharField() + phone = serializers.CharField() + + # NOTE: This serializer is only used for the emergency page in GO, # which has a very specific and limited use case. # It is not intended to be a general-purpose serializer for DREF objects, @@ -2265,6 +2307,7 @@ class EmergencyDrefFinalReportSerializer(serializers.ModelSerializer): cover_image_file = DrefFileSerializer(source="cover_image", read_only=True) proposed_action = ProposedActionSerializer(many=True, required=False) disaster_type_details = DisasterTypeSerializer(source="disaster_type", read_only=True) + dref_contacts = serializers.SerializerMethodField() class Meta: model = DrefFinalReport @@ -2293,6 +2336,7 @@ class Meta: "number_of_people_targeted", "number_of_people_affected", "total_targeted_population", + "people_targeted_with_early_actions", "estimated_number_of_affected_male", "estimated_number_of_affected_female", "estimated_number_of_affected_girls_under_18", @@ -2308,8 +2352,13 @@ class Meta: "date_of_approval", "created_at", "modified_at", + "dref_contacts", ) + @extend_schema_field(EmergencyDrefContactSerializer(many=True)) + def get_dref_contacts(self, obj): + return get_dref_emergency_contacts(obj) + class TimelineEmergencyDrefOperationalUpdateSerializer(serializers.ModelSerializer): class Meta: @@ -2318,6 +2367,8 @@ class Meta: "id", "summary_of_change", "date_of_approval", + # fallback timeline date when date_of_approval is unset + "modified_at", "operational_update_number", "total_targeted_population", "total_dref_allocation", @@ -2331,6 +2382,7 @@ class EmergencyDrefOperationalUpdateSerializer(serializers.ModelSerializer): planned_interventions = PlannedInterventionSerializer(many=True, read_only=True) cover_image_file = DrefFileSerializer(source="cover_image", required=False, allow_null=True) disaster_type_details = DisasterTypeSerializer(source="disaster_type", read_only=True) + dref_contacts = serializers.SerializerMethodField() class Meta: model = DrefOperationalUpdate @@ -2362,12 +2414,18 @@ class Meta: "number_of_people_targeted", "number_of_people_affected", "total_targeted_population", + "people_targeted_with_early_actions", "estimated_number_of_affected_male", "estimated_number_of_affected_female", "estimated_number_of_affected_girls_under_18", "estimated_number_of_affected_boys_under_18", + "dref_contacts", ) + @extend_schema_field(EmergencyDrefContactSerializer(many=True)) + def get_dref_contacts(self, obj): + return get_dref_emergency_contacts(obj) + class EmergencyDrefSerializer(serializers.ModelSerializer): status_display = serializers.CharField(source="get_status_display", read_only=True) @@ -2387,6 +2445,8 @@ class EmergencyDrefSerializer(serializers.ModelSerializer): # Timeline of operational updates timeline_operational_updates = serializers.SerializerMethodField() + dref_contacts = serializers.SerializerMethodField() + class Meta: model = Dref fields = ( @@ -2422,6 +2482,7 @@ class Meta: "amount_requested", "total_cost", "total_targeted_population", + "people_targeted_with_early_actions", "estimated_number_of_affected_male", "estimated_number_of_affected_female", "estimated_number_of_affected_girls_under_18", @@ -2435,17 +2496,26 @@ class Meta: # For Response Type "event_date", "timeline_operational_updates", + # Contacts + "dref_contacts", ) @extend_schema_field(TimelineEmergencyDrefOperationalUpdateSerializer(many=True)) def get_timeline_operational_updates(self, obj): - ops_updates = DrefOperationalUpdate.objects.filter(dref=obj).order_by("operational_update_number") + ops_updates = DrefOperationalUpdate.objects.filter( + dref=obj, + status=Dref.Status.APPROVED, + ).order_by("operational_update_number") serializer = TimelineEmergencyDrefOperationalUpdateSerializer( ops_updates, many=True, ) return serializer.data + @extend_schema_field(EmergencyDrefContactSerializer(many=True)) + def get_dref_contacts(self, obj): + return get_dref_emergency_contacts(obj) + @extend_schema_field(EmergencyDrefOperationalUpdateSerializer()) def get_operational_update_details(self, obj): if not obj.operational_update_id: diff --git a/dref/utils.py b/dref/utils.py index d7e04ccbb..3baa13521 100644 --- a/dref/utils.py +++ b/dref/utils.py @@ -1,6 +1,7 @@ from django.conf import settings from django.contrib.postgres.aggregates import ArrayAgg from django.db import models +from django.utils import timezone from api.models import Event from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate @@ -57,7 +58,7 @@ def get_dref_users(): def create_event_from_dref(dref: Dref) -> Event: - event = Event.objects.create( + create_kwargs = dict( name=dref.title, dtype=dref.disaster_type, summary=dref.event_description or dref.event_scope or "", @@ -67,6 +68,14 @@ def create_event_from_dref(dref: Dref) -> Event: source=Event.EventSource.DREF, ) + # Dref.DisasterCategory and api.AlertLevel share the same 0/1/2 indices, so + # the value maps directly with no remap. + if dref.disaster_category is not None: + create_kwargs["ifrc_severity_level"] = dref.disaster_category + create_kwargs["ifrc_severity_level_update_date"] = dref.date_of_approval or timezone.now() + + event = Event.objects.create(**create_kwargs) + country = getattr(dref, "country", None) if country: event.countries.add(dref.country) diff --git a/dref/views.py b/dref/views.py index 0c0a47881..2633f9798 100644 --- a/dref/views.py +++ b/dref/views.py @@ -232,7 +232,8 @@ def get_approved(self, request, pk=None, version=None): raise serializers.ValidationError(gettext("Must be finalized before it can be approved.")) operational_update.status = Dref.Status.APPROVED - operational_update.save(update_fields=["status"]) + operational_update.date_of_approval = timezone.now().date() + operational_update.save(update_fields=["status", "date_of_approval"]) serializer = DrefOperationalUpdateSerializer(operational_update, context={"request": request}) return response.Response(serializer.data) @@ -367,7 +368,22 @@ class CompletedDrefOperationsViewSet(viewsets.ReadOnlyModelViewSet): DenyGuestUserPermission, ] filterset_class = CompletedDrefOperationsFilterSet - queryset = DrefFinalReport.objects.filter(status=Dref.Status.APPROVED).order_by("-created_at").distinct() + queryset = ( + DrefFinalReport.objects.filter(status=Dref.Status.APPROVED) + .select_related("country", "dref", "dref__country") + .prefetch_related( + # MiniDrefSerializer.operational_update_details reads this prefetched + # attr; without it DRF silently drops the field. + Prefetch( + "dref__drefoperationalupdate_set", + queryset=DrefOperationalUpdate.objects.select_related("country").order_by("-created_at"), + to_attr="prefetched_operational_updates", + ), + "dref__dreffinalreport__country", + ) + .order_by("-created_at") + .distinct() + ) def get_queryset(self): user = self.request.user From 778d587ec178844ed93e0e2aa957e4fd102db98c Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 24 Jun 2026 13:17:50 +0545 Subject: [PATCH 18/28] perf(emergency): remove ORDER BY from EXISTS subqueries, prefetch field reports and ops updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Split ordered querysets (used in Subquery) from unordered ones (used in Exists) — removes redundant ORDER BY from EXISTS() SQL, cutting EA stage response time ~25% - Add prefetch_related for field_reports and drefoperationalupdate_set in viewset and serializers — reduces list endpoint queries ~15% (N+1 -> batch) --- api/drf_views.py | 53 +++++++++++++++++++++++++++------------------ api/serializers.py | 23 +++++++++----------- dref/serializers.py | 11 ++++++---- 3 files changed, 49 insertions(+), 38 deletions(-) diff --git a/api/drf_views.py b/api/drf_views.py index e2b0a8ce8..85c51dd47 100644 --- a/api/drf_views.py +++ b/api/drf_views.py @@ -1598,14 +1598,18 @@ def get_queryset(self): .order_by("priority", "-start_date") ) - active_dref_appeal_qs = appeal_priority_qs.filter( - atype=AppealType.DREF, - ) + active_dref_appeal_qs = appeal_priority_qs.filter(atype=AppealType.DREF) + active_emergency_appeal_qs = appeal_priority_qs.filter(atype=AppealType.APPEAL) - active_emergency_appeal_qs = appeal_priority_qs.filter( - atype=AppealType.APPEAL, + appeal_exists_base_qs = Appeal.objects.filter( + event=OuterRef("pk"), + status__in=[AppealStatus.ACTIVE, AppealStatus.CLOSED], ) + # Unordered: used only in Exists() checks + active_dref_appeal_exists_qs = appeal_exists_base_qs.filter(atype=AppealType.DREF) + active_emergency_appeal_exists_qs = appeal_exists_base_qs.filter(atype=AppealType.APPEAL) + field_report_qs = FieldReport.objects.filter(event=OuterRef("pk")) approved_dref_qs = Dref.objects.filter( @@ -1623,6 +1627,16 @@ def get_queryset(self): status=Dref.Status.APPROVED, ).order_by("-created_at") + # Unordered: used only in Exists() checks + approved_ops_update_exists_qs = DrefOperationalUpdate.objects.filter( + dref__event=OuterRef("pk"), + status=Dref.Status.APPROVED, + ) + approved_final_report_exists_qs = DrefFinalReport.objects.filter( + dref__event=OuterRef("pk"), + status=Dref.Status.APPROVED, + ) + return ( super() .get_queryset() @@ -1649,15 +1663,15 @@ def get_queryset(self): # Stage stage=Case( When( - Exists(active_emergency_appeal_qs), + Exists(active_emergency_appeal_exists_qs), then=Value(EventStage.EMERGENCY_APPEAL), ), When( - Exists(approved_final_report_qs), + Exists(approved_final_report_exists_qs), then=Value(EventStage.DREF_FINAL_REPORT), ), When( - Exists(approved_ops_update_qs), + Exists(approved_ops_update_exists_qs), then=Value(EventStage.DREF_OPERATIONAL_UPDATE), ), When( @@ -1669,11 +1683,11 @@ def get_queryset(self): # Reaches here only if no approved DREF/ops-update/final-report exists # So an active appeal type DREF appeal with no approved DREF = DREF_APPEAL_ONLY stage. When( - Exists(active_dref_appeal_qs), + Exists(active_dref_appeal_exists_qs), then=Value(EventStage.DREF_APPEAL_ONLY), ), When( - Exists(FieldReport.objects.filter(event=OuterRef("pk"))), + Exists(field_report_qs), then=Value(EventStage.FIELD_REPORT), ), default=Value(None), @@ -1702,7 +1716,7 @@ def get_queryset(self): EventStage.DREF_OPERATIONAL_UPDATE, EventStage.DREF_FINAL_REPORT, ], - then=Subquery(approved_dref_qs.values("id")[:1]), + then=Subquery(approved_dref_qs.order_by("-created_at").values("id")[:1]), ), default=Value(None), output_field=IntegerField(null=True), @@ -1710,11 +1724,7 @@ def get_queryset(self): stage_final_report_id=Case( When( stage=EventStage.DREF_FINAL_REPORT, - then=Subquery( - DrefFinalReport.objects.filter( - dref__id=OuterRef("stage_dref_id"), status=Dref.Status.APPROVED - ).values("id")[:1] - ), + then=Subquery(approved_final_report_qs.values("id")[:1]), ), default=Value(None), output_field=IntegerField(null=True), @@ -1725,11 +1735,7 @@ def get_queryset(self): EventStage.DREF_OPERATIONAL_UPDATE, EventStage.DREF_FINAL_REPORT, ], - then=Subquery( - DrefOperationalUpdate.objects.filter( - dref__id=OuterRef("stage_dref_id"), status=Dref.Status.APPROVED - ).values("id")[:1] - ), + then=Subquery(approved_ops_update_qs.values("id")[:1]), ), default=Value(None), output_field=IntegerField(null=True), @@ -1773,5 +1779,10 @@ def get_queryset(self): "featured_documents", queryset=EventFeaturedDocument.objects.order_by("-id"), ), + Prefetch( + "field_reports", + queryset=FieldReport.objects.order_by("-updated_at", "-fr_num"), + to_attr="prefetched_timeline_field_reports", + ), ) ) diff --git a/api/serializers.py b/api/serializers.py index 980ca40a1..61cc5dc1e 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -2902,19 +2902,9 @@ class Meta: @extend_schema_field(TimelineEmergencyFieldReportSerializer(many=True)) def get_timeline_field_reports(self, event): - field_reports = ( - FieldReport.objects.filter(event=event) - .order_by( - "-updated_at", - "-fr_num", - ) - .values( - "id", - "report_date", - "fr_num", - "start_date", - ) - ) + field_reports = getattr(event, "prefetched_timeline_field_reports", None) + if field_reports is None: + field_reports = FieldReport.objects.filter(event=event).order_by("-updated_at", "-fr_num") serializer = TimelineEmergencyFieldReportSerializer(field_reports, many=True) return serializer.data @@ -2992,6 +2982,13 @@ def _get_stage_instance(self, event): "proposed_action", queryset=ProposedAction.objects.prefetch_related("activities"), ), + Prefetch( + "drefoperationalupdate_set", + queryset=DrefOperationalUpdate.objects.filter( + status=Dref.Status.APPROVED, + ).order_by("operational_update_number"), + to_attr="prefetched_timeline_ops_updates", + ), ) .get(pk=event.stage_dref_id) ) diff --git a/dref/serializers.py b/dref/serializers.py index 9a75ec5bd..2a6525e57 100644 --- a/dref/serializers.py +++ b/dref/serializers.py @@ -2502,10 +2502,13 @@ class Meta: @extend_schema_field(TimelineEmergencyDrefOperationalUpdateSerializer(many=True)) def get_timeline_operational_updates(self, obj): - ops_updates = DrefOperationalUpdate.objects.filter( - dref=obj, - status=Dref.Status.APPROVED, - ).order_by("operational_update_number") + ops_updates = getattr(obj, "prefetched_timeline_ops_updates", None) + if ops_updates is None: + ops_updates = DrefOperationalUpdate.objects.filter( + dref=obj, + status=Dref.Status.APPROVED, + ).order_by("operational_update_number") + serializer = TimelineEmergencyDrefOperationalUpdateSerializer( ops_updates, many=True, From cecae328ae7dcd6527b8eb4dcdd342c087ba061f Mon Sep 17 00:00:00 2001 From: sandeshit Date: Mon, 22 Jun 2026 15:03:47 +0545 Subject: [PATCH 19/28] feat(dref-summary): add new model and register for translation. - Create new model DrefSummary. - Register for translation. - Create migrations. --- dref/admin.py | 47 ++++++- dref/migrations/0090_drefsummary.py | 30 +++++ ...mmary_challenges_identified_ar_and_more.py | 123 ++++++++++++++++++ dref/models.py | 52 ++++++++ dref/translation.py | 12 ++ 5 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 dref/migrations/0090_drefsummary.py create mode 100644 dref/migrations/0091_drefsummary_challenges_identified_ar_and_more.py diff --git a/dref/admin.py b/dref/admin.py index 02e84fa7e..5ae9d5a42 100644 --- a/dref/admin.py +++ b/dref/admin.py @@ -1,13 +1,14 @@ from django.contrib import admin from reversion_compare.admin import CompareVersionAdmin -from lang.admin import TranslationAdmin +from lang.admin import TranslationAdmin, TranslationInlineModelAdmin from .models import ( Dref, DrefFile, DrefFinalReport, DrefOperationalUpdate, + DrefSummary, IdentifiedNeed, NationalSocietyAction, PlannedIntervention, @@ -125,8 +126,31 @@ class SourceInformationAdmin(admin.ModelAdmin): search_fields = ("source_name",) +class DrefSummaryInline(admin.StackedInline, TranslationInlineModelAdmin): + model = DrefSummary + extra = 0 + readonly_fields = ( + "status", + "hash", + "created_at", + "updated_at", + ) + fields = ( + "status", + "hash", + "situational_overview", + "operational_strategy", + "people_centered_approach", + "challenges_identified", + "lessons_learned", + "created_at", + "updated_at", + ) + + @admin.register(Dref) class DrefAdmin(CompareVersionAdmin, TranslationAdmin, admin.ModelAdmin): + inlines = [DrefSummaryInline] search_fields = ("title", "appeal_code") list_display = ( "title", @@ -301,3 +325,24 @@ def save_model(self, request, obj, form, change): @admin.register(ProposedAction) class ProposedActionAdmin(ReadOnlyMixin, admin.ModelAdmin): search_fields = ["action"] + + +@admin.register(DrefSummary) +class DrefSummaryAdmin(TranslationAdmin, admin.ModelAdmin): + list_display = ("dref", "status", "created_at", "updated_at") + list_filter = ("status",) + search_fields = ("dref__title", "dref__appeal_code") + readonly_fields = ("hash", "created_at", "updated_at") + autocomplete_fields = ("dref",) + fields = ( + "dref", + "status", + "hash", + "situational_overview", + "operational_strategy", + "people_centered_approach", + "challenges_identified", + "lessons_learned", + "created_at", + "updated_at", + ) diff --git a/dref/migrations/0090_drefsummary.py b/dref/migrations/0090_drefsummary.py new file mode 100644 index 000000000..8548f3564 --- /dev/null +++ b/dref/migrations/0090_drefsummary.py @@ -0,0 +1,30 @@ +# Generated by Django 4.2.30 on 2026-06-22 05:01 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('dref', '0089_remove_dref_field_report_dref_event'), + ] + + operations = [ + migrations.CreateModel( + name='DrefSummary', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('hash', models.CharField(max_length=64, unique=True)), + ('situational_overview', models.TextField(blank=True, null=True)), + ('operational_strategy', models.TextField(blank=True, null=True)), + ('people_centered_approach', models.TextField(blank=True, null=True)), + ('challenges_identified', models.TextField(blank=True, null=True)), + ('lessons_learned', models.TextField(blank=True, null=True)), + ('status', models.IntegerField(choices=[(100, 'Pending'), (200, 'Success'), (300, 'Failed')], default=100)), + ('dref', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='summary', to='dref.dref')), + ], + ), + ] diff --git a/dref/migrations/0091_drefsummary_challenges_identified_ar_and_more.py b/dref/migrations/0091_drefsummary_challenges_identified_ar_and_more.py new file mode 100644 index 000000000..0139d1f67 --- /dev/null +++ b/dref/migrations/0091_drefsummary_challenges_identified_ar_and_more.py @@ -0,0 +1,123 @@ +# Generated by Django 4.2.30 on 2026-06-22 08:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dref', '0090_drefsummary'), + ] + + operations = [ + migrations.AddField( + model_name='drefsummary', + name='challenges_identified_ar', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='challenges_identified_en', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='challenges_identified_es', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='challenges_identified_fr', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='lessons_learned_ar', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='lessons_learned_en', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='lessons_learned_es', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='lessons_learned_fr', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='operational_strategy_ar', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='operational_strategy_en', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='operational_strategy_es', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='operational_strategy_fr', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='people_centered_approach_ar', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='people_centered_approach_en', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='people_centered_approach_es', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='people_centered_approach_fr', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='situational_overview_ar', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='situational_overview_en', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='situational_overview_es', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='situational_overview_fr', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='drefsummary', + name='translation_module_original_language', + field=models.CharField(choices=[('en', 'English'), ('es', 'Spanish'), ('fr', 'French'), ('ar', 'Arabic')], default='en', help_text='Language used to create this entity', max_length=2, verbose_name='Entity Original language'), + ), + migrations.AddField( + model_name='drefsummary', + name='translation_module_skip_auto_translation', + field=models.BooleanField(default=False, help_text='Skip auto translation operation for this entity?', verbose_name='Skip auto translation'), + ), + ] diff --git a/dref/models.py b/dref/models.py index 4be868395..7f97bec87 100644 --- a/dref/models.py +++ b/dref/models.py @@ -1700,3 +1700,55 @@ def get_for(user, status=None): if status == Dref.Status.APPROVED: return queryset.filter(status=Dref.Status.APPROVED) return queryset + + +class DrefSummary(models.Model): + + class SummaryStatus(models.IntegerChoices): + PENDING = 100, _("Pending") + SUCCESS = 200, _("Success") + FAILED = 300, _("Failed") + + dref = models.OneToOneField( + Dref, + on_delete=models.CASCADE, + related_name="summary", + ) + + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + hash = models.CharField( + max_length=64, + unique=True, + ) + + situational_overview = models.TextField( + blank=True, + null=True, + ) + + operational_strategy = models.TextField( + blank=True, + null=True, + ) + + people_centered_approach = models.TextField( + blank=True, + null=True, + ) + + challenges_identified = models.TextField( + blank=True, + null=True, + ) + + lessons_learned = models.TextField( + blank=True, + null=True, + ) + + status = models.IntegerField( + choices=SummaryStatus.choices, + default=SummaryStatus.PENDING, + ) diff --git a/dref/translation.py b/dref/translation.py index 1c3b29d50..58a7fc986 100644 --- a/dref/translation.py +++ b/dref/translation.py @@ -5,6 +5,7 @@ DrefFile, DrefFinalReport, DrefOperationalUpdate, + DrefSummary, IdentifiedNeed, NationalSocietyAction, PlannedIntervention, @@ -116,6 +117,17 @@ class DrefOperationalUpdateTO(TranslationOptions): ) +@register(DrefSummary) +class DrefSummaryTO(TranslationOptions): + fields = ( + "situational_overview", + "operational_strategy", + "people_centered_approach", + "challenges_identified", + "lessons_learned", + ) + + @register(IdentifiedNeed) class IdentifiedNeedTO(TranslationOptions): fields = ("description",) From e242196130d18b84888666f3f404ef028c7c894b Mon Sep 17 00:00:00 2001 From: sandeshit Date: Mon, 22 Jun 2026 15:05:46 +0545 Subject: [PATCH 20/28] feat(serializer): create a serializer for summary. --- api/test_views.py | 37 +++++- assets | 2 +- dref/admin.py | 8 +- dref/factories/dref.py | 15 +++ dref/migrations/0090_drefsummary.py | 30 ++++- ...mmary_challenges_identified_ar_and_more.py | 123 ------------------ dref/models.py | 7 +- dref/serializers.py | 18 +++ 8 files changed, 104 insertions(+), 136 deletions(-) delete mode 100644 dref/migrations/0091_drefsummary_challenges_identified_ar_and_more.py diff --git a/api/test_views.py b/api/test_views.py index 6f215aedc..a54da83a1 100644 --- a/api/test_views.py +++ b/api/test_views.py @@ -31,8 +31,9 @@ DrefFactory, DrefFinalReportFactory, DrefOperationalUpdateFactory, + DrefSummaryFactory, ) -from dref.models import Dref, DrefFile +from dref.models import Dref, DrefFile, DrefSummary from main.test_case import APITestCase from per.factories import OpsLearningFactory @@ -1535,3 +1536,37 @@ def test_dref_operational_update_with_timeline_ops_updates(self): self.assertIsNotNone(dref_data) self.assertIsNotNone(dref_data["final_report_details"]) self.assertEqual(len(dref_data["timeline_operational_updates"]), 3) + + def test_dref_summary_fields_present_when_summary_exists(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + DrefSummaryFactory.create( + dref=dref, + status=DrefSummary.SummaryStatus.SUCCESS, + situational_overview="overview text", + operational_strategy="strategy text", + people_centered_approach="approach text", + challenges_identified="challenges text", + lessons_learned="lessons text", + ) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_APPLICATION) + summary = data["dref"]["summary"] + self.assertIsNotNone(summary) + self.assertEqual(summary["status"], DrefSummary.SummaryStatus.SUCCESS) + self.assertEqual(summary["situational_overview"], "overview text") + self.assertEqual(summary["operational_strategy"], "strategy text") + self.assertEqual(summary["people_centered_approach"], "approach text") + self.assertEqual(summary["challenges_identified"], "challenges text") + self.assertEqual(summary["lessons_learned"], "lessons text") + + def test_dref_summary_is_none_when_no_summary_exists(self): + event = EventFactory.create(dtype=self.disaster_type) + self._approved_dref(event) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_APPLICATION) + self.assertIsNone(data["dref"]["summary"]) diff --git a/assets b/assets index 780b7a6c0..5fbdb9ac6 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit 780b7a6c0efc7b6e353015308c701be2d7c2c419 +Subproject commit 5fbdb9ac6ad83472709f248c4bbe32a70587b577 diff --git a/dref/admin.py b/dref/admin.py index 5ae9d5a42..8ff4f67ce 100644 --- a/dref/admin.py +++ b/dref/admin.py @@ -131,13 +131,13 @@ class DrefSummaryInline(admin.StackedInline, TranslationInlineModelAdmin): extra = 0 readonly_fields = ( "status", - "hash", + "prompt_hash", "created_at", "updated_at", ) fields = ( "status", - "hash", + "prompt_hash", "situational_overview", "operational_strategy", "people_centered_approach", @@ -332,12 +332,12 @@ class DrefSummaryAdmin(TranslationAdmin, admin.ModelAdmin): list_display = ("dref", "status", "created_at", "updated_at") list_filter = ("status",) search_fields = ("dref__title", "dref__appeal_code") - readonly_fields = ("hash", "created_at", "updated_at") + readonly_fields = ("prompt_hash", "created_at", "updated_at") autocomplete_fields = ("dref",) fields = ( "dref", "status", - "hash", + "prompt_hash", "situational_overview", "operational_strategy", "people_centered_approach", diff --git a/dref/factories/dref.py b/dref/factories/dref.py index ceabd59d0..1ca5efb69 100644 --- a/dref/factories/dref.py +++ b/dref/factories/dref.py @@ -9,6 +9,7 @@ DrefFile, DrefFinalReport, DrefOperationalUpdate, + DrefSummary, IdentifiedNeed, NationalSocietyAction, PlannedIntervention, @@ -198,3 +199,17 @@ class Meta: model = ProposedAction proposed_type = fuzzy.FuzzyChoice(ProposedAction.Action) + + +class DrefSummaryFactory(factory.django.DjangoModelFactory): + class Meta: + model = DrefSummary + + dref = factory.SubFactory(DrefFactory) + prompt_hash = factory.Sequence(lambda n: f"{n:064d}") + status = DrefSummary.SummaryStatus.SUCCESS + situational_overview = fuzzy.FuzzyText(length=100) + operational_strategy = fuzzy.FuzzyText(length=100) + people_centered_approach = fuzzy.FuzzyText(length=100) + challenges_identified = fuzzy.FuzzyText(length=100) + lessons_learned = fuzzy.FuzzyText(length=100) diff --git a/dref/migrations/0090_drefsummary.py b/dref/migrations/0090_drefsummary.py index 8548f3564..e68b40311 100644 --- a/dref/migrations/0090_drefsummary.py +++ b/dref/migrations/0090_drefsummary.py @@ -1,7 +1,7 @@ -# Generated by Django 4.2.30 on 2026-06-22 05:01 +# Generated by Django 5.2.14 on 2026-06-26 02:06 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): @@ -17,13 +17,35 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created_at', models.DateTimeField(auto_now_add=True)), ('updated_at', models.DateTimeField(auto_now=True)), - ('hash', models.CharField(max_length=64, unique=True)), + ('prompt_hash', models.CharField(max_length=64, unique=True)), ('situational_overview', models.TextField(blank=True, null=True)), + ('situational_overview_en', models.TextField(blank=True, null=True)), + ('situational_overview_es', models.TextField(blank=True, null=True)), + ('situational_overview_fr', models.TextField(blank=True, null=True)), + ('situational_overview_ar', models.TextField(blank=True, null=True)), ('operational_strategy', models.TextField(blank=True, null=True)), + ('operational_strategy_en', models.TextField(blank=True, null=True)), + ('operational_strategy_es', models.TextField(blank=True, null=True)), + ('operational_strategy_fr', models.TextField(blank=True, null=True)), + ('operational_strategy_ar', models.TextField(blank=True, null=True)), ('people_centered_approach', models.TextField(blank=True, null=True)), + ('people_centered_approach_en', models.TextField(blank=True, null=True)), + ('people_centered_approach_es', models.TextField(blank=True, null=True)), + ('people_centered_approach_fr', models.TextField(blank=True, null=True)), + ('people_centered_approach_ar', models.TextField(blank=True, null=True)), ('challenges_identified', models.TextField(blank=True, null=True)), + ('challenges_identified_en', models.TextField(blank=True, null=True)), + ('challenges_identified_es', models.TextField(blank=True, null=True)), + ('challenges_identified_fr', models.TextField(blank=True, null=True)), + ('challenges_identified_ar', models.TextField(blank=True, null=True)), ('lessons_learned', models.TextField(blank=True, null=True)), - ('status', models.IntegerField(choices=[(100, 'Pending'), (200, 'Success'), (300, 'Failed')], default=100)), + ('lessons_learned_en', models.TextField(blank=True, null=True)), + ('lessons_learned_es', models.TextField(blank=True, null=True)), + ('lessons_learned_fr', models.TextField(blank=True, null=True)), + ('lessons_learned_ar', models.TextField(blank=True, null=True)), + ('status', models.IntegerField(choices=[(100, 'Pending'), (200, 'Processing'), (300, 'Success'), (400, 'Failed')], default=100)), + ('translation_module_original_language', models.CharField(choices=[('en', 'English'), ('es', 'Spanish'), ('fr', 'French'), ('ar', 'Arabic')], default='en', help_text='Language used to create this entity', max_length=2, verbose_name='Entity Original language')), + ('translation_module_skip_auto_translation', models.BooleanField(default=False, help_text='Skip auto translation operation for this entity?', verbose_name='Skip auto translation')), ('dref', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='summary', to='dref.dref')), ], ), diff --git a/dref/migrations/0091_drefsummary_challenges_identified_ar_and_more.py b/dref/migrations/0091_drefsummary_challenges_identified_ar_and_more.py deleted file mode 100644 index 0139d1f67..000000000 --- a/dref/migrations/0091_drefsummary_challenges_identified_ar_and_more.py +++ /dev/null @@ -1,123 +0,0 @@ -# Generated by Django 4.2.30 on 2026-06-22 08:40 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('dref', '0090_drefsummary'), - ] - - operations = [ - migrations.AddField( - model_name='drefsummary', - name='challenges_identified_ar', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='challenges_identified_en', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='challenges_identified_es', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='challenges_identified_fr', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='lessons_learned_ar', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='lessons_learned_en', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='lessons_learned_es', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='lessons_learned_fr', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='operational_strategy_ar', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='operational_strategy_en', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='operational_strategy_es', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='operational_strategy_fr', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='people_centered_approach_ar', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='people_centered_approach_en', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='people_centered_approach_es', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='people_centered_approach_fr', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='situational_overview_ar', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='situational_overview_en', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='situational_overview_es', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='situational_overview_fr', - field=models.TextField(blank=True, null=True), - ), - migrations.AddField( - model_name='drefsummary', - name='translation_module_original_language', - field=models.CharField(choices=[('en', 'English'), ('es', 'Spanish'), ('fr', 'French'), ('ar', 'Arabic')], default='en', help_text='Language used to create this entity', max_length=2, verbose_name='Entity Original language'), - ), - migrations.AddField( - model_name='drefsummary', - name='translation_module_skip_auto_translation', - field=models.BooleanField(default=False, help_text='Skip auto translation operation for this entity?', verbose_name='Skip auto translation'), - ), - ] diff --git a/dref/models.py b/dref/models.py index 7f97bec87..8d2f9280a 100644 --- a/dref/models.py +++ b/dref/models.py @@ -1706,8 +1706,9 @@ class DrefSummary(models.Model): class SummaryStatus(models.IntegerChoices): PENDING = 100, _("Pending") - SUCCESS = 200, _("Success") - FAILED = 300, _("Failed") + PROCESSING = 200, _("Processing") + SUCCESS = 300, _("Success") + FAILED = 400, _("Failed") dref = models.OneToOneField( Dref, @@ -1718,7 +1719,7 @@ class SummaryStatus(models.IntegerChoices): created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) - hash = models.CharField( + prompt_hash = models.CharField( max_length=64, unique=True, ) diff --git a/dref/serializers.py b/dref/serializers.py index 2a6525e57..a144ade2e 100644 --- a/dref/serializers.py +++ b/dref/serializers.py @@ -25,6 +25,7 @@ DrefFile, DrefFinalReport, DrefOperationalUpdate, + DrefSummary, IdentifiedNeed, NationalSocietyAction, PlannedIntervention, @@ -2427,6 +2428,20 @@ def get_dref_contacts(self, obj): return get_dref_emergency_contacts(obj) +class DrefSummarySerializer(ModelSerializer): + class Meta: + model = DrefSummary + fields = ( + "id", + "status", + "situational_overview", + "operational_strategy", + "people_centered_approach", + "challenges_identified", + "lessons_learned", + ) + + class EmergencyDrefSerializer(serializers.ModelSerializer): status_display = serializers.CharField(source="get_status_display", read_only=True) country_details = MiniCountrySerializer(source="country", read_only=True) @@ -2436,6 +2451,7 @@ class EmergencyDrefSerializer(serializers.ModelSerializer): cover_image_file = DrefFileSerializer(source="cover_image", required=False, allow_null=True) disaster_type_details = DisasterTypeSerializer(source="disaster_type", read_only=True) type_of_dref_display = serializers.CharField(source="get_type_of_dref_display", read_only=True) + summary = DrefSummarySerializer(read_only=True) # Dref operational update operational_update_details = serializers.SerializerMethodField() @@ -2498,6 +2514,8 @@ class Meta: "timeline_operational_updates", # Contacts "dref_contacts", + # Summary + "summary", ) @extend_schema_field(TimelineEmergencyDrefOperationalUpdateSerializer(many=True)) From a9ac989f8274556496f80cfc3911e050b21b59e0 Mon Sep 17 00:00:00 2001 From: sandeshit Date: Mon, 29 Jun 2026 16:02:14 +0545 Subject: [PATCH 21/28] feat(dref): retrigger from admin panel and latest source doc. - Retrigger from the admin panel. - Store the source doc type used for summary generation --- dref/admin.py | 25 +++++++++++++++++-- ...source_id_drefsummary_source_model_name.py | 23 +++++++++++++++++ dref/models.py | 17 ++++++++++++- 3 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 dref/migrations/0092_drefsummary_source_id_drefsummary_source_model_name.py diff --git a/dref/admin.py b/dref/admin.py index 8ff4f67ce..0cb0af2cd 100644 --- a/dref/admin.py +++ b/dref/admin.py @@ -1,6 +1,7 @@ -from django.contrib import admin +from django.contrib import admin, messages from reversion_compare.admin import CompareVersionAdmin +from api.utils import get_model_name from lang.admin import TranslationAdmin, TranslationInlineModelAdmin from .models import ( @@ -16,6 +17,7 @@ RiskSecurity, SourceInformation, ) +from .tasks import generate_dref_summary class ReadOnlyMixin: @@ -329,11 +331,12 @@ class ProposedActionAdmin(ReadOnlyMixin, admin.ModelAdmin): @admin.register(DrefSummary) class DrefSummaryAdmin(TranslationAdmin, admin.ModelAdmin): - list_display = ("dref", "status", "created_at", "updated_at") + list_display = ("dref", "status", "source_model_name", "source_id", "created_at", "updated_at") list_filter = ("status",) search_fields = ("dref__title", "dref__appeal_code") readonly_fields = ("prompt_hash", "created_at", "updated_at") autocomplete_fields = ("dref",) + actions = ["regenerate_summary"] fields = ( "dref", "status", @@ -346,3 +349,21 @@ class DrefSummaryAdmin(TranslationAdmin, admin.ModelAdmin): "created_at", "updated_at", ) + + @admin.action(description="Regenerate summary for selected DREFs") + def regenerate_summary(self, request, queryset): + """Re-trigger summary generation, replaying the source the summary was last built from.""" + for summary in queryset: + generate_dref_summary.delay( + summary.dref_id, + # Fall back to the Dref itself for rows generated before the + # source was tracked. + source_model_name=summary.source_model_name or get_model_name(Dref), + source_id=summary.source_id or summary.dref_id, + overwrite=True, + ) + self.message_user( + request, + f"Queued summary regeneration for {queryset.count()} DREF(s).", + messages.SUCCESS, + ) diff --git a/dref/migrations/0092_drefsummary_source_id_drefsummary_source_model_name.py b/dref/migrations/0092_drefsummary_source_id_drefsummary_source_model_name.py new file mode 100644 index 000000000..3ebd249e2 --- /dev/null +++ b/dref/migrations/0092_drefsummary_source_id_drefsummary_source_model_name.py @@ -0,0 +1,23 @@ +# Generated by Django 5.2.14 on 2026-06-29 08:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dref', '0091_drefsummary_challenges_identified_ar_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='drefsummary', + name='source_id', + field=models.PositiveBigIntegerField(blank=True, null=True, verbose_name='source id'), + ), + migrations.AddField( + model_name='drefsummary', + name='source_model_name', + field=models.CharField(blank=True, max_length=255, null=True, verbose_name='source model name'), + ), + ] diff --git a/dref/models.py b/dref/models.py index 8d2f9280a..c52787538 100644 --- a/dref/models.py +++ b/dref/models.py @@ -1719,7 +1719,22 @@ class SummaryStatus(models.IntegerChoices): created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) - prompt_hash = models.CharField( + # The source document the summary was last generated from. Stored so a + # retrigger (e.g. from the admin) can replay the same source instead of + # re-resolving the latest approved document. + source_model_name = models.CharField( + max_length=255, + verbose_name=_("source model name"), + blank=True, + null=True, + ) + source_id = models.PositiveBigIntegerField( + verbose_name=_("source id"), + blank=True, + null=True, + ) + + hash = models.CharField( max_length=64, unique=True, ) From 5306b8bbe6ecd33b5123e8dffb30814f7d0d22de Mon Sep 17 00:00:00 2001 From: sandeshit Date: Mon, 29 Jun 2026 16:03:18 +0545 Subject: [PATCH 22/28] feat(dref-summary): summary generation logic and scaffolding. - Summary generation logic. - Task for summary generation. - Triggers in views --- dref/summary.py | 391 ++++++++++++++++++++++++++++++++++++++++++++++++ dref/tasks.py | 68 +++++++++ dref/views.py | 23 ++- 3 files changed, 481 insertions(+), 1 deletion(-) create mode 100644 dref/summary.py diff --git a/dref/summary.py b/dref/summary.py new file mode 100644 index 000000000..493810232 --- /dev/null +++ b/dref/summary.py @@ -0,0 +1,391 @@ +""" +DREF AI summary generation. +""" + +import hashlib +import json +import logging +from typing import Callable, Dict, List, Optional + +import tiktoken +from django.conf import settings +from django.utils.functional import cached_property +from openai import AzureOpenAI + +from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate + +logger = logging.getLogger(__name__) + +ENCODING_NAME = "cl100k_base" + +MAX_OUTPUT_CHARS_PER_FIELD = 1500 +MAX_INPUT_TOKENS = 10000 + +# DrefSummary fields — order is the iteration order for prompt assembly. +SUMMARY_FIELDS: List[str] = [ + "situational_overview", + "operational_strategy", + "people_centered_approach", + "challenges_identified", + "lessons_learned", +] + +SYSTEM_MESSAGE = ( + "You are an IFRC expert analyst specializing in DREF (Disaster Response Emergency Fund) " + "operations. Analyze the provided DREF data and produce clear, professional humanitarian " + "summaries suitable for IFRC staff and National Society personnel. Use only the information " + "provided in the data; do not invent facts, figures, or details. Where supporting information " + "for a section is absent, return an empty string for that section rather than speculating." +) + +# Section prompt builders + + +def _build_situational_overview_prompt(**kwargs) -> str: + data = {k: v for k, v in kwargs.items() if v not in (None, "", [], {})} + data_json = json.dumps(data, indent=2, ensure_ascii=False, default=str) + return ( + 'Data for "situational_overview" — disaster context, affected areas, ' + f"scale of impact and strategic rationale for the operation:\n{data_json}" + ) + + +def _build_operational_strategy_prompt(**kwargs) -> str: + data = {k: v for k, v in kwargs.items() if v not in (None, "", [], {})} + data_json = json.dumps(data, indent=2, ensure_ascii=False, default=str) + return ( + 'Data for "operational_strategy" — operation objective, response strategy, ' + f"target population, timeframe and budget:\n{data_json}" + ) + + +def _build_people_centered_approach_prompt(**kwargs) -> str: + data = {k: v for k, v in kwargs.items() if v not in (None, "", [], {})} + data_json = json.dumps(data, indent=2, ensure_ascii=False, default=str) + return ( + 'Data for "people_centered_approach" — targeting criteria, beneficiary selection, ' + f"engagement and protection approach:\n{data_json}" + ) + + +def _build_challenges_identified_prompt(**kwargs) -> str: + data = {k: v for k, v in kwargs.items() if v not in (None, "", [], {})} + data_json = json.dumps(data, indent=2, ensure_ascii=False, default=str) + return ( + 'Data for "challenges_identified" — operational challenges, gaps, coordination ' + f"issues and security/safety risks recorded per planned intervention:\n{data_json}" + ) + + +def _build_lessons_learned_prompt(**kwargs) -> str: + data = {k: v for k, v in kwargs.items() if v not in (None, "", [], {})} + data_json = json.dumps(data, indent=2, ensure_ascii=False, default=str) + return ( + 'Data for "lessons_learned" — lessons learnt recorded per planned intervention ' f"(empty if none recorded):\n{data_json}" + ) + + +# Registry +SECTION_PROMPT_BUILDERS: Dict[str, Callable[..., str]] = { + "situational_overview": _build_situational_overview_prompt, + "operational_strategy": _build_operational_strategy_prompt, + "people_centered_approach": _build_people_centered_approach_prompt, + "challenges_identified": _build_challenges_identified_prompt, + "lessons_learned": _build_lessons_learned_prompt, +} + +GLOBAL_PROMPT = ( + "The DREF data above is organised by summary section. Using ONLY that data, write five concise " + "summary sections. Return a single JSON object (and nothing else) with exactly these keys:\n" + "\n" + ' "situational_overview": The disaster situation, affected areas and scale of impact, plus the ' + 'strategic rationale for the operation. Use the data under the "situational_overview" key. ' + "Write a coherent paragraph; include specific figures where available (e.g. people affected, displaced).\n" + ' "operational_strategy": The overall objective and strategic approach of the response. Use the ' + 'data under the "operational_strategy" key. State the target population, timeframe and headline ' + "interventions where available.\n" + ' "people_centered_approach": Who is targeted and how they are selected, engaged and protected. ' + 'Use the data under the "people_centered_approach" key, including the women/men/girls/boys ' + "breakdown where given.\n" + ' "challenges_identified": The challenges, gaps, coordination issues and security/safety risks. ' + 'Use the data under the "challenges_identified" key.\n' + ' "lessons_learned": The lessons learned from this operation. Use the data under the ' + '"lessons_learned" key. If none are recorded, return an empty string.\n' + "\n" + "Requirements:\n" + "- Each value must be plain text (no markdown, no bullet lists, no nested JSON).\n" + "- Each section should be one well-structured paragraph in professional humanitarian language.\n" + "- Be precise with any facts or numbers; never fabricate them.\n" + "- Return ONLY the JSON object, with no surrounding prose or code fences." +) + + +class DrefSummaryLLMClient: + @cached_property + def client(self): + return AzureOpenAI( + azure_endpoint=settings.AZURE_OPENAI_ENDPOINT, + api_key=settings.AZURE_OPENAI_KEY, + api_version="2023-05-15", + ) + + def get_response(self, messages) -> Optional[str]: + try: + response = self.client.chat.completions.create( + model=settings.AZURE_OPENAI_DEPLOYMENT_NAME, + messages=messages, + temperature=0.5, + ) + return response.choices[0].message.content + except Exception as e: + logger.error(f"Error while generating DREF summary response: {e}", exc_info=True) + return None + + +def count_tokens(text: str) -> int: + try: + return len(tiktoken.get_encoding(ENCODING_NAME).encode(text)) + except Exception as e: + logger.error(f"Error counting tokens for DREF summary: {e}") + return len(text) // 4 + + +def _field_val(obj, field_name): + """Return a model field's value, preferring human-readable display for choice fields.""" + v = getattr(obj, field_name, None) + if v in (None, ""): + return None + display = getattr(obj, f"get_{field_name}_display", None) + return display() if callable(display) else v + + +def _extract_fields(obj, field_names: List[str]) -> dict: + return {name: v for name in field_names if (v := _field_val(obj, name)) is not None} + + +# Shared field lists. + +SITUATIONAL_COMMON_FIELDS: List[str] = ["title", "event_description", "date_of_approval"] + +OPERATIONAL_COMMON_FIELDS: List[str] = [ + "operation_objective", + "response_strategy", + "total_targeted_population", + "people_in_need", + "event_date", +] + +PEOPLE_COMMON_FIELDS: List[str] = ["people_assisted", "selection_criteria"] + +DEMOGRAPHIC_FIELDS: List[str] = ["women", "men", "girls", "boys"] + + +class DrefSummaryGenerator: + """Assembles per-section prompts from a source document and produces all summaries.""" + + def __init__(self, client: Optional[DrefSummaryLLMClient] = None): + self.client = client or DrefSummaryLLMClient() + + @classmethod + def _model_label(cls, source_doc) -> str: + return f"{source_doc._meta.app_label}.{type(source_doc).__name__}" + + # Shared helpers — called by multiple extractors + + @staticmethod + def _situational_overview_kwargs(source_doc) -> dict: + """Build situational_overview kwargs — common across all document types. + + ``event_scope`` is omitted for Imminent DREFs since the scope is not + yet known at that stage. + """ + is_imminent = source_doc.type_of_dref == Dref.DrefType.IMMINENT + kwargs = _extract_fields(source_doc, SITUATIONAL_COMMON_FIELDS) + if source_doc.country: + kwargs["country"] = {"name": source_doc.country.name, "iso": source_doc.country.iso} + if source_doc.disaster_type: + kwargs["disaster_type"] = {"name": source_doc.disaster_type.name} + # event_scope is not yet known for Imminent DREFs, so it is excluded there. + if not is_imminent: + kwargs.update(_extract_fields(source_doc, ["event_scope"])) + return kwargs + + @staticmethod + def _challenges_and_lessons_kwargs(source_doc) -> Dict[str, dict]: + """Collect challenges and lessons from ``planned_interventions`` M2M. + + Only called for ``DrefFinalReport`` + """ + + def pi_title(pi): + display = getattr(pi, "get_title_display", None) + return display() if callable(display) else pi.title + + # Order explicitly: planned_interventions has no Meta.ordering, so an + # unordered .all() can return rows in different orders across queries, + # which would change the source hash and trigger needless regeneration. + planned = list(source_doc.planned_interventions.order_by("id")) + return { + "challenges_identified": { + "planned_interventions": [{"title": pi_title(pi), "challenges": pi.challenges} for pi in planned if pi.challenges] + or None, + }, + "lessons_learned": { + "planned_interventions": [ + {"title": pi_title(pi), "lessons_learnt": pi.lessons_learnt} for pi in planned if pi.lessons_learnt + ] + or None, + }, + } + + @classmethod + def _extract_dref_kwargs(cls, dref) -> Dict[str, dict]: + """Dref Application / Assessment / Imminent — three sections only. + + Challenges and lessons are not applicable at the application stage; + they are formally recorded only in the Final Report. + """ + is_imminent = dref.type_of_dref == Dref.DrefType.IMMINENT + operational = _extract_fields(dref, OPERATIONAL_COMMON_FIELDS + ["amount_requested", "end_date"]) + # Normalise to a single "operation_timeframe" key regardless of source field. + timeframe = _field_val(dref, "operation_timeframe_imminent" if is_imminent else "operation_timeframe") + if timeframe is not None: + operational["operation_timeframe"] = timeframe + return { + "situational_overview": cls._situational_overview_kwargs(dref), + "operational_strategy": operational, + "people_centered_approach": _extract_fields(dref, PEOPLE_COMMON_FIELDS), + } + + @classmethod + def _extract_dref_ops_kwargs(cls, ops) -> Dict[str, dict]: + """DrefOperationalUpdate — three sections; different field names from Dref. + + Includes demographic breakdown (women/men/girls/boys) in people_centered_approach. + Challenges and lessons are not generated for Operational Updates. + """ + operational = _extract_fields(ops, OPERATIONAL_COMMON_FIELDS + ["total_dref_allocation"]) + # Ops Update names these differently; normalise to the shared key names. + operational.update( + { + k: v + for k, v in { + "operation_end_date": _field_val(ops, "new_operational_end_date"), + "operation_timeframe": _field_val(ops, "total_operation_timeframe"), + }.items() + if v is not None + } + ) + return { + "situational_overview": cls._situational_overview_kwargs(ops), + "operational_strategy": operational, + "people_centered_approach": _extract_fields(ops, PEOPLE_COMMON_FIELDS + DEMOGRAPHIC_FIELDS), + } + + @classmethod + def _extract_dref_final_kwargs(cls, final) -> Dict[str, dict]: + """DrefFinalReport — all five sections. + + Challenges and lessons come from ``planned_interventions`` M2M via + ``_challenges_and_lessons_kwargs``; this is the only document type + where those two sections are generated. + """ + is_imminent = final.type_of_dref == Dref.DrefType.IMMINENT + operational = _extract_fields(final, OPERATIONAL_COMMON_FIELDS + ["total_dref_allocation", "operation_end_date"]) + # Normalise to a single "operation_timeframe" key regardless of source field. + timeframe = _field_val(final, "total_operation_timeframe_imminent" if is_imminent else "total_operation_timeframe") + if timeframe is not None: + operational["operation_timeframe"] = timeframe + kwargs = { + "situational_overview": cls._situational_overview_kwargs(final), + "operational_strategy": operational, + "people_centered_approach": _extract_fields(final, PEOPLE_COMMON_FIELDS + DEMOGRAPHIC_FIELDS), + } + kwargs.update(cls._challenges_and_lessons_kwargs(final)) + return kwargs + + @classmethod + def _get_section_kwargs(cls, source_doc) -> Dict[str, dict]: + """Dispatch to the model-specific extractor.""" + if isinstance(source_doc, Dref): + return cls._extract_dref_kwargs(source_doc) + if isinstance(source_doc, DrefOperationalUpdate): + return cls._extract_dref_ops_kwargs(source_doc) + if isinstance(source_doc, DrefFinalReport): + return cls._extract_dref_final_kwargs(source_doc) + return {} + + @classmethod + def build_source_text(cls, source_doc) -> str: + """JSON representation of all section kwargs for inspection and debugging.""" + return json.dumps(cls._get_section_kwargs(source_doc), indent=2, ensure_ascii=False, default=str) + + @classmethod + def compute_source_hash(cls, source_doc) -> str: + """Hash of all source content feeding the summary, for change detection.""" + model_label = cls._model_label(source_doc) + payload = {"model": model_label, "id": source_doc.id, "source": cls._get_section_kwargs(source_doc)} + content = json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str) + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + @staticmethod + def _parse_response(response: str) -> Dict[str, str]: + """Parse the model's JSON object, tolerating ```json code fences.""" + text = response.strip() + if text.startswith("```"): + text = text.split("\n", 1)[1] if "\n" in text else text + text = text.rsplit("```", 1)[0] + data = json.loads(text) + if not isinstance(data, dict): + raise ValueError("Expected a JSON object of summary fields") + return data + + def generate_all(self, source_doc) -> Dict[str, str]: + """Generate every summary for ``source_doc`` in a single LLM call. + + Always returns a value for every ``SUMMARY_FIELDS`` key (empty string + when a section has no content), so callers overwrite stale values + instead of leaving them in place on regeneration. + """ + empty_results: Dict[str, str] = {field_name: "" for field_name in SUMMARY_FIELDS} + + section_kwargs = self._get_section_kwargs(source_doc) + if not section_kwargs: + logger.info(f"No source content for ({type(source_doc).__name__}) ({source_doc.id}) summary; skipping generation.") + return empty_results + + # Call each builder with only its section's kwargs, then join blocks. + section_blocks = [SECTION_PROMPT_BUILDERS[section](**section_kwargs.get(section, {})) for section in SUMMARY_FIELDS] + user_content = "\n\n".join(section_blocks) + "\n\n" + GLOBAL_PROMPT + + # Respect the token budget: trim the data blocks if the prompt is too long. + if count_tokens(SYSTEM_MESSAGE + user_content) > MAX_INPUT_TOKENS: + logger.warning(f"({type(source_doc).__name__}) ({source_doc.id}) summary prompt too long; truncating source text.") + char_budget = MAX_INPUT_TOKENS * 4 - len(GLOBAL_PROMPT) - len(SYSTEM_MESSAGE) + data_blocks = "\n\n".join(section_blocks) + user_content = data_blocks[: max(char_budget, 0)] + "\n\n" + GLOBAL_PROMPT + + messages = [ + {"role": "system", "content": SYSTEM_MESSAGE}, + {"role": "user", "content": user_content}, + ] + + response = self.client.get_response(messages) + if not response: + raise RuntimeError(f"No LLM response for ({type(source_doc).__name__}) ({source_doc.id}) summary") + + try: + parsed = self._parse_response(response) + except (json.JSONDecodeError, ValueError) as e: + raise RuntimeError(f"Could not parse ({type(source_doc).__name__}) ({source_doc.id}) summary response: {e}") from e + + results = dict(empty_results) + for field_name in SUMMARY_FIELDS: + value = parsed.get(field_name) + if not isinstance(value, str): + continue + summary = value.strip() + if len(summary) > MAX_OUTPUT_CHARS_PER_FIELD: + summary = summary[:MAX_OUTPUT_CHARS_PER_FIELD].rstrip() + results[field_name] = summary + return results diff --git a/dref/tasks.py b/dref/tasks.py index 0b0d08c96..63ade5f3a 100644 --- a/dref/tasks.py +++ b/dref/tasks.py @@ -12,6 +12,7 @@ from .models import ( Dref, DrefFile, + DrefSummary, IdentifiedNeed, NationalSocietyAction, PlannedIntervention, @@ -21,6 +22,7 @@ RiskSecurity, SourceInformation, ) +from .summary import DrefSummaryGenerator from .utils import get_email_context logger = logging.getLogger(__name__) @@ -57,6 +59,72 @@ def send_dref_email(dref_id, users_emails, new_or_updated=""): ] +@shared_task +def generate_dref_summary(dref_id, source_model_name=None, source_id=None, overwrite=False): + """Generate and store the AI-assisted summaries for a DREF.""" + dref = Dref.objects.filter(id=dref_id).first() + if not dref: + logger.error(f"DREF not found for summary generation: ({dref_id})") + return False + + if source_model_name and source_id: + try: + source_model = apps.get_model(source_model_name) + source_doc = source_model.objects.filter(id=source_id).first() + except Exception: + logger.error( + f"Could not resolve source model ({source_model_name}) for DREF ({dref_id}) summary", + exc_info=True, + ) + return False + if not source_doc: + logger.error(f"Source document ({source_model_name}) ({source_id}) not found for DREF ({dref_id}) summary") + return False + else: + source_doc = dref + + # Normalise the source identity so it is always stored, even when the task + # is called with just a dref_id (source_doc is the Dref itself). + source_model_name = get_model_name(type(source_doc)) + source_id = source_doc.id + + generator = DrefSummaryGenerator() + source_hash = generator.compute_source_hash(source_doc) + + summary_instance = DrefSummary.objects.filter(dref=dref).first() + if ( + summary_instance + and not overwrite + and summary_instance.hash == source_hash + and summary_instance.status == DrefSummary.SummaryStatus.SUCCESS + ): + logger.info(f"DREF summary up to date for DREF ({dref_id}); skipping generation.") + return True + + if summary_instance is None: + summary_instance = DrefSummary(dref=dref) + summary_instance.hash = source_hash + summary_instance.source_model_name = source_model_name + summary_instance.source_id = source_id + + try: + logger.info( + f"Generating DREF summaries for DREF ({dref_id}) from ({source_model_name or 'dref.Dref'}) ({source_id or dref_id})" + ) + results = generator.generate_all(source_doc) + for field_name, value in results.items(): + setattr(summary_instance, field_name, value) + summary_instance.status = DrefSummary.SummaryStatus.SUCCESS + summary_instance.save() + logger.info(f"Successfully generated DREF summaries for DREF ({dref_id})") + return True + except Exception: + summary_instance.status = DrefSummary.SummaryStatus.FAILED + summary_instance.save() + logger.warning(f"DREF summary generation failed for DREF ({dref_id})", exc_info=True) + return False + + @shared_task def process_dref_translation(model_name, instance_pk): """ diff --git a/dref/views.py b/dref/views.py index 2633f9798..799cfc456 100644 --- a/dref/views.py +++ b/dref/views.py @@ -50,7 +50,7 @@ DrefShareUserSerializer, MiniDrefSerializer, ) -from dref.tasks import process_dref_translation +from dref.tasks import generate_dref_summary, process_dref_translation from dref.utils import create_event_from_dref from lang.serializers import TranslatedModelSerializerMixin from main.permissions import DenyGuestUserPermission @@ -137,6 +137,7 @@ def get_approved(self, request, pk=None, version=None): dref.status = Dref.Status.APPROVED dref.save(update_fields=["event", "status"]) + transaction.on_commit(lambda: generate_dref_summary.delay(dref.id)) return response.Response(DrefSerializer(dref, context={"request": request}).data) @@ -234,6 +235,16 @@ def get_approved(self, request, pk=None, version=None): operational_update.status = Dref.Status.APPROVED operational_update.date_of_approval = timezone.now().date() operational_update.save(update_fields=["status", "date_of_approval"]) + _ops_update_id = operational_update.pk + _dref_id = operational_update.dref_id + transaction.on_commit( + lambda: generate_dref_summary.delay( + _dref_id, + source_model_name=get_model_name(DrefOperationalUpdate), + source_id=_ops_update_id, + overwrite=True, + ) + ) serializer = DrefOperationalUpdateSerializer(operational_update, context={"request": request}) return response.Response(serializer.data) @@ -302,6 +313,16 @@ def get_approved(self, request, pk=None, version=None): final_report.dref.is_active = False final_report.date_of_approval = timezone.now().date() final_report.dref.save(update_fields=["is_active", "date_of_approval"]) + _final_report_id = final_report.pk + _dref_id = final_report.dref_id + transaction.on_commit( + lambda: generate_dref_summary.delay( + _dref_id, + source_model_name=get_model_name(DrefFinalReport), + source_id=_final_report_id, + overwrite=True, + ) + ) serializer = DrefFinalReportSerializer(final_report, context={"request": request}) return response.Response(serializer.data) From 597c182fcabd372f89e608bef16de2df20510d6e Mon Sep 17 00:00:00 2001 From: sandeshit Date: Mon, 29 Jun 2026 17:01:10 +0545 Subject: [PATCH 23/28] feat(migrations): merge migrations --- assets | 2 +- dref/admin.py | 12 ++- dref/factories/dref.py | 4 +- dref/migrations/0090_drefsummary.py | 6 +- ...source_id_drefsummary_source_model_name.py | 23 ------ dref/models.py | 15 ++-- dref/serializers.py | 5 ++ dref/summary.py | 7 +- dref/tasks.py | 80 ++++++++++++------- dref/views.py | 21 +++-- 10 files changed, 92 insertions(+), 83 deletions(-) delete mode 100644 dref/migrations/0092_drefsummary_source_id_drefsummary_source_model_name.py diff --git a/assets b/assets index 5fbdb9ac6..03bb6f64a 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit 5fbdb9ac6ad83472709f248c4bbe32a70587b577 +Subproject commit 03bb6f64ab4e668fff21b0c6d10b799b53742284 diff --git a/dref/admin.py b/dref/admin.py index 0cb0af2cd..e5849fb98 100644 --- a/dref/admin.py +++ b/dref/admin.py @@ -1,7 +1,6 @@ from django.contrib import admin, messages from reversion_compare.admin import CompareVersionAdmin -from api.utils import get_model_name from lang.admin import TranslationAdmin, TranslationInlineModelAdmin from .models import ( @@ -133,13 +132,13 @@ class DrefSummaryInline(admin.StackedInline, TranslationInlineModelAdmin): extra = 0 readonly_fields = ( "status", - "prompt_hash", + "source_hash", "created_at", "updated_at", ) fields = ( "status", - "prompt_hash", + "source_hash", "situational_overview", "operational_strategy", "people_centered_approach", @@ -334,13 +333,13 @@ class DrefSummaryAdmin(TranslationAdmin, admin.ModelAdmin): list_display = ("dref", "status", "source_model_name", "source_id", "created_at", "updated_at") list_filter = ("status",) search_fields = ("dref__title", "dref__appeal_code") - readonly_fields = ("prompt_hash", "created_at", "updated_at") + readonly_fields = ("source_hash", "created_at", "updated_at") autocomplete_fields = ("dref",) actions = ["regenerate_summary"] fields = ( "dref", "status", - "prompt_hash", + "source_hash", "situational_overview", "operational_strategy", "people_centered_approach", @@ -355,10 +354,9 @@ def regenerate_summary(self, request, queryset): """Re-trigger summary generation, replaying the source the summary was last built from.""" for summary in queryset: generate_dref_summary.delay( - summary.dref_id, # Fall back to the Dref itself for rows generated before the # source was tracked. - source_model_name=summary.source_model_name or get_model_name(Dref), + source_model_name=summary.source_model_name or DrefSummary.SourceModel.DREF, source_id=summary.source_id or summary.dref_id, overwrite=True, ) diff --git a/dref/factories/dref.py b/dref/factories/dref.py index 1ca5efb69..4ebb6d509 100644 --- a/dref/factories/dref.py +++ b/dref/factories/dref.py @@ -206,7 +206,9 @@ class Meta: model = DrefSummary dref = factory.SubFactory(DrefFactory) - prompt_hash = factory.Sequence(lambda n: f"{n:064d}") + source_hash = factory.Sequence(lambda n: f"{n:064d}") + source_model_name = DrefSummary.SourceModel.DREF + source_id = factory.SelfAttribute("dref.id") status = DrefSummary.SummaryStatus.SUCCESS situational_overview = fuzzy.FuzzyText(length=100) operational_strategy = fuzzy.FuzzyText(length=100) diff --git a/dref/migrations/0090_drefsummary.py b/dref/migrations/0090_drefsummary.py index e68b40311..401009b54 100644 --- a/dref/migrations/0090_drefsummary.py +++ b/dref/migrations/0090_drefsummary.py @@ -1,4 +1,4 @@ -# Generated by Django 5.2.14 on 2026-06-26 02:06 +# Generated by Django 5.2.14 on 2026-07-03 00:58 import django.db.models.deletion from django.db import migrations, models @@ -17,7 +17,9 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created_at', models.DateTimeField(auto_now_add=True)), ('updated_at', models.DateTimeField(auto_now=True)), - ('prompt_hash', models.CharField(max_length=64, unique=True)), + ('source_model_name', models.IntegerField(choices=[(100, 'Dref'), (200, 'Dref Operational Update'), (300, 'Dref Final Report')], verbose_name='source model name')), + ('source_id', models.PositiveBigIntegerField(verbose_name='source id')), + ('source_hash', models.CharField(max_length=64, unique=True)), ('situational_overview', models.TextField(blank=True, null=True)), ('situational_overview_en', models.TextField(blank=True, null=True)), ('situational_overview_es', models.TextField(blank=True, null=True)), diff --git a/dref/migrations/0092_drefsummary_source_id_drefsummary_source_model_name.py b/dref/migrations/0092_drefsummary_source_id_drefsummary_source_model_name.py deleted file mode 100644 index 3ebd249e2..000000000 --- a/dref/migrations/0092_drefsummary_source_id_drefsummary_source_model_name.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 5.2.14 on 2026-06-29 08:01 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('dref', '0091_drefsummary_challenges_identified_ar_and_more'), - ] - - operations = [ - migrations.AddField( - model_name='drefsummary', - name='source_id', - field=models.PositiveBigIntegerField(blank=True, null=True, verbose_name='source id'), - ), - migrations.AddField( - model_name='drefsummary', - name='source_model_name', - field=models.CharField(blank=True, max_length=255, null=True, verbose_name='source model name'), - ), - ] diff --git a/dref/models.py b/dref/models.py index c52787538..da4ff5f76 100644 --- a/dref/models.py +++ b/dref/models.py @@ -1710,6 +1710,11 @@ class SummaryStatus(models.IntegerChoices): SUCCESS = 300, _("Success") FAILED = 400, _("Failed") + class SourceModel(models.IntegerChoices): + DREF = 100, _("Dref") + DREF_OPERATIONAL_UPDATE = 200, _("Dref Operational Update") + DREF_FINAL_REPORT = 300, _("Dref Final Report") + dref = models.OneToOneField( Dref, on_delete=models.CASCADE, @@ -1722,19 +1727,15 @@ class SummaryStatus(models.IntegerChoices): # The source document the summary was last generated from. Stored so a # retrigger (e.g. from the admin) can replay the same source instead of # re-resolving the latest approved document. - source_model_name = models.CharField( - max_length=255, + source_model_name = models.IntegerField( verbose_name=_("source model name"), - blank=True, - null=True, + choices=SourceModel.choices, ) source_id = models.PositiveBigIntegerField( verbose_name=_("source id"), - blank=True, - null=True, ) - hash = models.CharField( + source_hash = models.CharField( max_length=64, unique=True, ) diff --git a/dref/serializers.py b/dref/serializers.py index a144ade2e..c8c8bd1ad 100644 --- a/dref/serializers.py +++ b/dref/serializers.py @@ -2429,16 +2429,21 @@ def get_dref_contacts(self, obj): class DrefSummarySerializer(ModelSerializer): + status_display = serializers.CharField(source="get_status_display", read_only=True) + class Meta: model = DrefSummary fields = ( "id", "status", + "status_display", "situational_overview", "operational_strategy", "people_centered_approach", "challenges_identified", "lessons_learned", + "created_at", + "updated_at", ) diff --git a/dref/summary.py b/dref/summary.py index 493810232..013b3f21c 100644 --- a/dref/summary.py +++ b/dref/summary.py @@ -12,6 +12,7 @@ from django.utils.functional import cached_property from openai import AzureOpenAI +from api.utils import get_model_name from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate logger = logging.getLogger(__name__) @@ -186,10 +187,6 @@ class DrefSummaryGenerator: def __init__(self, client: Optional[DrefSummaryLLMClient] = None): self.client = client or DrefSummaryLLMClient() - @classmethod - def _model_label(cls, source_doc) -> str: - return f"{source_doc._meta.app_label}.{type(source_doc).__name__}" - # Shared helpers — called by multiple extractors @staticmethod @@ -323,7 +320,7 @@ def build_source_text(cls, source_doc) -> str: @classmethod def compute_source_hash(cls, source_doc) -> str: """Hash of all source content feeding the summary, for change detection.""" - model_label = cls._model_label(source_doc) + model_label = get_model_name(type(source_doc)) payload = {"model": model_label, "id": source_doc.id, "source": cls._get_section_kwargs(source_doc)} content = json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str) return hashlib.sha256(content.encode("utf-8")).hexdigest() diff --git a/dref/tasks.py b/dref/tasks.py index 63ade5f3a..c7e0d328d 100644 --- a/dref/tasks.py +++ b/dref/tasks.py @@ -7,11 +7,14 @@ from api.utils import get_model_name from lang.tasks import translate_model_fields from main.translation import TRANSLATOR_ORIGINAL_LANGUAGE_FIELD_NAME +from main.utils import logger_context from notifications.notification import send_notification from .models import ( Dref, DrefFile, + DrefFinalReport, + DrefOperationalUpdate, DrefSummary, IdentifiedNeed, NationalSocietyAction, @@ -59,33 +62,50 @@ def send_dref_email(dref_id, users_emails, new_or_updated=""): ] +def _resolve_source_model(source_model_name): + """Map a ``DrefSummary.SourceModel`` choice to its model class.""" + if source_model_name == DrefSummary.SourceModel.DREF: + return Dref + elif source_model_name == DrefSummary.SourceModel.DREF_OPERATIONAL_UPDATE: + return DrefOperationalUpdate + elif source_model_name == DrefSummary.SourceModel.DREF_FINAL_REPORT: + return DrefFinalReport + return None + + @shared_task -def generate_dref_summary(dref_id, source_model_name=None, source_id=None, overwrite=False): - """Generate and store the AI-assisted summaries for a DREF.""" - dref = Dref.objects.filter(id=dref_id).first() - if not dref: - logger.error(f"DREF not found for summary generation: ({dref_id})") +def generate_dref_summary(source_model_name, source_id, overwrite=False): + """Generate and store the AI-assisted summaries for a DREF. + + The DREF is derived from the source document, so the source (a Dref, + DrefOperationalUpdate or DrefFinalReport) is the single source of truth for + what the summary is built from. + """ + source_model = _resolve_source_model(source_model_name) + if source_model is None: + logger.error( + "Could not resolve source model for DREF summary", + extra=logger_context({"source_model_name": source_model_name, "source_id": source_id}), + ) + return False + source_doc = source_model.objects.filter(id=source_id).first() + if not source_doc: + logger.error( + "Source document not found for DREF summary", + extra=logger_context({"source_model_name": source_model_name, "source_id": source_id}), + ) return False - if source_model_name and source_id: - try: - source_model = apps.get_model(source_model_name) - source_doc = source_model.objects.filter(id=source_id).first() - except Exception: - logger.error( - f"Could not resolve source model ({source_model_name}) for DREF ({dref_id}) summary", - exc_info=True, - ) - return False - if not source_doc: - logger.error(f"Source document ({source_model_name}) ({source_id}) not found for DREF ({dref_id}) summary") - return False - else: - source_doc = dref - - # Normalise the source identity so it is always stored, even when the task - # is called with just a dref_id (source_doc is the Dref itself). - source_model_name = get_model_name(type(source_doc)) + # Derive the DREF from the source: the Dref is its own source, while + # DrefOperationalUpdate and DrefFinalReport both point back via ``dref``. + dref = source_doc if isinstance(source_doc, Dref) else source_doc.dref + if not dref: + logger.error( + "DREF not found for summary generation", + extra=logger_context({"source_model_name": source_model_name, "source_id": source_id}), + ) + return False + dref_id = dref.id source_id = source_doc.id generator = DrefSummaryGenerator() @@ -95,7 +115,7 @@ def generate_dref_summary(dref_id, source_model_name=None, source_id=None, overw if ( summary_instance and not overwrite - and summary_instance.hash == source_hash + and summary_instance.source_hash == source_hash and summary_instance.status == DrefSummary.SummaryStatus.SUCCESS ): logger.info(f"DREF summary up to date for DREF ({dref_id}); skipping generation.") @@ -103,20 +123,22 @@ def generate_dref_summary(dref_id, source_model_name=None, source_id=None, overw if summary_instance is None: summary_instance = DrefSummary(dref=dref) - summary_instance.hash = source_hash + summary_instance.source_hash = source_hash summary_instance.source_model_name = source_model_name summary_instance.source_id = source_id try: - logger.info( - f"Generating DREF summaries for DREF ({dref_id}) from ({source_model_name or 'dref.Dref'}) ({source_id or dref_id})" - ) + logger.info(f"Generating DREF summaries for DREF ({dref_id}) from ({source_model_name}) ({source_id})") results = generator.generate_all(source_doc) for field_name, value in results.items(): setattr(summary_instance, field_name, value) summary_instance.status = DrefSummary.SummaryStatus.SUCCESS + # Generated summaries are always English; set the source language explicitly. + summary_instance.translation_module_original_language = "en" summary_instance.save() logger.info(f"Successfully generated DREF summaries for DREF ({dref_id})") + # Trigger translation eagerly since the summary is not saved through a serializer. + translate_model_fields.delay(get_model_name(DrefSummary), summary_instance.pk) return True except Exception: summary_instance.status = DrefSummary.SummaryStatus.FAILED diff --git a/dref/views.py b/dref/views.py index 799cfc456..480dd7c56 100644 --- a/dref/views.py +++ b/dref/views.py @@ -32,7 +32,13 @@ DrefOperationalUpdateFilter, DrefShareUserFilterSet, ) -from dref.models import Dref, DrefFile, DrefFinalReport, DrefOperationalUpdate +from dref.models import ( + Dref, + DrefFile, + DrefFinalReport, + DrefOperationalUpdate, + DrefSummary, +) from dref.permissions import ApproveDrefPermission from dref.serializers import ( AddDrefUserSerializer, @@ -137,7 +143,10 @@ def get_approved(self, request, pk=None, version=None): dref.status = Dref.Status.APPROVED dref.save(update_fields=["event", "status"]) - transaction.on_commit(lambda: generate_dref_summary.delay(dref.id)) + _dref_id = dref.id + transaction.on_commit( + lambda: generate_dref_summary.delay(source_model_name=DrefSummary.SourceModel.DREF, source_id=_dref_id) + ) return response.Response(DrefSerializer(dref, context={"request": request}).data) @@ -236,11 +245,9 @@ def get_approved(self, request, pk=None, version=None): operational_update.date_of_approval = timezone.now().date() operational_update.save(update_fields=["status", "date_of_approval"]) _ops_update_id = operational_update.pk - _dref_id = operational_update.dref_id transaction.on_commit( lambda: generate_dref_summary.delay( - _dref_id, - source_model_name=get_model_name(DrefOperationalUpdate), + source_model_name=DrefSummary.SourceModel.DREF_OPERATIONAL_UPDATE, source_id=_ops_update_id, overwrite=True, ) @@ -314,11 +321,9 @@ def get_approved(self, request, pk=None, version=None): final_report.date_of_approval = timezone.now().date() final_report.dref.save(update_fields=["is_active", "date_of_approval"]) _final_report_id = final_report.pk - _dref_id = final_report.dref_id transaction.on_commit( lambda: generate_dref_summary.delay( - _dref_id, - source_model_name=get_model_name(DrefFinalReport), + source_model_name=DrefSummary.SourceModel.DREF_FINAL_REPORT, source_id=_final_report_id, overwrite=True, ) From d6e0ff9ac442de25d93f1e10baaa3bd9305f6ae8 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Fri, 3 Jul 2026 14:27:57 +0545 Subject: [PATCH 24/28] feat(dref): add step for the processing status and use delay translation --- dref/models.py | 4 ++++ dref/tasks.py | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dref/models.py b/dref/models.py index da4ff5f76..2715ebaa2 100644 --- a/dref/models.py +++ b/dref/models.py @@ -1233,6 +1233,10 @@ class DrefOperationalUpdate(models.Model): source_information = models.ManyToManyField(SourceInformation, blank=True, verbose_name=_("Source Information")) __budget_file_id = None + # TYPING + id: int + budget_file_id: int | None + class Meta: verbose_name = _("Dref Operational Update") verbose_name_plural = _("Dref Operational Updates") diff --git a/dref/tasks.py b/dref/tasks.py index c7e0d328d..d4c2ae2cd 100644 --- a/dref/tasks.py +++ b/dref/tasks.py @@ -2,6 +2,7 @@ from celery import shared_task from django.apps import apps +from django.db import transaction from django.template.loader import render_to_string from api.utils import get_model_name @@ -126,6 +127,8 @@ def generate_dref_summary(source_model_name, source_id, overwrite=False): summary_instance.source_hash = source_hash summary_instance.source_model_name = source_model_name summary_instance.source_id = source_id + summary_instance.status = DrefSummary.SummaryStatus.PROCESSING + summary_instance.save() try: logger.info(f"Generating DREF summaries for DREF ({dref_id}) from ({source_model_name}) ({source_id})") @@ -133,12 +136,9 @@ def generate_dref_summary(source_model_name, source_id, overwrite=False): for field_name, value in results.items(): setattr(summary_instance, field_name, value) summary_instance.status = DrefSummary.SummaryStatus.SUCCESS - # Generated summaries are always English; set the source language explicitly. - summary_instance.translation_module_original_language = "en" summary_instance.save() logger.info(f"Successfully generated DREF summaries for DREF ({dref_id})") - # Trigger translation eagerly since the summary is not saved through a serializer. - translate_model_fields.delay(get_model_name(DrefSummary), summary_instance.pk) + transaction.on_commit(lambda: translate_model_fields.delay(get_model_name(DrefSummary), summary_instance.pk)) return True except Exception: summary_instance.status = DrefSummary.SummaryStatus.FAILED From 4f0e87d87c0d7376cd73d72d6361b58f9b9603de Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Tue, 7 Jul 2026 11:28:04 +0545 Subject: [PATCH 25/28] feat(dref): Add DummyClientLLM and redislock check - Add redislock check on dref summary generation - Update on dref/task for the latest dref source checks - Add flag USE_DUMMY_LLM_CLIENT for enable/disable LLM client - rename source_model_name to source in DrefSummary --- docker-compose.yml | 2 + dref/admin.py | 13 +- dref/factories/dref.py | 2 +- dref/migrations/0090_drefsummary.py | 2 +- dref/models.py | 11 +- dref/summary.py | 102 +++++---- dref/tasks.py | 189 +++++++++------- dref/test_summary.py | 327 ++++++++++++++++++++++++++++ dref/views.py | 32 +-- main/llm.py | 183 ++++++++++++++++ main/lock.py | 1 + main/settings.py | 3 + per/models.py | 3 + per/ops_learning_summary.py | 28 +-- 14 files changed, 716 insertions(+), 182 deletions(-) create mode 100644 dref/test_summary.py create mode 100644 main/llm.py diff --git a/docker-compose.yml b/docker-compose.yml index 6ff7c3f57..a9a8faf89 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,8 @@ x-server: &base_server_setup CACHE_TEST_REDIS_URL: ${CACHE_TEST_REDIS_URL:-redis://redis:6379/11} CACHE_MIDDLEWARE_SECONDS: ${CACHE_MIDDLEWARE_SECONDS:-600} ELASTIC_SEARCH_HOST: ${ELASTIC_SEARCH_HOST:-elasticsearch://elasticsearch:9200} + # LLM + USE_DUMMY_LLM_CLIENT: ${USE_DUMMY_LLM_CLIENT:-true} # Appeal API APPEALS_USER: ${APPEALS_USER:-} APPEALS_PASS: ${APPEALS_PASS:-} diff --git a/dref/admin.py b/dref/admin.py index e5849fb98..390b4c4d9 100644 --- a/dref/admin.py +++ b/dref/admin.py @@ -330,7 +330,7 @@ class ProposedActionAdmin(ReadOnlyMixin, admin.ModelAdmin): @admin.register(DrefSummary) class DrefSummaryAdmin(TranslationAdmin, admin.ModelAdmin): - list_display = ("dref", "status", "source_model_name", "source_id", "created_at", "updated_at") + list_display = ("dref", "status", "source", "source_id", "created_at", "updated_at") list_filter = ("status",) search_fields = ("dref__title", "dref__appeal_code") readonly_fields = ("source_hash", "created_at", "updated_at") @@ -351,15 +351,10 @@ class DrefSummaryAdmin(TranslationAdmin, admin.ModelAdmin): @admin.action(description="Regenerate summary for selected DREFs") def regenerate_summary(self, request, queryset): - """Re-trigger summary generation, replaying the source the summary was last built from.""" + """Re-trigger summary generation from the DREF's current latest approved source.""" + queryset.filter(status=DrefSummary.SummaryStatus.PROCESSING).update(status=DrefSummary.SummaryStatus.FAILED) for summary in queryset: - generate_dref_summary.delay( - # Fall back to the Dref itself for rows generated before the - # source was tracked. - source_model_name=summary.source_model_name or DrefSummary.SourceModel.DREF, - source_id=summary.source_id or summary.dref_id, - overwrite=True, - ) + generate_dref_summary.delay(dref_id=summary.dref_id, overwrite=True) self.message_user( request, f"Queued summary regeneration for {queryset.count()} DREF(s).", diff --git a/dref/factories/dref.py b/dref/factories/dref.py index 4ebb6d509..4cc61d27a 100644 --- a/dref/factories/dref.py +++ b/dref/factories/dref.py @@ -207,7 +207,7 @@ class Meta: dref = factory.SubFactory(DrefFactory) source_hash = factory.Sequence(lambda n: f"{n:064d}") - source_model_name = DrefSummary.SourceModel.DREF + source = DrefSummary.SourceModel.DREF source_id = factory.SelfAttribute("dref.id") status = DrefSummary.SummaryStatus.SUCCESS situational_overview = fuzzy.FuzzyText(length=100) diff --git a/dref/migrations/0090_drefsummary.py b/dref/migrations/0090_drefsummary.py index 401009b54..6d9b171ef 100644 --- a/dref/migrations/0090_drefsummary.py +++ b/dref/migrations/0090_drefsummary.py @@ -17,7 +17,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('created_at', models.DateTimeField(auto_now_add=True)), ('updated_at', models.DateTimeField(auto_now=True)), - ('source_model_name', models.IntegerField(choices=[(100, 'Dref'), (200, 'Dref Operational Update'), (300, 'Dref Final Report')], verbose_name='source model name')), + ('source', models.IntegerField(choices=[(100, 'Dref'), (200, 'Dref Operational Update'), (300, 'Dref Final Report')], help_text='The model this summary was generated from.', verbose_name='source')), ('source_id', models.PositiveBigIntegerField(verbose_name='source id')), ('source_hash', models.CharField(max_length=64, unique=True)), ('situational_overview', models.TextField(blank=True, null=True)), diff --git a/dref/models.py b/dref/models.py index 2715ebaa2..84f63b7fd 100644 --- a/dref/models.py +++ b/dref/models.py @@ -1661,6 +1661,9 @@ class DrefFinalReport(models.Model): ) __financial_report_id = None + # TYPING + id: int + class Meta: verbose_name = _("Dref Final Report") verbose_name_plural = _("Dref Final Reports") @@ -1728,11 +1731,9 @@ class SourceModel(models.IntegerChoices): created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) - # The source document the summary was last generated from. Stored so a - # retrigger (e.g. from the admin) can replay the same source instead of - # re-resolving the latest approved document. - source_model_name = models.IntegerField( - verbose_name=_("source model name"), + source = models.IntegerField( + verbose_name=_("source"), + help_text=_("The model this summary was generated from."), choices=SourceModel.choices, ) source_id = models.PositiveBigIntegerField( diff --git a/dref/summary.py b/dref/summary.py index 013b3f21c..4fae765ca 100644 --- a/dref/summary.py +++ b/dref/summary.py @@ -5,15 +5,14 @@ import hashlib import json import logging -from typing import Callable, Dict, List, Optional +from typing import Callable, Dict, List, Optional, Union import tiktoken -from django.conf import settings -from django.utils.functional import cached_property -from openai import AzureOpenAI +from django.db.models import F from api.utils import get_model_name -from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate +from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate, DrefSummary +from main.llm import get_dref_summary_llm_client logger = logging.getLogger(__name__) @@ -22,6 +21,16 @@ MAX_OUTPUT_CHARS_PER_FIELD = 1500 MAX_INPUT_TOKENS = 10000 + +# The models a DrefSummary can be generated from. +DrefSummarySource = Union[Dref, DrefOperationalUpdate, DrefFinalReport] + +SOURCE_BY_MODEL: Dict[type, DrefSummary.SourceModel] = { + Dref: DrefSummary.SourceModel.DREF, + DrefOperationalUpdate: DrefSummary.SourceModel.DREF_OPERATIONAL_UPDATE, + DrefFinalReport: DrefSummary.SourceModel.DREF_FINAL_REPORT, +} + # DrefSummary fields — order is the iteration order for prompt assembly. SUMMARY_FIELDS: List[str] = [ "situational_overview", @@ -121,28 +130,6 @@ def _build_lessons_learned_prompt(**kwargs) -> str: ) -class DrefSummaryLLMClient: - @cached_property - def client(self): - return AzureOpenAI( - azure_endpoint=settings.AZURE_OPENAI_ENDPOINT, - api_key=settings.AZURE_OPENAI_KEY, - api_version="2023-05-15", - ) - - def get_response(self, messages) -> Optional[str]: - try: - response = self.client.chat.completions.create( - model=settings.AZURE_OPENAI_DEPLOYMENT_NAME, - messages=messages, - temperature=0.5, - ) - return response.choices[0].message.content - except Exception as e: - logger.error(f"Error while generating DREF summary response: {e}", exc_info=True) - return None - - def count_tokens(text: str) -> int: try: return len(tiktoken.get_encoding(ENCODING_NAME).encode(text)) @@ -184,8 +171,8 @@ def _extract_fields(obj, field_names: List[str]) -> dict: class DrefSummaryGenerator: """Assembles per-section prompts from a source document and produces all summaries.""" - def __init__(self, client: Optional[DrefSummaryLLMClient] = None): - self.client = client or DrefSummaryLLMClient() + def __init__(self): + self.client = get_dref_summary_llm_client() # Shared helpers — called by multiple extractors @@ -302,8 +289,13 @@ def _extract_dref_final_kwargs(cls, final) -> Dict[str, dict]: return kwargs @classmethod - def _get_section_kwargs(cls, source_doc) -> Dict[str, dict]: - """Dispatch to the model-specific extractor.""" + def get_section_kwargs(cls, source_doc: DrefSummarySource) -> Dict[str, dict]: + """Dispatch to the model-specific extractor. + + Public so callers that need both the hash and the generated summary + for the same ``source_doc`` (see ``compute_source_hash``/``generate_all``) + can compute this once and pass it to both instead of extracting twice. + """ if isinstance(source_doc, Dref): return cls._extract_dref_kwargs(source_doc) if isinstance(source_doc, DrefOperationalUpdate): @@ -312,16 +304,47 @@ def _get_section_kwargs(cls, source_doc) -> Dict[str, dict]: return cls._extract_dref_final_kwargs(source_doc) return {} + @staticmethod + def get_latest_approved_source(dref: Dref) -> Optional[tuple[DrefSummary.SourceModel, DrefSummarySource]]: + """Most authoritative (source, source object) pair for ``dref``. + + Priority: Final Report > latest Operational Update > Dref itself. + """ + final_report = ( + DrefFinalReport.objects.select_related("country", "disaster_type") + .filter(dref=dref, status=Dref.Status.APPROVED) + .order_by("-created_at") + .first() + ) + if final_report: + return SOURCE_BY_MODEL[DrefFinalReport], final_report + + latest_ops_update = ( + DrefOperationalUpdate.objects.select_related("country", "disaster_type") + .filter(dref=dref, status=Dref.Status.APPROVED) + .order_by(F("operational_update_number").desc(nulls_last=True), "-created_at") + .first() + ) + if latest_ops_update: + return SOURCE_BY_MODEL[DrefOperationalUpdate], latest_ops_update + + if dref.status == Dref.Status.APPROVED: + return SOURCE_BY_MODEL[Dref], dref + + return None + @classmethod - def build_source_text(cls, source_doc) -> str: + def build_source_text(cls, source_doc: DrefSummarySource) -> str: """JSON representation of all section kwargs for inspection and debugging.""" - return json.dumps(cls._get_section_kwargs(source_doc), indent=2, ensure_ascii=False, default=str) + return json.dumps(cls.get_section_kwargs(source_doc), indent=2, ensure_ascii=False, default=str) @classmethod - def compute_source_hash(cls, source_doc) -> str: + def compute_source_hash(cls, source_doc: DrefSummarySource, section_kwargs: Optional[Dict[str, dict]] = None) -> str: """Hash of all source content feeding the summary, for change detection.""" model_label = get_model_name(type(source_doc)) - payload = {"model": model_label, "id": source_doc.id, "source": cls._get_section_kwargs(source_doc)} + if section_kwargs is None: + section_kwargs = cls.get_section_kwargs(source_doc) + payload = {"model": model_label, "id": source_doc.id, "source": section_kwargs} content = json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str) return hashlib.sha256(content.encode("utf-8")).hexdigest() @@ -337,16 +360,19 @@ def _parse_response(response: str) -> Dict[str, str]: raise ValueError("Expected a JSON object of summary fields") return data - def generate_all(self, source_doc) -> Dict[str, str]: + def generate_all(self, source_doc: DrefSummarySource, section_kwargs: Optional[Dict[str, dict]] = None) -> Dict[str, str]: """Generate every summary for ``source_doc`` in a single LLM call. Always returns a value for every ``SUMMARY_FIELDS`` key (empty string when a section has no content), so callers overwrite stale values - instead of leaving them in place on regeneration. + instead of leaving them in place on regeneration. ``section_kwargs`` + can be passed in by a caller that already computed it (e.g. via + ``compute_source_hash``) to avoid re-extracting it from ``source_doc``. """ empty_results: Dict[str, str] = {field_name: "" for field_name in SUMMARY_FIELDS} - section_kwargs = self._get_section_kwargs(source_doc) + if section_kwargs is None: + section_kwargs = self.get_section_kwargs(source_doc) if not section_kwargs: logger.info(f"No source content for ({type(source_doc).__name__}) ({source_doc.id}) summary; skipping generation.") return empty_results diff --git a/dref/tasks.py b/dref/tasks.py index d4c2ae2cd..698124eaa 100644 --- a/dref/tasks.py +++ b/dref/tasks.py @@ -1,12 +1,17 @@ import logging +from datetime import timedelta +from enum import Enum +from typing import Optional from celery import shared_task from django.apps import apps from django.db import transaction from django.template.loader import render_to_string +from django.utils import timezone from api.utils import get_model_name from lang.tasks import translate_model_fields +from main.lock import RedisLockKey, redis_lock from main.translation import TRANSLATOR_ORIGINAL_LANGUAGE_FIELD_NAME from main.utils import logger_context from notifications.notification import send_notification @@ -14,8 +19,6 @@ from .models import ( Dref, DrefFile, - DrefFinalReport, - DrefOperationalUpdate, DrefSummary, IdentifiedNeed, NationalSocietyAction, @@ -31,6 +34,22 @@ logger = logging.getLogger(__name__) +# The PROCESSING status has no TTL of its own (see comment in generate_dref_summary +# below), so a row can get stuck PROCESSING forever if a worker dies mid-generation. +# Treat it as stale past this, so a later trigger can take over. +PROCESSING_STALE_AFTER = timedelta(minutes=10) + + +class DrefSummaryGenerationResult(str, Enum): + """Outcome of a ``generate_dref_summary`` run.""" + + SUCCESS = "success" + SOURCE_NOT_FOUND = "source_not_found" + ALREADY_IN_PROGRESS = "already_in_progress" + UP_TO_DATE = "up_to_date" + FAILED = "failed" + SUPERSEDED = "superseded" + @shared_task def send_dref_email(dref_id, users_emails, new_or_updated=""): @@ -63,88 +82,106 @@ def send_dref_email(dref_id, users_emails, new_or_updated=""): ] -def _resolve_source_model(source_model_name): - """Map a ``DrefSummary.SourceModel`` choice to its model class.""" - if source_model_name == DrefSummary.SourceModel.DREF: - return Dref - elif source_model_name == DrefSummary.SourceModel.DREF_OPERATIONAL_UPDATE: - return DrefOperationalUpdate - elif source_model_name == DrefSummary.SourceModel.DREF_FINAL_REPORT: - return DrefFinalReport - return None - - -@shared_task -def generate_dref_summary(source_model_name, source_id, overwrite=False): +@shared_task(soft_time_limit=600, time_limit=630) +def generate_dref_summary(dref_id: int, overwrite: bool = False) -> DrefSummaryGenerationResult: """Generate and store the AI-assisted summaries for a DREF. - The DREF is derived from the source document, so the source (a Dref, - DrefOperationalUpdate or DrefFinalReport) is the single source of truth for - what the summary is built from. + Always (re)generates from whichever approved source is currently latest + for the DREF (see ``get_latest_approved_source``), rather than a + specific source passed in, so it self-corrects no matter which + approval triggered it or the order concurrent runs execute in. """ - source_model = _resolve_source_model(source_model_name) - if source_model is None: - logger.error( - "Could not resolve source model for DREF summary", - extra=logger_context({"source_model_name": source_model_name, "source_id": source_id}), - ) - return False - source_doc = source_model.objects.filter(id=source_id).first() - if not source_doc: - logger.error( - "Source document not found for DREF summary", - extra=logger_context({"source_model_name": source_model_name, "source_id": source_id}), - ) - return False - - # Derive the DREF from the source: the Dref is its own source, while - # DrefOperationalUpdate and DrefFinalReport both point back via ``dref``. - dref = source_doc if isinstance(source_doc, Dref) else source_doc.dref + dref = Dref.objects.select_related("country", "disaster_type").filter(id=dref_id).first() if not dref: - logger.error( - "DREF not found for summary generation", - extra=logger_context({"source_model_name": source_model_name, "source_id": source_id}), - ) - return False - dref_id = dref.id - source_id = source_doc.id - - generator = DrefSummaryGenerator() - source_hash = generator.compute_source_hash(source_doc) - - summary_instance = DrefSummary.objects.filter(dref=dref).first() - if ( - summary_instance - and not overwrite - and summary_instance.source_hash == source_hash - and summary_instance.status == DrefSummary.SummaryStatus.SUCCESS - ): - logger.info(f"DREF summary up to date for DREF ({dref_id}); skipping generation.") - return True - - if summary_instance is None: - summary_instance = DrefSummary(dref=dref) - summary_instance.source_hash = source_hash - summary_instance.source_model_name = source_model_name - summary_instance.source_id = source_id - summary_instance.status = DrefSummary.SummaryStatus.PROCESSING - summary_instance.save() - + logger.error("Dref not found for summary generation", extra=logger_context({"dref_id": dref_id})) + return DrefSummaryGenerationResult.SOURCE_NOT_FOUND + + # The Redis lock only needs to guard the brief read-check-mark-PROCESSING + # section below against a concurrent trigger for the same DREF (double- + # approve, admin retrigger, ...); it is released before the LLM call. + # The PROCESSING status written inside the lock is what actually blocks + # a second run for as long as generation takes - unlike the lock, it has + # no TTL, so it still holds even if generation outlives lock_expire. + with redis_lock(key=RedisLockKey.DREF_SUMMARY, id=dref_id) as acquired: + if not acquired: + logger.warning(f"DREF summary generation already in progress for DREF ({dref_id}); skipping.") + return DrefSummaryGenerationResult.ALREADY_IN_PROGRESS + + latest_source = DrefSummaryGenerator.get_latest_approved_source(dref) + if not latest_source: + logger.error(f"No approved source found for DREF ({dref_id}) summary") + return DrefSummaryGenerationResult.SOURCE_NOT_FOUND + source_type, source_obj = latest_source + + section_kwargs = DrefSummaryGenerator.get_section_kwargs(source_obj) + source_hash = DrefSummaryGenerator.compute_source_hash(source_obj, section_kwargs=section_kwargs) + summary_instance: Optional[DrefSummary] = DrefSummary.objects.filter(dref=dref).first() + + if ( + summary_instance + and summary_instance.status == DrefSummary.SummaryStatus.PROCESSING + and timezone.now() - summary_instance.updated_at < PROCESSING_STALE_AFTER + and summary_instance.source == source_type + and summary_instance.source_id == source_obj.id + ): + # Same source already in flight; a different one falls through to regenerate below. + logger.warning(f"DREF summary already in progress for DREF ({dref_id}); skipping.") + return DrefSummaryGenerationResult.ALREADY_IN_PROGRESS + + if ( + summary_instance + and not overwrite + and summary_instance.source_hash == source_hash + and summary_instance.status == DrefSummary.SummaryStatus.SUCCESS + ): + logger.info(f"DREF summary up to date for DREF ({dref_id}); skipping generation.") + return DrefSummaryGenerationResult.UP_TO_DATE + + if summary_instance is None: + summary_instance = DrefSummary(dref=dref) + + try: + summary_instance.source_hash = source_hash + summary_instance.source = source_type + summary_instance.source_id = source_obj.id + summary_instance.status = DrefSummary.SummaryStatus.PROCESSING + summary_instance.save() + except Exception: + logger.warning(f"Failed to mark DREF summary as processing for DREF ({dref_id})", exc_info=True) + if summary_instance.pk: + summary_instance.status = DrefSummary.SummaryStatus.FAILED + summary_instance.save() + return DrefSummaryGenerationResult.FAILED + + # Lock released. The (possibly slow) LLM call runs unlocked; the + # PROCESSING status set above is what a concurrent trigger checks. + own_marker = { + "pk": summary_instance.pk, + "source": source_type, + "source_id": source_obj.id, + "status": DrefSummary.SummaryStatus.PROCESSING, + } try: - logger.info(f"Generating DREF summaries for DREF ({dref_id}) from ({source_model_name}) ({source_id})") - results = generator.generate_all(source_doc) - for field_name, value in results.items(): - setattr(summary_instance, field_name, value) - summary_instance.status = DrefSummary.SummaryStatus.SUCCESS - summary_instance.save() + logger.info(f"Generating DREF summaries for DREF ({dref_id}) from ({source_type.label}) ({source_obj.id})") + results = DrefSummaryGenerator().generate_all(source_obj, section_kwargs=section_kwargs) + updated = DrefSummary.objects.filter(**own_marker).update( + status=DrefSummary.SummaryStatus.SUCCESS, updated_at=timezone.now(), **results + ) + if not updated: + logger.warning(f"DREF summary run for DREF ({dref_id}) was superseded by a newer trigger; discarding result.") + return DrefSummaryGenerationResult.SUPERSEDED logger.info(f"Successfully generated DREF summaries for DREF ({dref_id})") transaction.on_commit(lambda: translate_model_fields.delay(get_model_name(DrefSummary), summary_instance.pk)) - return True + return DrefSummaryGenerationResult.SUCCESS except Exception: - summary_instance.status = DrefSummary.SummaryStatus.FAILED - summary_instance.save() - logger.warning(f"DREF summary generation failed for DREF ({dref_id})", exc_info=True) - return False + updated = DrefSummary.objects.filter(**own_marker).update( + status=DrefSummary.SummaryStatus.FAILED, updated_at=timezone.now() + ) + if not updated: + logger.warning(f"DREF summary run for DREF ({dref_id}) failed but was already superseded; leaving it as is.") + else: + logger.warning(f"DREF summary generation failed for DREF ({dref_id})", exc_info=True) + return DrefSummaryGenerationResult.FAILED @shared_task diff --git a/dref/test_summary.py b/dref/test_summary.py new file mode 100644 index 000000000..8c725356b --- /dev/null +++ b/dref/test_summary.py @@ -0,0 +1,327 @@ +from datetime import date + +from django.test import TestCase, override_settings + +from dref.factories.dref import ( + DrefFactory, + DrefFinalReportFactory, + DrefOperationalUpdateFactory, + PlannedInterventionFactory, +) +from dref.models import Dref, DrefSummary +from dref.summary import SUMMARY_FIELDS, DrefSummaryGenerator +from dref.tasks import DrefSummaryGenerationResult, generate_dref_summary +from main.llm import ( + AzureOpenAiChat, + DummyDrefSummaryLLMClient, + get_dref_summary_llm_client, +) + + +class DrefSummaryLLMClientTest(TestCase): + def test_uses_dummy_client_during_tests_by_default(self): + # settings.TESTING is always true under the test runner, so the + # factory should return the dummy client even though + # USE_DUMMY_LLM_CLIENT itself defaults to False. + self.assertIsInstance(get_dref_summary_llm_client(), DummyDrefSummaryLLMClient) + + @override_settings( + TESTING=False, + USE_DUMMY_LLM_CLIENT=False, + AZURE_OPENAI_ENDPOINT="https://example.com", + AZURE_OPENAI_KEY="fake-key", + AZURE_OPENAI_DEPLOYMENT_NAME="fake-deployment", + ) + def test_uses_real_client_when_flag_off_and_not_testing(self): + self.assertIsInstance(get_dref_summary_llm_client(), AzureOpenAiChat) + + @override_settings(TESTING=False, USE_DUMMY_LLM_CLIENT=True) + def test_uses_dummy_client_when_flag_on(self): + self.assertIsInstance(get_dref_summary_llm_client(), DummyDrefSummaryLLMClient) + + +class DrefSummaryGeneratorTest(TestCase): + def test_generate_all_succeeds_end_to_end_for_dref(self): + dref = DrefFactory.create(title="Test Dref") + + results = DrefSummaryGenerator().generate_all(dref) + + self.assertEqual(set(results.keys()), set(SUMMARY_FIELDS)) + self.assertIn("DUMMY RESPONSE", results["situational_overview"]) + + def test_generate_all_succeeds_end_to_end_for_dref_operational_update(self): + ops_update = DrefOperationalUpdateFactory.create(dref=DrefFactory.create(), title="Test Ops Update") + + results = DrefSummaryGenerator().generate_all(ops_update) + + self.assertEqual(set(results.keys()), set(SUMMARY_FIELDS)) + self.assertIn("DUMMY RESPONSE", results["situational_overview"]) + + def test_generate_all_succeeds_end_to_end_for_dref_final_report(self): + planned_intervention = PlannedInterventionFactory.create(challenges="Some challenge", lessons_learnt="Some lesson") + final_report = DrefFinalReportFactory.create(title="Test Final Report", planned_interventions=[planned_intervention]) + + results = DrefSummaryGenerator().generate_all(final_report) + + self.assertEqual(set(results.keys()), set(SUMMARY_FIELDS)) + self.assertIn("DUMMY RESPONSE", results["situational_overview"]) + self.assertIn("DUMMY RESPONSE", results["challenges_identified"]) + self.assertIn("DUMMY RESPONSE", results["lessons_learned"]) + + def test_get_section_kwargs_for_dref(self): + dref = DrefFactory.create( + title="Test Dref", + event_description="Severe flooding", + type_of_dref=Dref.DrefType.RESPONSE, + event_scope="Affected 3 districts", + operation_objective="Provide shelter", + response_strategy="Cash and shelter support", + total_targeted_population=5000, + people_in_need=8000, + people_assisted="5000 people", + selection_criteria="Most vulnerable households", + ) + + kwargs = DrefSummaryGenerator.get_section_kwargs(dref) + + self.assertEqual(set(kwargs.keys()), {"situational_overview", "operational_strategy", "people_centered_approach"}) + self.assertEqual(kwargs["situational_overview"]["title"], "Test Dref") + self.assertEqual(kwargs["situational_overview"]["event_description"], "Severe flooding") + self.assertEqual(kwargs["situational_overview"]["event_scope"], "Affected 3 districts") + self.assertEqual(kwargs["operational_strategy"]["operation_objective"], "Provide shelter") + self.assertEqual(kwargs["operational_strategy"]["response_strategy"], "Cash and shelter support") + self.assertEqual(kwargs["operational_strategy"]["total_targeted_population"], 5000) + self.assertEqual(kwargs["operational_strategy"]["people_in_need"], 8000) + self.assertEqual(kwargs["people_centered_approach"]["people_assisted"], "5000 people") + self.assertEqual(kwargs["people_centered_approach"]["selection_criteria"], "Most vulnerable households") + self.assertNotIn("women", kwargs["people_centered_approach"]) + + def test_get_section_kwargs_for_dref_operational_update(self): + ops_update = DrefOperationalUpdateFactory.create( + dref=DrefFactory.create(), + title="Test Ops Update", + event_description="Flooding continues", + operation_objective="Extend shelter support", + response_strategy="Extended cash support", + total_dref_allocation=100000, + people_assisted="6000 people", + selection_criteria="Vulnerable households", + women=1000, + men=900, + girls=500, + boys=600, + new_operational_end_date=date(2025, 6, 1), + total_operation_timeframe=6, + ) + + kwargs = DrefSummaryGenerator.get_section_kwargs(ops_update) + + self.assertEqual(set(kwargs.keys()), {"situational_overview", "operational_strategy", "people_centered_approach"}) + self.assertEqual(kwargs["situational_overview"]["title"], "Test Ops Update") + self.assertEqual(kwargs["situational_overview"]["event_description"], "Flooding continues") + self.assertEqual(kwargs["operational_strategy"]["operation_objective"], "Extend shelter support") + self.assertEqual(kwargs["operational_strategy"]["total_dref_allocation"], 100000) + self.assertEqual(kwargs["operational_strategy"]["operation_end_date"], date(2025, 6, 1)) + self.assertEqual(kwargs["operational_strategy"]["operation_timeframe"], 6) + self.assertEqual(kwargs["people_centered_approach"]["people_assisted"], "6000 people") + self.assertEqual(kwargs["people_centered_approach"]["women"], 1000) + self.assertEqual(kwargs["people_centered_approach"]["men"], 900) + self.assertEqual(kwargs["people_centered_approach"]["girls"], 500) + self.assertEqual(kwargs["people_centered_approach"]["boys"], 600) + + def test_get_section_kwargs_for_dref_final_report(self): + planned_intervention = PlannedInterventionFactory.create( + challenges="Access constraints due to weather", + lessons_learnt="Earlier pre-positioning of stock helps", + ) + final_report = DrefFinalReportFactory.create( + title="Test Final Report", + event_description="Response concluded", + operation_objective="Deliver relief items", + response_strategy="Direct distribution", + total_dref_allocation=150000, + people_assisted="7000 people", + selection_criteria="Most affected households", + women=1200, + men=1100, + girls=600, + boys=700, + planned_interventions=[planned_intervention], + ) + + kwargs = DrefSummaryGenerator.get_section_kwargs(final_report) + + self.assertEqual( + set(kwargs.keys()), + { + "situational_overview", + "operational_strategy", + "people_centered_approach", + "challenges_identified", + "lessons_learned", + }, + ) + self.assertEqual(kwargs["situational_overview"]["title"], "Test Final Report") + self.assertEqual(kwargs["operational_strategy"]["total_dref_allocation"], 150000) + self.assertEqual(kwargs["people_centered_approach"]["women"], 1200) + + challenges = kwargs["challenges_identified"]["planned_interventions"] + self.assertEqual(len(challenges), 1) + self.assertEqual(challenges[0]["challenges"], "Access constraints due to weather") + + lessons = kwargs["lessons_learned"]["planned_interventions"] + self.assertEqual(len(lessons), 1) + self.assertEqual(lessons[0]["lessons_learnt"], "Earlier pre-positioning of stock helps") + + def test_get_section_kwargs_excludes_event_scope_for_imminent_dref(self): + imminent_dref = DrefFactory.create(type_of_dref=Dref.DrefType.IMMINENT, event_scope="scope text") + self.assertNotIn("event_scope", DrefSummaryGenerator.get_section_kwargs(imminent_dref)["situational_overview"]) + + response_dref = DrefFactory.create(type_of_dref=Dref.DrefType.RESPONSE, event_scope="scope text") + self.assertIn("event_scope", DrefSummaryGenerator.get_section_kwargs(response_dref)["situational_overview"]) + + def test_compute_source_hash_changes_with_content_and_is_deterministic(self): + dref = DrefFactory.create(title="Original Title", type_of_dref=Dref.DrefType.RESPONSE) + + hash_a = DrefSummaryGenerator.compute_source_hash(dref) + hash_b = DrefSummaryGenerator.compute_source_hash(dref) + self.assertEqual(hash_a, hash_b) + + dref.title = "Changed Title" + dref.save(update_fields=["title"]) + hash_c = DrefSummaryGenerator.compute_source_hash(dref) + self.assertNotEqual(hash_a, hash_c) + + def test_compute_source_hash_uses_provided_section_kwargs(self): + dref = DrefFactory.create(title="Test Dref", type_of_dref=Dref.DrefType.RESPONSE) + default_hash = DrefSummaryGenerator.compute_source_hash(dref) + + actual_kwargs = DrefSummaryGenerator.get_section_kwargs(dref) + self.assertEqual(default_hash, DrefSummaryGenerator.compute_source_hash(dref, section_kwargs=actual_kwargs)) + + custom_kwargs = {"situational_overview": {"title": "overridden"}} + self.assertNotEqual(default_hash, DrefSummaryGenerator.compute_source_hash(dref, section_kwargs=custom_kwargs)) + + def test_get_latest_approved_source_returns_none_when_dref_not_approved(self): + dref = DrefFactory.create(status=Dref.Status.FINALIZED) + self.assertIsNone(DrefSummaryGenerator.get_latest_approved_source(dref)) + + def test_get_latest_approved_source_follows_priority_order(self): + dref = DrefFactory.create(status=Dref.Status.APPROVED) + self.assertEqual(DrefSummaryGenerator.get_latest_approved_source(dref), (DrefSummary.SourceModel.DREF, dref)) + + # Unapproved updates don't count, so the Dref itself still wins. + DrefOperationalUpdateFactory.create(dref=dref, status=Dref.Status.FINALIZED, operational_update_number=1) + self.assertEqual(DrefSummaryGenerator.get_latest_approved_source(dref), (DrefSummary.SourceModel.DREF, dref)) + + # An approved Operational Update supersedes the Dref; the higher + # operational_update_number wins, regardless of insertion order. + DrefOperationalUpdateFactory.create(dref=dref, status=Dref.Status.APPROVED, operational_update_number=1) + ops_update_2 = DrefOperationalUpdateFactory.create(dref=dref, status=Dref.Status.APPROVED, operational_update_number=2) + self.assertEqual( + DrefSummaryGenerator.get_latest_approved_source(dref), + (DrefSummary.SourceModel.DREF_OPERATIONAL_UPDATE, ops_update_2), + ) + + # An approved Final Report supersedes everything else. + final_report = DrefFinalReportFactory.create(dref=dref, status=Dref.Status.APPROVED) + self.assertEqual( + DrefSummaryGenerator.get_latest_approved_source(dref), + (DrefSummary.SourceModel.DREF_FINAL_REPORT, final_report), + ) + + def test_returns_source_not_found_when_dref_missing_or_nothing_approved(self): + self.assertEqual(generate_dref_summary(dref_id=999999), DrefSummaryGenerationResult.SOURCE_NOT_FOUND) + + dref = DrefFactory.create(status=Dref.Status.FINALIZED) + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.SOURCE_NOT_FOUND) + + def test_generates_from_dref_and_is_up_to_date_on_rerun(self): + dref = DrefFactory.create(status=Dref.Status.APPROVED) + + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.SUCCESS) + summary = DrefSummary.objects.get(dref=dref) + self.assertEqual(summary.source, DrefSummary.SourceModel.DREF) + self.assertEqual(summary.source_id, dref.id) + + # Nothing changed since - re-running should not regenerate. + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.UP_TO_DATE) + + def test_retrigger_with_overwrite_regenerates_even_when_up_to_date(self): + dref = DrefFactory.create(status=Dref.Status.APPROVED) + + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.SUCCESS) + summary = DrefSummary.objects.get(dref=dref) + first_hash = summary.source_hash + first_updated_at = summary.updated_at + + # Nothing changed since, so a plain retrigger would be UP_TO_DATE ... + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.UP_TO_DATE) + + # ... but overwrite=True must bypass that check and regenerate anyway. + self.assertEqual(generate_dref_summary(dref_id=dref.id, overwrite=True), DrefSummaryGenerationResult.SUCCESS) + + self.assertEqual(DrefSummary.objects.filter(dref=dref).count(), 1) + summary.refresh_from_db() + self.assertEqual(summary.status, DrefSummary.SummaryStatus.SUCCESS) + self.assertEqual(summary.source, DrefSummary.SourceModel.DREF) + self.assertEqual(summary.source_id, dref.id) + self.assertEqual(summary.source_hash, first_hash) + self.assertGreater(summary.updated_at, first_updated_at) + + def test_processing_in_flight_skips_same_source_but_regenerates_for_a_newer_one(self): + dref = DrefFactory.create(status=Dref.Status.APPROVED) + # Simulate a run already in flight for the Dref itself. + DrefSummary.objects.create( + dref=dref, + source=DrefSummary.SourceModel.DREF, + source_id=dref.id, + source_hash="in-flight-hash", + status=DrefSummary.SummaryStatus.PROCESSING, + ) + + # A duplicate trigger for the same in-flight source is dropped. + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.ALREADY_IN_PROGRESS) + + # A newer approval lands while that run is still (supposedly) processing. + ops_update = DrefOperationalUpdateFactory.create(dref=dref, status=Dref.Status.APPROVED, operational_update_number=1) + + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.SUCCESS) + summary = DrefSummary.objects.get(dref=dref) + self.assertEqual(summary.source, DrefSummary.SourceModel.DREF_OPERATIONAL_UPDATE) + self.assertEqual(summary.source_id, ops_update.id) + + def test_generate_all_succeeds_for_newer_ops_update_while_older_one_in_flight(self): + """Two Operational Updates on one Dref: a run in flight for the older + one must not block generation once a newer one is approved.""" + dref = DrefFactory.create(status=Dref.Status.APPROVED) + ops_update_1 = DrefOperationalUpdateFactory.create(dref=dref, status=Dref.Status.APPROVED, operational_update_number=1) + DrefSummary.objects.create( + dref=dref, + source=DrefSummary.SourceModel.DREF_OPERATIONAL_UPDATE, + source_id=ops_update_1.id, + source_hash="in-flight-hash", + status=DrefSummary.SummaryStatus.PROCESSING, + ) + + # A duplicate trigger for the same in-flight source (ops_update_1) is dropped. + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.ALREADY_IN_PROGRESS) + + # A second, newer Operational Update is approved while that run is still (supposedly) processing. + ops_update_2 = DrefOperationalUpdateFactory.create(dref=dref, status=Dref.Status.APPROVED, operational_update_number=2) + + # The in-flight record is for a now-stale source (ops_update_1), so this + # trigger falls through and regenerates from ops_update_2 instead of + # being blocked as "already in progress". + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.SUCCESS) + + # Exactly one DrefSummary row for the Dref - updated in place, not duplicated. + self.assertEqual(DrefSummary.objects.filter(dref=dref).count(), 1) + summary = DrefSummary.objects.get(dref=dref) + self.assertEqual(summary.source, DrefSummary.SourceModel.DREF_OPERATIONAL_UPDATE) + self.assertEqual(summary.source_id, ops_update_2.id) + self.assertEqual(summary.status, DrefSummary.SummaryStatus.SUCCESS) + self.assertNotEqual(summary.source_hash, "in-flight-hash") + + # Nothing changed since - re-running should not regenerate. + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.UP_TO_DATE) diff --git a/dref/views.py b/dref/views.py index 480dd7c56..7dfe58e34 100644 --- a/dref/views.py +++ b/dref/views.py @@ -32,13 +32,7 @@ DrefOperationalUpdateFilter, DrefShareUserFilterSet, ) -from dref.models import ( - Dref, - DrefFile, - DrefFinalReport, - DrefOperationalUpdate, - DrefSummary, -) +from dref.models import Dref, DrefFile, DrefFinalReport, DrefOperationalUpdate from dref.permissions import ApproveDrefPermission from dref.serializers import ( AddDrefUserSerializer, @@ -143,11 +137,7 @@ def get_approved(self, request, pk=None, version=None): dref.status = Dref.Status.APPROVED dref.save(update_fields=["event", "status"]) - _dref_id = dref.id - transaction.on_commit( - lambda: generate_dref_summary.delay(source_model_name=DrefSummary.SourceModel.DREF, source_id=_dref_id) - ) - + transaction.on_commit(lambda: generate_dref_summary.delay(dref_id=dref.id)) return response.Response(DrefSerializer(dref, context={"request": request}).data) @extend_schema(request=None, responses=DrefSerializer) @@ -244,14 +234,7 @@ def get_approved(self, request, pk=None, version=None): operational_update.status = Dref.Status.APPROVED operational_update.date_of_approval = timezone.now().date() operational_update.save(update_fields=["status", "date_of_approval"]) - _ops_update_id = operational_update.pk - transaction.on_commit( - lambda: generate_dref_summary.delay( - source_model_name=DrefSummary.SourceModel.DREF_OPERATIONAL_UPDATE, - source_id=_ops_update_id, - overwrite=True, - ) - ) + transaction.on_commit(lambda: generate_dref_summary.delay(dref_id=operational_update.dref_id)) serializer = DrefOperationalUpdateSerializer(operational_update, context={"request": request}) return response.Response(serializer.data) @@ -320,14 +303,7 @@ def get_approved(self, request, pk=None, version=None): final_report.dref.is_active = False final_report.date_of_approval = timezone.now().date() final_report.dref.save(update_fields=["is_active", "date_of_approval"]) - _final_report_id = final_report.pk - transaction.on_commit( - lambda: generate_dref_summary.delay( - source_model_name=DrefSummary.SourceModel.DREF_FINAL_REPORT, - source_id=_final_report_id, - overwrite=True, - ) - ) + transaction.on_commit(lambda: generate_dref_summary.delay(dref_id=final_report.dref_id)) serializer = DrefFinalReportSerializer(final_report, context={"request": request}) return response.Response(serializer.data) diff --git a/main/llm.py b/main/llm.py new file mode 100644 index 000000000..87a33e4ff --- /dev/null +++ b/main/llm.py @@ -0,0 +1,183 @@ +import json +import logging +import re +from typing import Dict, List, Optional + +from django.conf import settings +from django.utils.functional import cached_property +from openai import AzureOpenAI + +logger = logging.getLogger(__name__) + +# A chat completion prompt: the ``[{"role": ..., "content": ...}, ...]`` list +# every real call site builds and passes in. +Message = Dict[str, str] +Messages = List[Message] + + +PLACEHOLDER_TEXT = ( + "This is placeholder text standing in for a real AI-generated summary. It is returned by " + "the dummy LLM client Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt " + "ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco " + "laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in " + "voluptate velit esse cillum dolore eu fugiat nulla pariatur." +) + + +class BaseLLMClient: + def get_response(self, messages: Messages) -> Optional[str]: + raise NotImplementedError + + @staticmethod + def _last_user_message(messages: Messages) -> str: + return next((message["content"] for message in reversed(messages) if message["role"] == "user"), "") + + def _fake_content(self) -> str: + """Placeholder text every dummy response uses in place of real LLM output.""" + return f"DUMMY RESPONSE, {PLACEHOLDER_TEXT}" + + +class AzureOpenAiChat(BaseLLMClient): + """The real Azure OpenAI-backed client, shared by every feature.""" + + def __init__(self, temperature: float = 0.5): + self.temperature = temperature + + if settings.TESTING: + # Tests never hit Azure, so a missing/placeholder config shouldn't fail construction. + return + + if not settings.AZURE_OPENAI_ENDPOINT or not settings.AZURE_OPENAI_KEY or not settings.AZURE_OPENAI_DEPLOYMENT_NAME: + raise Exception("Azure OpenAI configuration missing") + + @cached_property + def client(self): + return AzureOpenAI( + azure_endpoint=settings.AZURE_OPENAI_ENDPOINT, + api_key=settings.AZURE_OPENAI_KEY, + api_version="2023-05-15", + ) + + def get_response(self, messages: Messages) -> Optional[str]: + if settings.USE_DUMMY_LLM_CLIENT or settings.TESTING: + logger.info(f"{type(self).__name__}: dummy mode is on; returning a fake response instead of calling Azure OpenAI.") + return self._fake_content() + + try: + response = self.client.chat.completions.create( + model=settings.AZURE_OPENAI_DEPLOYMENT_NAME, + messages=messages, + temperature=self.temperature, + ) + return response.choices[0].message.content + except Exception as e: + logger.error(f"Error while generating LLM response: {e}", exc_info=True) + return None + + +class BaseDummyLLMClient(BaseLLMClient): + """Summaries generated for local development/testing, without hitting Azure OpenAI. Each subclass implements + ``_fake_response()`` to return a JSON object with the expected fields for that feature + """ + + def get_response(self, messages: Messages) -> Optional[str]: + return self._fake_response(messages) + + def _fake_response(self, messages: Messages) -> str: + raise NotImplementedError + + +class DummyDrefSummaryLLMClient(BaseDummyLLMClient): + """Fake response for DREF summary generation: a JSON object with all of + DREF's summary fields, each set to the same fake content, so + ``dref.summary.DrefSummaryGenerator`` parses/saves it unchanged. + """ + + def _fake_response(self, messages: Messages) -> str: + from dref.summary import SUMMARY_FIELDS + + logger.info("DummyDrefSummaryLLMClient: generating fake response for DREF summary fields") + fake_content = self._fake_content() + payload: Dict[str, str] = {field_name: fake_content for field_name in SUMMARY_FIELDS} + return json.dumps(payload) + + +class DummyOpsLearningLLMClient(BaseDummyLLMClient): + """Fake response for PER Ops Learning summaries.""" + + _EXCERPT_ID_RE = re.compile(r"^(\d+)\. In ", re.MULTILINE) + + def _prompt_excerpt_ids(self, user_content: str) -> List[int]: + return [int(excerpt_id) for excerpt_id in self._EXCERPT_ID_RE.findall(user_content)] + + def _fake_response(self, messages: Messages) -> str: + from per.models import OpsLearning + from per.ops_learning_summary import OpsLearningSummaryTask + + logger.info("DummyOpsLearningLLMClient: generating fake response for Ops Learning summary") + user_content = self._last_user_message(messages) + fake_content = self._fake_content() + excerpt_ids = self._prompt_excerpt_ids(user_content) + logger.info(f"DummyOpsLearningLLMClient: found {len(excerpt_ids)} excerpt id(s) in the prompt") + + all_excerpts_id = ", ".join(str(excerpt_id) for excerpt_id in excerpt_ids) + + payload: Dict[str, Dict[str, str]] + if OpsLearningSummaryTask.component_prompt in user_content: + logger.info("DummyOpsLearningLLMClient: detected the component prompt shape.") + excerpts = list( + OpsLearning.objects.filter(id__in=excerpt_ids, is_validated=True, per_component_validated__isnull=False) + .prefetch_related("per_component_validated") + .distinct() + ) + component = excerpts[0].per_component_validated.first() if excerpts else None + subtype = component.title if component else "Logistics" + excerpts_id = ", ".join(str(excerpt.id) for excerpt in excerpts) or all_excerpts_id + logger.info( + f"DummyOpsLearningLLMClient: {len(excerpts)} excerpt(s) matched a validated component; " + f"using subtype '{subtype}'." + ) + payload = {"0": {"type": "component", "subtype": subtype, "excerpts id": excerpts_id, "content": fake_content}} + + elif OpsLearningSummaryTask.sector_prompt in user_content: + logger.info("DummyOpsLearningLLMClient: detected the sector prompt shape.") + excerpts = list( + OpsLearning.objects.filter(id__in=excerpt_ids, is_validated=True, sector_validated__isnull=False) + .prefetch_related("sector_validated") + .distinct() + ) + sector = excerpts[0].sector_validated.first() if excerpts else None + subtype = sector.title if sector else "Shelter" + excerpts_id = ", ".join(str(excerpt.id) for excerpt in excerpts) or all_excerpts_id + logger.info( + f"DummyOpsLearningLLMClient: {len(excerpts)} excerpt(s) matched a validated sector; " + f"using subtype '{subtype}'." + ) + payload = {"0": {"type": "sector", "subtype": subtype, "excerpts id": excerpts_id, "content": fake_content}} + else: + logger.info( + f"DummyOpsLearningLLMClient: detected the primary prompt shape; " + f"returning 3 fake findings with excerpts id(s) [{all_excerpts_id}]." + ) + payload = { + str(i): { + "title": f"Dummy Finding {i + 1}", + "excerpts id": all_excerpts_id, + "content": fake_content, + "confidence level": "1/5", + } + for i in range(3) + } + return json.dumps(payload) + + +def get_dref_summary_llm_client() -> BaseLLMClient: + if settings.USE_DUMMY_LLM_CLIENT or settings.TESTING: + return DummyDrefSummaryLLMClient() + return AzureOpenAiChat(temperature=0.5) + + +def get_ops_learning_llm_client() -> BaseLLMClient: + if settings.USE_DUMMY_LLM_CLIENT or settings.TESTING: + return DummyOpsLearningLLMClient() + return AzureOpenAiChat(temperature=0.7) diff --git a/main/lock.py b/main/lock.py index 3d1fd2de1..9de5b7aaf 100644 --- a/main/lock.py +++ b/main/lock.py @@ -20,6 +20,7 @@ class RedisLockKey(models.TextChoices): OPERATION_LEARNING_SUMMARY = _BASE + "-operation-learning-summary-{0}" OPERATION_LEARNING_SUMMARY_EXPORT = _BASE + "-operation-learning-summary-export-{0}" MODEL_TRANSLATION = _BASE + "-{model_name}-translation-{id}" + DREF_SUMMARY = _BASE + "-dref-summary-{0}" @contextmanager diff --git a/main/settings.py b/main/settings.py index a500bef8c..b82853c20 100644 --- a/main/settings.py +++ b/main/settings.py @@ -146,6 +146,8 @@ AZURE_OPENAI_ENDPOINT=(str, None), AZURE_OPENAI_KEY=(str, None), AZURE_OPENAI_DEPLOYMENT_NAME=(str, None), + # Use a fake LLM client instead of calling Azure OpenAI + USE_DUMMY_LLM_CLIENT=(bool, False), # ReliefWeb appname RELIEF_WEB_APP_NAME=(str, None), # PowerBI @@ -868,6 +870,7 @@ def decode_base64(env_key, fallback_env_key): AZURE_OPENAI_ENDPOINT = env("AZURE_OPENAI_ENDPOINT") AZURE_OPENAI_KEY = env("AZURE_OPENAI_KEY") AZURE_OPENAI_DEPLOYMENT_NAME = env("AZURE_OPENAI_DEPLOYMENT_NAME") +USE_DUMMY_LLM_CLIENT = env("USE_DUMMY_LLM_CLIENT") OIDC_ENABLE = env("OIDC_ENABLE") OIDC_RSA_PRIVATE_KEY = None diff --git a/per/models.py b/per/models.py index 4cd7e7488..3b792c426 100644 --- a/per/models.py +++ b/per/models.py @@ -705,6 +705,9 @@ class OpsLearning(models.Model): modified_at = models.DateTimeField(verbose_name=_("modified_at"), auto_now=True) created_at = models.DateTimeField(verbose_name=_("created at"), auto_now_add=True) + # TYPING + id: int + class Meta: ordering = ("learning",) verbose_name = _("Operational Learning") diff --git a/per/ops_learning_summary.py b/per/ops_learning_summary.py index d2d3d6cd4..5a5d150ed 100644 --- a/per/ops_learning_summary.py +++ b/per/ops_learning_summary.py @@ -5,17 +5,15 @@ import pandas as pd import tiktoken -from django.conf import settings from django.db import transaction from django.db.models import F -from django.utils.functional import cached_property -from openai import AzureOpenAI from api.logger import logger from api.models import Country from api.utils import get_model_name from deployments.models import SectorTag from lang.tasks import translate_model_fields +from main.llm import get_ops_learning_llm_client from per.cache import OpslearningSummaryCacheHelper from per.models import ( FormComponent, @@ -29,24 +27,6 @@ ) -class AzureOpenAiChat: - - @cached_property - def client(self): - return AzureOpenAI( - azure_endpoint=settings.AZURE_OPENAI_ENDPOINT, api_key=settings.AZURE_OPENAI_KEY, api_version="2023-05-15" - ) - - def get_response(self, message): - try: - response = self.client.chat.completions.create( - model=settings.AZURE_OPENAI_DEPLOYMENT_NAME, messages=message, temperature=0.7 - ) - return response.choices[0].message.content - except Exception as e: - logger.error(f"Error while generating response: {e}", exc_info=True) - - class OpsLearningSummaryTask: PROMPT_DATA_LENGTH_LIMIT = 5000 @@ -815,9 +795,9 @@ def _summarize(prompt, type: OpsLearningPromptResponseCache.PromptType, system_m logger.warning("The length of the prompt might be too long.") return "{}" - # Using Azure OpenAI to summarize the prompt - client = AzureOpenAiChat() - response = client.get_response(message=messages) + # Using Azure OpenAI (or the dummy, per settings.USE_DUMMY_LLM_CLIENT) to summarize the prompt + client = get_ops_learning_llm_client() + response = client.get_response(messages) return response def _validate_format(summary, MAX_RETRIES=3): From d472073078ab6556604bf2b49ded4789ce56f91a Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 8 Jul 2026 10:42:54 +0545 Subject: [PATCH 26/28] fix(dref): guard summary generation writes against concurrent-source races --- dref/tasks.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/dref/tasks.py b/dref/tasks.py index 698124eaa..fde4ef745 100644 --- a/dref/tasks.py +++ b/dref/tasks.py @@ -164,23 +164,27 @@ def generate_dref_summary(dref_id: int, overwrite: bool = False) -> DrefSummaryG try: logger.info(f"Generating DREF summaries for DREF ({dref_id}) from ({source_type.label}) ({source_obj.id})") results = DrefSummaryGenerator().generate_all(source_obj, section_kwargs=section_kwargs) - updated = DrefSummary.objects.filter(**own_marker).update( - status=DrefSummary.SummaryStatus.SUCCESS, updated_at=timezone.now(), **results - ) - if not updated: - logger.warning(f"DREF summary run for DREF ({dref_id}) was superseded by a newer trigger; discarding result.") - return DrefSummaryGenerationResult.SUPERSEDED + with transaction.atomic(): + latest_summary_instance = DrefSummary.objects.select_for_update().filter(**own_marker).first() + if latest_summary_instance is None: + logger.warning(f"DREF summary run for DREF ({dref_id}) was superseded by a newer trigger; discarding result.") + return DrefSummaryGenerationResult.SUPERSEDED + for field_name, value in results.items(): + setattr(latest_summary_instance, field_name, value) + latest_summary_instance.status = DrefSummary.SummaryStatus.SUCCESS + latest_summary_instance.save() logger.info(f"Successfully generated DREF summaries for DREF ({dref_id})") - transaction.on_commit(lambda: translate_model_fields.delay(get_model_name(DrefSummary), summary_instance.pk)) + transaction.on_commit(lambda: translate_model_fields.delay(get_model_name(DrefSummary), latest_summary_instance.pk)) return DrefSummaryGenerationResult.SUCCESS except Exception: - updated = DrefSummary.objects.filter(**own_marker).update( - status=DrefSummary.SummaryStatus.FAILED, updated_at=timezone.now() - ) - if not updated: - logger.warning(f"DREF summary run for DREF ({dref_id}) failed but was already superseded; leaving it as is.") - else: - logger.warning(f"DREF summary generation failed for DREF ({dref_id})", exc_info=True) + with transaction.atomic(): + latest_summary_instance = DrefSummary.objects.select_for_update().filter(**own_marker).first() + if latest_summary_instance is None: + logger.warning(f"DREF summary run for DREF ({dref_id}) failed but was already superseded; leaving it as is.") + else: + latest_summary_instance.status = DrefSummary.SummaryStatus.FAILED + latest_summary_instance.save() + logger.warning(f"DREF summary generation failed for DREF ({dref_id})", exc_info=True) return DrefSummaryGenerationResult.FAILED From 600a92197cd0ba802843b3facdbccb4fbbaeb827 Mon Sep 17 00:00:00 2001 From: frozenhelium Date: Mon, 6 Jul 2026 11:46:18 +0545 Subject: [PATCH 27/28] fix(dref): expose disaster_category on emergency DREF serializers - add disaster_category and disaster_category_display to the emergency dref, operational update and final report serializers --- assets | 2 +- dref/serializers.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/assets b/assets index 03bb6f64a..7c5b98dfc 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit 03bb6f64ab4e668fff21b0c6d10b799b53742284 +Subproject commit 7c5b98dfc773808c7a918040f386cc3b11929a2e diff --git a/dref/serializers.py b/dref/serializers.py index c8c8bd1ad..a055d710a 100644 --- a/dref/serializers.py +++ b/dref/serializers.py @@ -2302,6 +2302,7 @@ class EmergencyDrefContactSerializer(serializers.Serializer): # It is designed to provide the necessary data for the emergency page in a format that is easy to consume and display. class EmergencyDrefFinalReportSerializer(serializers.ModelSerializer): status_display = serializers.CharField(source="get_status_display", read_only=True) + disaster_category_display = serializers.CharField(source="get_disaster_category_display", read_only=True) country_details = MiniCountrySerializer(source="country", read_only=True) district_details = MiniDistrictSerializer(source="district", read_only=True, many=True) planned_interventions = PlannedInterventionSerializer(many=True, read_only=True) @@ -2319,6 +2320,8 @@ class Meta: "cover_image_file", "status", "status_display", + "disaster_category", + "disaster_category_display", "event_scope", "event_description", "event_date", @@ -2378,6 +2381,7 @@ class Meta: class EmergencyDrefOperationalUpdateSerializer(serializers.ModelSerializer): status_display = serializers.CharField(source="get_status_display", read_only=True) + disaster_category_display = serializers.CharField(source="get_disaster_category_display", read_only=True) country_details = MiniCountrySerializer(source="country", read_only=True) district_details = MiniDistrictSerializer(source="district", read_only=True, many=True) planned_interventions = PlannedInterventionSerializer(many=True, read_only=True) @@ -2394,6 +2398,8 @@ class Meta: "cover_image_file", "status", "status_display", + "disaster_category", + "disaster_category_display", "event_scope", "event_description", "operational_update_number", @@ -2449,6 +2455,7 @@ class Meta: class EmergencyDrefSerializer(serializers.ModelSerializer): status_display = serializers.CharField(source="get_status_display", read_only=True) + disaster_category_display = serializers.CharField(source="get_disaster_category_display", read_only=True) country_details = MiniCountrySerializer(source="country", read_only=True) district_details = MiniDistrictSerializer(source="district", read_only=True, many=True) planned_interventions = PlannedInterventionSerializer(many=True, read_only=True) @@ -2475,6 +2482,8 @@ class Meta: "title", "type_of_dref", "type_of_dref_display", + "disaster_category", + "disaster_category_display", "disaster_type_details", "cover_image_file", "event_description", From 40d199b1c63f19d36343fda0bb0c066b25ed0b4c Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 8 Jul 2026 12:46:17 +0545 Subject: [PATCH 28/28] chore: update env key for azure api key --- dref/test_summary.py | 2 +- main/llm.py | 4 ++-- main/settings.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dref/test_summary.py b/dref/test_summary.py index 8c725356b..f291d47bc 100644 --- a/dref/test_summary.py +++ b/dref/test_summary.py @@ -29,7 +29,7 @@ def test_uses_dummy_client_during_tests_by_default(self): TESTING=False, USE_DUMMY_LLM_CLIENT=False, AZURE_OPENAI_ENDPOINT="https://example.com", - AZURE_OPENAI_KEY="fake-key", + AZURE_OPENAI_API_KEY="fake-key", AZURE_OPENAI_DEPLOYMENT_NAME="fake-deployment", ) def test_uses_real_client_when_flag_off_and_not_testing(self): diff --git a/main/llm.py b/main/llm.py index 87a33e4ff..46860e10a 100644 --- a/main/llm.py +++ b/main/llm.py @@ -47,14 +47,14 @@ def __init__(self, temperature: float = 0.5): # Tests never hit Azure, so a missing/placeholder config shouldn't fail construction. return - if not settings.AZURE_OPENAI_ENDPOINT or not settings.AZURE_OPENAI_KEY or not settings.AZURE_OPENAI_DEPLOYMENT_NAME: + if not settings.AZURE_OPENAI_ENDPOINT or not settings.AZURE_OPENAI_API_KEY or not settings.AZURE_OPENAI_DEPLOYMENT_NAME: raise Exception("Azure OpenAI configuration missing") @cached_property def client(self): return AzureOpenAI( azure_endpoint=settings.AZURE_OPENAI_ENDPOINT, - api_key=settings.AZURE_OPENAI_KEY, + api_key=settings.AZURE_OPENAI_API_KEY, api_version="2023-05-15", ) diff --git a/main/settings.py b/main/settings.py index b82853c20..0927e30ef 100644 --- a/main/settings.py +++ b/main/settings.py @@ -144,7 +144,7 @@ NS_INITIATIVES_API_TOKEN=(str, None), # OpenAi Azure AZURE_OPENAI_ENDPOINT=(str, None), - AZURE_OPENAI_KEY=(str, None), + AZURE_OPENAI_API_KEY=(str, None), AZURE_OPENAI_DEPLOYMENT_NAME=(str, None), # Use a fake LLM client instead of calling Azure OpenAI USE_DUMMY_LLM_CLIENT=(bool, False), @@ -868,7 +868,7 @@ def decode_base64(env_key, fallback_env_key): JWT_EXPIRE_TIMESTAMP_DAYS = env("JWT_EXPIRE_TIMESTAMP_DAYS") AZURE_OPENAI_ENDPOINT = env("AZURE_OPENAI_ENDPOINT") -AZURE_OPENAI_KEY = env("AZURE_OPENAI_KEY") +AZURE_OPENAI_API_KEY = env("AZURE_OPENAI_API_KEY") AZURE_OPENAI_DEPLOYMENT_NAME = env("AZURE_OPENAI_DEPLOYMENT_NAME") USE_DUMMY_LLM_CLIENT = env("USE_DUMMY_LLM_CLIENT")