diff --git a/api/admin.py b/api/admin.py index 9dab94449..01352ed76 100644 --- a/api/admin.py +++ b/api/admin.py @@ -26,7 +26,6 @@ import api.models as models from api.admin_classes import RegionRestrictedAdmin -from api.event_sources import SOURCES from api.management.commands.index_and_notify import Command as Notify from lang.admin import TranslationAdmin, TranslationInlineModelAdmin from notifications.models import RecordType, SubscriptionType @@ -158,38 +157,6 @@ def queryset(self, request, queryset): return queryset.filter(is_featured=False) -class EventSourceFilter(admin.SimpleListFilter): - title = _("source") - parameter_name = "event_source" - - def lookups(self, request, model_admin): - return ( - ("input", _("Manual input")), - ("gdacs", _("GDACs scraper")), - ("who", _("WHO scraper")), - ("report_ingest", _("Field report ingest")), - ("report_admin", _("Field report admin")), - ("appeal_admin", _("Appeals admin")), - ("unknown", _("Unknown automated")), - ) - - def queryset(self, request, queryset): - if self.value() == "input": - return queryset.filter(auto_generated=False) - elif self.value() == "gdacs": - return queryset.filter(auto_generated_source=SOURCES["gdacs"]) - elif self.value() == "who": - return queryset.filter(auto_generated_source__startswith="www.who.int") - elif self.value() == "report_ingest": - return queryset.filter(auto_generated_source=SOURCES["report_ingest"]) - elif self.value() == "report_admin": - return queryset.filter(auto_generated_source=SOURCES["report_admin"]) - elif self.value() == "appeal_admin": - return queryset.filter(auto_generated_source=SOURCES["appeal_admin"]) - elif self.value() == "unknown": - return queryset.filter(auto_generated=True).filter(auto_generated_source__isnull=True) - - class DisasterTypeAdmin(CompareVersionAdmin, TranslationAdmin, admin.ModelAdmin): search_fields = ("name",) @@ -246,9 +213,9 @@ def level_updated_at(self, obj): "cc_status", "glide", "auto_generated", - "auto_generated_source", + "source", ) - list_filter = [IsFeaturedFilter, EventSourceFilter] + list_filter = [IsFeaturedFilter, "source"] actions = ["create_field_reports"] search_fields = ( "name", @@ -369,7 +336,7 @@ def changeform_view(self, request, object_id=None, form_url="", extra_context=No self.readonly_fields = ( "appeals", "field_reports", - "auto_generated_source", + "source", "parent_event", "created_at", "updated_at", @@ -378,9 +345,10 @@ def changeform_view(self, request, object_id=None, form_url="", extra_context=No self.readonly_fields = ( "appeals", "field_reports", - "auto_generated_source", + "source", "created_at", "updated_at", + "who_guid", ) # Set severity level from GET parameter @@ -651,7 +619,7 @@ def create_events(self, request, queryset): dtype=getattr(report, "dtype"), disaster_start_date=getattr(report, "created_at"), auto_generated=True, - auto_generated_source=SOURCES["report_admin"], + source=models.Event.EventSource.FIELD_REPORT_ADMIN, ) if getattr(report, "countries").exists(): for country in report.countries.all(): @@ -760,7 +728,7 @@ def create_events(self, request, queryset): dtype=getattr(appeal, "dtype"), disaster_start_date=getattr(appeal, "start_date"), auto_generated=True, - auto_generated_source=SOURCES["appeal_admin"], + source=models.Event.EventSource.APPEAL_ADMIN, ) if appeal.country is not None: event.countries.add(appeal.country) diff --git a/api/drf_views.py b/api/drf_views.py index f6d7eeea6..85c51dd47 100644 --- a/api/drf_views.py +++ b/api/drf_views.py @@ -6,6 +6,7 @@ Avg, Case, Count, + Exists, ExpressionWrapper, F, OuterRef, @@ -13,6 +14,7 @@ Q, Subquery, Sum, + Value, When, ) from django.db.models.fields import IntegerField @@ -59,6 +61,7 @@ from databank.serializers import CountryOverviewSerializer from deployments.models import ERU, Personnel from deployments.serializers import ListDeployedERUByEventSerializer +from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate from main.enums import GlobalEnumSerializer, get_enum_values from main.filters import NullsLastOrderingFilter from main.permissions import DenyGuestUserMutationPermission, DenyGuestUserPermission @@ -73,6 +76,7 @@ Appeal, AppealDocument, AppealHistory, + AppealStatus, AppealType, Country, CountryKeyDocument, @@ -85,10 +89,13 @@ Event, EventContact, EventFeaturedDocument, + EventLink, EventSeverityLevelHistory, + EventStage, Export, ExternalPartner, FieldReport, + KeyFigure, MainContact, Profile, Region, @@ -122,6 +129,7 @@ CountrySupportingPartnerSerializer, CountryTableauSerializer, DeploymentsByEventSerializer, + DetailEmergencySerializer, DetailEventSerializer, DisasterTypeSerializer, DistrictSerializer, @@ -747,11 +755,12 @@ def get_queryset(self, *args, **kwargs): qset = super().get_queryset() if self.action == "mini_events": # return Event.objects.filter(parent_event__isnull=True).select_related('dtype') - return qset.filter(parent_event__isnull=True).select_related("dtype") + return qset.filter(parent_event__isnull=True).select_related("dtype").prefetch_related("countries_for_preview") + if self.action == "response_activity_events": return ( qset.filter(parent_event__isnull=True) - .filter(Q(auto_generated=False) | Q(auto_generated_source="New field report")) + .filter(Q(auto_generated=False) | Q(source=Event.EventSource.NEW_FIELD_REPORT)) .select_related("dtype") ) return ( @@ -867,7 +876,11 @@ def retrieve(self, request, pk=None, *args, **kwargs): ) @action(methods=["get"], detail=False, url_path="mini") def mini_events(self, request): - queryset = self.filter_queryset(self.get_queryset()) + queryset = self.filter_queryset(self.get_queryset()).annotate( + latest_field_report_id=Subquery( + FieldReport.objects.filter(event=OuterRef("pk")).order_by("-updated_at").values("id")[:1] + ) + ) serializer = ListMiniEventSerializer(queryset, many=True) page = self.paginate_queryset(queryset) if page is not None: @@ -1349,7 +1362,7 @@ class SupportedActivityViewset(viewsets.ReadOnlyModelViewSet): # summary=report.description or "", # disaster_start_date=report.start_date, # auto_generated=True, -# auto_generated_source=SOURCES["new_report"], +# source=Event.EventSource.NEW_FIELD_REPORT, # visibility=report.visibility, # **{TRANSLATOR_ORIGINAL_LANGUAGE_FIELD_NAME: django_get_language()}, # ) @@ -1555,3 +1568,221 @@ class CountrySupportingPartnerViewSet(viewsets.ModelViewSet): def get_queryset(self): return CountrySupportingPartner.objects.select_related("country") + + +class EmergencyViewset( + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, + ReadOnlyVisibilityViewsetMixin, +): + queryset = Event.objects.all() + lookup_field = "id" + serializer_class = DetailEmergencySerializer + filterset_class = EventFilter + + def get_queryset(self): + today = timezone.now().date() + + appeal_priority_qs = ( + Appeal.objects.filter( + event=OuterRef("pk"), + status__in=[AppealStatus.ACTIVE, AppealStatus.CLOSED], + ) + .annotate( + priority=Case( + When(status=AppealStatus.ACTIVE, then=Value(1)), + When(status=AppealStatus.CLOSED, then=Value(2)), + output_field=IntegerField(), + ) + ) + .order_by("priority", "-start_date") + ) + + active_dref_appeal_qs = appeal_priority_qs.filter(atype=AppealType.DREF) + active_emergency_appeal_qs = appeal_priority_qs.filter(atype=AppealType.APPEAL) + + appeal_exists_base_qs = Appeal.objects.filter( + event=OuterRef("pk"), + status__in=[AppealStatus.ACTIVE, AppealStatus.CLOSED], + ) + + # Unordered: used only in Exists() checks + active_dref_appeal_exists_qs = appeal_exists_base_qs.filter(atype=AppealType.DREF) + active_emergency_appeal_exists_qs = appeal_exists_base_qs.filter(atype=AppealType.APPEAL) + + field_report_qs = FieldReport.objects.filter(event=OuterRef("pk")) + + approved_dref_qs = Dref.objects.filter( + event=OuterRef("pk"), + status=Dref.Status.APPROVED, + ) + + approved_ops_update_qs = DrefOperationalUpdate.objects.filter( + dref__event=OuterRef("pk"), + status=Dref.Status.APPROVED, + ).order_by("-created_at") + + approved_final_report_qs = DrefFinalReport.objects.filter( + dref__event=OuterRef("pk"), + status=Dref.Status.APPROVED, + ).order_by("-created_at") + + # Unordered: used only in Exists() checks + approved_ops_update_exists_qs = DrefOperationalUpdate.objects.filter( + dref__event=OuterRef("pk"), + status=Dref.Status.APPROVED, + ) + approved_final_report_exists_qs = DrefFinalReport.objects.filter( + dref__event=OuterRef("pk"), + status=Dref.Status.APPROVED, + ) + + return ( + super() + .get_queryset() + .annotate( + # Aggregated Values + response_activity_count=Count( + "emergency_projects", + distinct=True, + ), + active_deployments_count=Count( + "personneldeployment__personnel", + filter=Q( + personneldeployment__personnel__type=Personnel.TypeChoices.RR, + personneldeployment__personnel__start_date__date__lte=today, + personneldeployment__personnel__end_date__date__gte=today, + personneldeployment__personnel__is_active=True, + ), + distinct=True, + ), + surge_alerts_count=Count( + "surgealert", + distinct=True, + ), + # Stage + stage=Case( + When( + Exists(active_emergency_appeal_exists_qs), + then=Value(EventStage.EMERGENCY_APPEAL), + ), + When( + Exists(approved_final_report_exists_qs), + then=Value(EventStage.DREF_FINAL_REPORT), + ), + When( + Exists(approved_ops_update_exists_qs), + then=Value(EventStage.DREF_OPERATIONAL_UPDATE), + ), + When( + Exists(approved_dref_qs), + then=Value(EventStage.DREF_APPLICATION), + ), + # If there is an active appeal of DREF type, but no approved DREF yet, + # we consider the emergency to be in the Dref Appeal only stage. + # Reaches here only if no approved DREF/ops-update/final-report exists + # So an active appeal type DREF appeal with no approved DREF = DREF_APPEAL_ONLY stage. + When( + Exists(active_dref_appeal_exists_qs), + then=Value(EventStage.DREF_APPEAL_ONLY), + ), + When( + Exists(field_report_qs), + then=Value(EventStage.FIELD_REPORT), + ), + default=Value(None), + output_field=IntegerField(null=True), + ), + ) + .annotate( + # Passing values for the current stage's instance, + # to avoid extra queries in serializer. + stage_appeal_id=Case( + When( + stage=EventStage.EMERGENCY_APPEAL, + then=Subquery(active_emergency_appeal_qs.values("id")[:1]), + ), + When( + stage=EventStage.DREF_APPEAL_ONLY, + then=Subquery(active_dref_appeal_qs.values("id")[:1]), + ), + default=Value(None), + output_field=IntegerField(null=True), + ), + stage_dref_id=Case( + When( + stage__in=[ + EventStage.DREF_APPLICATION, + EventStage.DREF_OPERATIONAL_UPDATE, + EventStage.DREF_FINAL_REPORT, + ], + then=Subquery(approved_dref_qs.order_by("-created_at").values("id")[:1]), + ), + default=Value(None), + output_field=IntegerField(null=True), + ), + stage_final_report_id=Case( + When( + stage=EventStage.DREF_FINAL_REPORT, + then=Subquery(approved_final_report_qs.values("id")[:1]), + ), + default=Value(None), + output_field=IntegerField(null=True), + ), + stage_ops_update_id=Case( + When( + stage__in=[ + EventStage.DREF_OPERATIONAL_UPDATE, + EventStage.DREF_FINAL_REPORT, + ], + then=Subquery(approved_ops_update_qs.values("id")[:1]), + ), + default=Value(None), + output_field=IntegerField(null=True), + ), + stage_field_report_id=Case( + When( + stage=EventStage.FIELD_REPORT, + then=Subquery(field_report_qs.order_by("-updated_at", "-fr_num").values("id")[:1]), + ), + default=Value(None), + output_field=IntegerField(null=True), + ), + first_field_report_created_at=Subquery(field_report_qs.order_by("created_at", "fr_num").values("created_at")[:1]), + latest_field_report_created_at=Subquery( + field_report_qs.order_by("-created_at", "-fr_num").values("created_at")[:1] + ), + ) + .select_related("dtype") + .prefetch_related( + Prefetch( + "countries", + queryset=Country.objects.select_related("region"), + ), + Prefetch( + "districts", + queryset=District.objects.select_related("country"), + ), + Prefetch( + "key_figures", + queryset=KeyFigure.objects.all(), + ), + Prefetch( + "contacts", + queryset=EventContact.objects.all(), + ), + Prefetch( + "links", + queryset=EventLink.objects.all(), + ), + Prefetch( + "featured_documents", + queryset=EventFeaturedDocument.objects.order_by("-id"), + ), + Prefetch( + "field_reports", + queryset=FieldReport.objects.order_by("-updated_at", "-fr_num"), + to_attr="prefetched_timeline_field_reports", + ), + ) + ) diff --git a/api/enums.py b/api/enums.py index 83e81850e..6d9b843c2 100644 --- a/api/enums.py +++ b/api/enums.py @@ -20,4 +20,6 @@ "action_category": models.ActionCategory, "profile_org_types": models.Profile.OrgTypes, "supporting_type": models.CountrySupportingPartner.SupportingPartnerType, + "event_source": models.Event.EventSource, + "emergency_stage": models.EventStage, } diff --git a/api/factories/event.py b/api/factories/event.py index f3656a2e6..c868ff5de 100644 --- a/api/factories/event.py +++ b/api/factories/event.py @@ -12,6 +12,7 @@ AppealHistory, AppealType, Event, + EventContact, EventFeaturedDocument, EventLink, ) @@ -25,6 +26,7 @@ class Meta: name = fuzzy.FuzzyText(length=50) slug = fuzzy.FuzzyText(length=50) dtype = factory.SubFactory(DisasterTypeFactory) + source = fuzzy.FuzzyChoice(Event.EventSource) @factory.post_generation def districts(self, create, extracted, **kwargs): @@ -72,7 +74,7 @@ def regions(self, create, extracted, **kwargs): previous_update = fuzzy.FuzzyDateTime(datetime.datetime(2008, 1, 1, tzinfo=pytz.utc)) auto_generated = fuzzy.FuzzyChoice([True, False]) - auto_generated_source = fuzzy.FuzzyText(length=50) + source = fuzzy.FuzzyChoice(Event.EventSource.values) is_featured = fuzzy.FuzzyChoice([True, False]) is_featured_region = fuzzy.FuzzyChoice([True, False]) @@ -133,3 +135,9 @@ class AppealHistoryFactory(factory.django.DjangoModelFactory): class Meta: model = AppealHistory + + +class EventContactFactory(factory.django.DjangoModelFactory): + + class Meta: + model = EventContact diff --git a/api/filter_set.py b/api/filter_set.py index 625c4da7a..0f0d1ff41 100644 --- a/api/filter_set.py +++ b/api/filter_set.py @@ -2,7 +2,6 @@ from django.contrib.auth.models import User from django.db import models -from api.event_sources import SOURCES from api.models import ( Admin2, Appeal, @@ -183,10 +182,8 @@ class EventFilter(filters.FilterSet): countries__in = ListFilter(field_name="countries__id") regions__in = ListFilter(field_name="regions__id") id = filters.NumberFilter(field_name="id", lookup_expr="exact") - auto_generated_source = filters.ChoiceFilter( - label="Auto generated source choices", - choices=[(v, v) for v in SOURCES.values()], - ) + source = filters.ChoiceFilter(choices=Event.EventSource.choices, lookup_expr="exact") + is_subscribed = filters.BooleanFilter(label="is_subscribed", method="get_is_subcribed_event") class Meta: diff --git a/api/management/commands/create_events.py b/api/management/commands/create_events.py index 7078b772f..df6bef81d 100644 --- a/api/management/commands/create_events.py +++ b/api/management/commands/create_events.py @@ -1,6 +1,5 @@ from django.core.management.base import BaseCommand -from api.event_sources import SOURCES from api.models import Appeal, Event @@ -18,7 +17,7 @@ def handle(self, *args, **options): "dtype": appeal.dtype, "disaster_start_date": appeal.start_date, "auto_generated": True, - "auto_generated_source": SOURCES["appeal_admin"], + "source": Event.EventSource.APPEAL_ADMIN, } event = Event.objects.create(**fields) if appeal.country is not None: diff --git a/api/management/commands/index_and_notify.py b/api/management/commands/index_and_notify.py index d46620ffe..0156bb792 100644 --- a/api/management/commands/index_and_notify.py +++ b/api/management/commands/index_and_notify.py @@ -1036,7 +1036,7 @@ def handle(self, *args, **options): condR = Q(real_data_update__gte=time_diff) # instead of modified at cond2 = ~Q(previous_update__gte=time_diff_1_day) # negate (~) no previous_update in the last day, so send once a day condF = Q( - auto_generated_source="New field report" + source=Event.EventSource.NEW_FIELD_REPORT ) # exclude those events that were generated from field reports, to avoid 2x notif. condE = Q(status=CronJobStatus.ERRONEOUS) diff --git a/api/management/commands/ingest_gdacs.py b/api/management/commands/ingest_gdacs.py index c356d1cfc..5a99466d2 100644 --- a/api/management/commands/ingest_gdacs.py +++ b/api/management/commands/ingest_gdacs.py @@ -3,7 +3,6 @@ from dateutil.parser import parse from django.core.management.base import BaseCommand -from api.event_sources import SOURCES from api.logger import logger from api.models import Country, CronJob, CronJobStatus, DisasterType, Event, GDACSEvent @@ -104,7 +103,7 @@ def handle(self, *args, **options): "summary": data["description"], "disaster_start_date": data["publication_date"], "auto_generated": True, - "auto_generated_source": SOURCES["gdacs"], + "source": Event.EventSource.GDACS, "ifrc_severity_level": data["alert_level"], } event = Event.objects.create(**fields) diff --git a/api/management/commands/ingest_mdb.py b/api/management/commands/ingest_mdb.py index 7e8ac772d..0784016f1 100644 --- a/api/management/commands/ingest_mdb.py +++ b/api/management/commands/ingest_mdb.py @@ -13,7 +13,6 @@ from django.core.management.base import BaseCommand from django.utils import timezone -from api.event_sources import SOURCES from api.fixtures.dtype_map import PK_MAP from api.logger import logger from api.models import ( @@ -249,7 +248,7 @@ def handle(self, *args, **options): "dtype": report_dtype, "disaster_start_date": timezone.now(), "auto_generated": True, - "auto_generated_source": SOURCES["report_ingest"], + "source": Event.EventSource.FIELD_REPORT_DMIS_INGEST, } event = Event(**event_record) event.save() diff --git a/api/management/commands/ingest_who.py b/api/management/commands/ingest_who.py index bdc0704c8..55987f2f1 100644 --- a/api/management/commands/ingest_who.py +++ b/api/management/commands/ingest_who.py @@ -11,7 +11,8 @@ class Command(BaseCommand): help = "Add new event (=emergency) entries from WHO API" def handle(self, *args, **options): - guids = [e.auto_generated_source for e in Event.objects.filter(auto_generated_source__startswith="www.who.int")] + # Change this at the top + guids = list(Event.objects.filter(source=Event.EventSource.WHO).values_list("who_guid", flat=True)) logger.info("Querying WHO RSS feed for new emergency data") # get latest @@ -48,10 +49,15 @@ def handle(self, *args, **options): "category": row.pop("category"), "pubDate": row.pop("pubDate"), } - if data["guid"] in guids: - continue if data["guid"] in ["WeDontWantThis", "NeitherThis"]: continue + try: + who_guid = int(data["guid"].strip().split(".")[-1]) + except (ValueError, IndexError): + who_guid = None + + if who_guid is None or who_guid in guids: + continue title = data["title"] # for csr link short = title.replace(" (ex-China)", "") @@ -131,7 +137,8 @@ def handle(self, *args, **options): "summary": summary, "disaster_start_date": date, "auto_generated": True, - "auto_generated_source": data["guid"], + "source": Event.EventSource.WHO, + "who_guid": who_guid, "ifrc_severity_level": alert_level, } # TODO: fields['name'] sometimes exceeds 100 maxlength, so will need some altering if this will be used diff --git a/api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py b/api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py new file mode 100644 index 000000000..c81006a44 --- /dev/null +++ b/api/migrations/0232_remove_event_auto_generated_source_event_source_and_more.py @@ -0,0 +1,93 @@ +# Generated by Django 4.2.30 on 2026-05-21 04:49 + +from django.db import migrations, models + + +def migrate_sources(apps, _): + """ + Populate the source field for Event records: + - Non auto generated events are mapped to MANUAL_INPUT. + - Events that are auto generated and have auto_generated_source are mapped to the corresponding enum value. + - Events that are auto generated but do not have auto_generated_source are mapped to MANUAL_INPUT. + - For WHO, the who_guid field is populated with the id extracted from the auto_generated_source field. + """ + + Event = apps.get_model("api", "Event") + + for obj in Event.objects.iterator(): + update_fields = ["source"] + + if not obj.auto_generated: + source = 100 # MANUAL_INPUT + + else: + raw_source = (obj.auto_generated_source or "").lower() + if "gdacs" in raw_source: + source = 110 # GDACS + + elif "who.int" in raw_source: + source = 120 + + parts = raw_source.strip().split(".") + who_id = parts[-1].strip() + + try: + obj.who_guid = int(who_id) + update_fields.append("who_guid") + except (ValueError, TypeError): + pass + elif "field report dmis ingest" in raw_source: + source = 130 # FIELD_REPORT_DMIS_INGEST + elif "field report admin" in raw_source: + source = 140 # FIELD_REPORT_ADMIN + elif "appeal" in raw_source: + source = 150 # APPEAL_ADMIN + elif "new field report" in raw_source: + source = 160 # NEW_FIELD_REPORT + else: + # Using title for backward compatibility, as some events have auto_generated_source empty but title containing "GDACS" + if obj.name and obj.name.lower().startswith("gdacs"): + source = 110 # GDACS + else: + source = 100 # MANUAL_INPUT + obj.auto_generated = False + update_fields.append("auto_generated") + obj.source = source + obj.save(update_fields=update_fields) + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0231_alter_export_export_type"), + ] + + operations = [ + migrations.AddField( + model_name="event", + name="source", + field=models.IntegerField( + choices=[ + (100, "Manual input"), + (110, "GDACs scraper"), + (120, "WHO scraper"), + (130, "Field report DMIS ingest"), + (140, "Field report admin"), + (150, "Appeal admin"), + (160, "New field report"), + (170, "DREF"), + ], + default=100, + verbose_name="Event source", + ), + ), + migrations.AddField( + model_name="event", + name="who_guid", + field=models.IntegerField(blank=True, null=True, verbose_name="Who guid"), + ), + migrations.RunPython(migrate_sources, reverse_code=migrations.RunPython.noop), + migrations.RemoveField( + model_name="event", + name="auto_generated_source", + ), + ] diff --git a/api/models.py b/api/models.py index 8bfcffea9..b58bab95b 100644 --- a/api/models.py +++ b/api/models.py @@ -761,12 +761,48 @@ def snippet_image_path(instance, filename): return "emergencies/%s/%s" % (instance.id, filename) +# NOTE: Stage for the emergency timeline +class EventStage(models.IntegerChoices): + EMERGENCY_APPEAL = 1, _("Emergency Appeal") + DREF_APPLICATION = 2, _("DREF Application") + DREF_OPERATIONAL_UPDATE = 3, _("DREF Operational Update") + DREF_FINAL_REPORT = 4, _("DREF Final Report") + FIELD_REPORT = 5, _("Field Report") + DREF_APPEAL_ONLY = 6, _("DREF") + + # NOTE: If ever in future we need to create an api to update the event table # we also need to make sure to add appropriate signal to create ifrc severity level event history @reversion.register() class Event(models.Model): """A disaster, which could cover multiple countries""" + class EventSource(models.IntegerChoices): + + MANUAL_INPUT = 100, _("Manual input") + """MANUAL_INPUT: Event data manually entered by a user through the event administration interface.""" + + GDACS = 110, _("GDACs scraper") + """GDACS: Event data automatically ingested from the GDACS scraper.""" + + WHO = 120, _("WHO scraper") + """WHO: Event data automatically ingested from the (WHO) scraper.""" + + FIELD_REPORT_DMIS_INGEST = 130, _("Field report DMIS ingest") + """FIELD_REPORT_DMIS_INGEST: Event data imported through the DMIS field report.""" + + FIELD_REPORT_ADMIN = 140, _("Field report admin") + """FIELD_REPORT_ADMIN: Event data created or modified via the field report administration interface.""" + + APPEAL_ADMIN = 150, _("Appeal admin") + """APPEAL_ADMIN: Event data created or managed through the appeal administration interface.""" + + NEW_FIELD_REPORT = 160, _("New field report") + """NEW_FIELD_REPORT: Event data originating from newly created field reports.""" + + DREF = 170, _("DREF") + """DREF: Event originating records.""" + name = models.CharField(verbose_name=_("name"), max_length=256) dtype = models.ForeignKey(DisasterType, verbose_name=_("disaster type"), null=True, on_delete=models.SET_NULL) disaster_start_date = models.DateTimeField(verbose_name=_("disaster start date")) @@ -819,15 +855,15 @@ class Event(models.Model): verbose_name=_("ifrc severity level update date"), null=True, blank=True ) glide = models.CharField(verbose_name=_("glide"), max_length=18, blank=True) - + # NOTE: who_guid field Stores WHO source ID extracted from auto_generated_source. + # Example normalization: "www.who.int.247249"->247249 + who_guid = models.IntegerField(verbose_name=("Who guid"), blank=True, null=True) created_at = models.DateTimeField(verbose_name=_("created at"), auto_now_add=True) updated_at = models.DateTimeField(verbose_name=_("updated at"), auto_now=True) previous_update = models.DateTimeField(verbose_name=_("previous update"), null=True, blank=True) auto_generated = models.BooleanField(verbose_name=_("auto generated"), default=False, editable=False) - auto_generated_source = models.CharField( - verbose_name=_("auto generated source"), max_length=50, null=True, blank=True, editable=False - ) + source = models.IntegerField(choices=EventSource.choices, default=EventSource.MANUAL_INPUT, verbose_name=_("Event source")) # Meant to give the organization a way of highlighting certain, important events. is_featured = models.BooleanField(default=False, verbose_name=_("is featured on home page")) @@ -850,6 +886,11 @@ class Event(models.Model): verbose_name=_("emergency response contact email"), null=True, blank=True, max_length=255 ) + # TYPING + + id: int + pk: int + class Meta: ordering = ( "-disaster_start_date", diff --git a/api/serializers.py b/api/serializers.py index 351749947..61cc5dc1e 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -4,7 +4,10 @@ from django.conf import settings from django.contrib.auth.models import Permission, User +from django.contrib.gis.db.models import OuterRef, Subquery, Value from django.db import models, transaction +from django.db.models.fields import IntegerField +from django.db.models.query import Prefetch from django.utils import timezone from django.utils.translation import get_language as django_get_language from drf_spectacular.utils import extend_schema_field @@ -14,7 +17,13 @@ from api.tasks import generate_export_pdf from api.utils import CountryValidator, RegionValidator, generate_eap_export_url from deployments.models import EmergencyProject, Personnel, PersonnelDeployment -from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate +from dref.models import ( + Dref, + DrefFinalReport, + DrefOperationalUpdate, + PlannedIntervention, + ProposedAction, +) from eap.models import EAPRegistration, FullEAP, SimplifiedEAP from lang.models import String from lang.serializers import ModelSerializer @@ -27,7 +36,6 @@ from per.models import Overview from utils.file_check import validate_file_type -from .event_sources import SOURCES from .models import ( Action, ActionsTaken, @@ -53,6 +61,7 @@ EventFeaturedDocument, EventLink, EventSeverityLevelHistory, + EventStage, Export, ExternalPartner, FieldReport, @@ -1061,6 +1070,8 @@ class Meta: class ListMiniEventSerializer(ModelSerializer): dtype = DisasterTypeSerializer(required=False) countries_for_preview = MiniCountrySerializer(many=True, read_only=True) + latest_field_report_id = serializers.IntegerField(read_only=True) + source_display = serializers.CharField(source="get_source_display", read_only=True) class Meta: model = Event @@ -1069,9 +1080,11 @@ class Meta: "name", "slug", "dtype", - "auto_generated_source", + "source", + "source_display", "emergency_response_contact_email", "countries_for_preview", + "latest_field_report_id", ) @@ -2127,6 +2140,107 @@ class Meta: fields = "__all__" +# NOTE: Using this specific serializer for emergency page schema +# Contains information from latest field report mostly and some fields from first field report +class EmergencyPreviousFieldReportSerializer( + serializers.ModelSerializer, +): + class Meta: + model = FieldReport + fields = ( + "id", + "event_id", + "fr_num", + "start_date", + "report_date", + "created_at", + "updated_at", + ) + + +class EmergencyFieldReportSerializer( + serializers.ModelSerializer, +): + actions_taken = ActionsTakenSerializer(many=True) + status_display = serializers.CharField(source="get_status_display", read_only=True) + appeal_display = serializers.CharField(source="get_appeal_display", read_only=True) + bulletin_display = serializers.CharField(source="get_bulletin_display", read_only=True) + dtype = DisasterTypeSerializer(read_only=True) + dref_display = serializers.CharField(source="get_dref_display", read_only=True) + visibility_display = serializers.CharField(source="get_visibility_display", read_only=True) + imminent_dref_display = serializers.CharField(source="get_imminent_dref_display", read_only=True) + contacts = FieldReportContactSerializer(many=True) + countries = MiniCountrySerializer(many=True) + districts = MiniDistrictSerializer(many=True) + + first_fr_ns_request_assistance = serializers.BooleanField(read_only=True) + first_fr_request_assistance = serializers.BooleanField(read_only=True) + + class Meta: + model = FieldReport + fields = ( + "id", + "summary", + "countries", + "districts", + "status", + "status_display", + "appeal", + "appeal_display", + "bulletin", + "bulletin_display", + "imminent_dref", + "imminent_dref_display", + "visibility", + "visibility_display", + "dref", + "dref_display", + "description", + "dtype", + "contacts", + # Key Figures + # IFRC figures + "num_injured", + "num_dead", + "num_missing", + "num_affected", + "num_displaced", + "epi_num_dead", + "num_assisted", + "num_localstaff", + "num_volunteers", + "num_expats_delegates", + "num_highest_risk", + "num_potentially_affected", + # Government figures + "gov_num_injured", + "gov_num_dead", + "gov_num_missing", + "gov_num_affected", + "gov_num_displaced", + "gov_num_assisted", + "gov_num_highest_risk", + "gov_num_potentially_affected", + # Other figures + "other_num_injured", + "other_num_dead", + "other_num_missing", + "other_num_affected", + "other_num_displaced", + "other_num_assisted", + "other_num_highest_risk", + "other_num_potentially_affected", + "start_date", + "report_date", + "created_at", + "updated_at", + "actions_taken", + # First field report related fields, annotate + "first_fr_ns_request_assistance", + "first_fr_request_assistance", + ) + + class DetailFieldReportSerializer(FieldReportEnumDisplayMixin, ModelSerializer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -2218,7 +2332,7 @@ def create_event(self, report): summary=report.description or "", disaster_start_date=report.start_date, auto_generated=True, - auto_generated_source=SOURCES["new_report"], + source=Event.EventSource.NEW_FIELD_REPORT, visibility=report.visibility, **{TRANSLATOR_ORIGINAL_LANGUAGE_FIELD_NAME: django_get_language()}, ) @@ -2683,3 +2797,236 @@ class CountrySupportingPartnerSerializer(serializers.ModelSerializer): class Meta: model = CountrySupportingPartner fields = "__all__" + + +class TimelineEmergencyFieldReportSerializer(serializers.ModelSerializer): + class Meta: + model = FieldReport + fields = ( + "id", + "report_date", + "fr_num", + "start_date", + ) + + +class DetailEmergencySerializer(ModelSerializer): + from dref.serializers import EmergencyDrefSerializer + + contacts = EventContactSerializer(many=True, read_only=True) + key_figures = KeyFigureSerializer(many=True, read_only=True) + countries = MiniCountrySerializer(many=True, read_only=True) + ifrc_severity_level_display = serializers.CharField(source="get_ifrc_severity_level_display", read_only=True) + visibility_display = serializers.CharField(source="get_visibility_display", read_only=True) + source_display = serializers.CharField(source="get_source_display", read_only=True) + links = EventLinkSerializer(many=True, read_only=True) + districts = MiniDistrictSerializer(many=True) + featured_documents = EventFeaturedDocumentSerializer(many=True, read_only=True) + + # NOTE: Populated from Queryset using Annotate + + # Aggregated values + response_activity_count = serializers.IntegerField(read_only=True) + active_deployments_count = serializers.IntegerField(read_only=True) + surge_alerts_count = serializers.IntegerField(read_only=True) + + # Stages + stage = serializers.IntegerField(read_only=True) + stage_display = serializers.SerializerMethodField() + + field_report = serializers.SerializerMethodField() + appeal = serializers.SerializerMethodField() + dref = serializers.SerializerMethodField() + + # Operational timeframe + first_field_report_created_at = serializers.DateTimeField(read_only=True) + latest_field_report_created_at = serializers.DateTimeField(read_only=True) + + timeline_field_reports = serializers.SerializerMethodField() + + class Meta: + model = Event + fields = ( + "id", + "slug", + "name", + "dtype", + "glide", + "countries", + "summary", + "disaster_start_date", + "auto_generated", + "source", + "source_display", + "key_figures", + "is_featured", + "is_featured_region", + "tab_one_title", + "tab_two_title", + "tab_three_title", + "hide_attached_field_reports", + "hide_field_report_map", + "ifrc_severity_level", + "ifrc_severity_level_display", + "ifrc_severity_level_update_date", + "parent_event", + "emergency_response_contact_email", + "visibility", + "visibility_display", + "contacts", + "links", + "districts", + "featured_documents", + "num_injured", + "num_dead", + "num_missing", + "num_affected", + "num_displaced", + "created_at", + "updated_at", + "previous_update", + # Aggregated values + "response_activity_count", + "active_deployments_count", + "surge_alerts_count", + # Stages + "stage", + "stage_display", + "field_report", + "appeal", + "dref", + "first_field_report_created_at", + "latest_field_report_created_at", + "timeline_field_reports", + ) + + @extend_schema_field(TimelineEmergencyFieldReportSerializer(many=True)) + def get_timeline_field_reports(self, event): + field_reports = getattr(event, "prefetched_timeline_field_reports", None) + if field_reports is None: + field_reports = FieldReport.objects.filter(event=event).order_by("-updated_at", "-fr_num") + serializer = TimelineEmergencyFieldReportSerializer(field_reports, many=True) + return serializer.data + + def _get_stage_instance(self, event): + stage = getattr(event, "stage") + if stage == EventStage.FIELD_REPORT and event.stage_field_report_id: + _first_field_report_queryset = FieldReport.objects.filter(event_id=OuterRef("event_id")).order_by( + "created_at", "fr_num" + ) + instance = ( + FieldReport.objects.select_related( + "dtype", + "event", + ) + .annotate( + first_fr_ns_request_assistance=Subquery(_first_field_report_queryset.values("ns_request_assistance")[:1]), + first_fr_request_assistance=Subquery(_first_field_report_queryset.values("request_assistance")[:1]), + ) + .prefetch_related( + "contacts", + "countries", + "districts", + Prefetch( + "actions_taken", + queryset=ActionsTaken.objects.prefetch_related("actions"), + ), + ) + .get(pk=event.stage_field_report_id) + ) + return instance + + elif ( + stage + in [ + EventStage.EMERGENCY_APPEAL, + EventStage.DREF_APPEAL_ONLY, + ] + and event.stage_appeal_id + ): + instance = Appeal.objects.get(pk=event.stage_appeal_id) + return instance + elif ( + stage + in [ + EventStage.DREF_APPLICATION, + EventStage.DREF_OPERATIONAL_UPDATE, + EventStage.DREF_FINAL_REPORT, + ] + and event.stage_dref_id + ): + instance = ( + Dref.objects.select_related( + "country", + "disaster_type", + "cover_image", + "cover_image__created_by", + ) + .annotate( + operational_update_id=Value( + event.stage_ops_update_id, + output_field=IntegerField(null=True), + ), + final_report_id=Value( + event.stage_final_report_id, + output_field=IntegerField(null=True), + ), + ) + .prefetch_related( + "district", + Prefetch( + "planned_interventions", + queryset=PlannedIntervention.objects.prefetch_related("indicators"), + ), + Prefetch( + "proposed_action", + queryset=ProposedAction.objects.prefetch_related("activities"), + ), + Prefetch( + "drefoperationalupdate_set", + queryset=DrefOperationalUpdate.objects.filter( + status=Dref.Status.APPROVED, + ).order_by("operational_update_number"), + to_attr="prefetched_timeline_ops_updates", + ), + ) + .get(pk=event.stage_dref_id) + ) + return instance + + def get_stage_display(self, event): + stage = getattr(event, "stage", None) + return EventStage(stage).label if stage is not None else None + + @extend_schema_field(EmergencyFieldReportSerializer()) + def get_field_report(self, event): + if getattr(event, "stage", None) != EventStage.FIELD_REPORT: + return None + instance = self._get_stage_instance(event) + return EmergencyFieldReportSerializer(instance, context=self.context).data if instance else None + + @extend_schema_field(RelatedAppealSerializer()) + def get_appeal(self, event): + if getattr(event, "stage", None) not in ( + EventStage.EMERGENCY_APPEAL, + EventStage.DREF_APPEAL_ONLY, + ): + return None + instance = self._get_stage_instance(event) + return RelatedAppealSerializer(instance, context=self.context).data if instance else None + + @extend_schema_field(EmergencyDrefSerializer()) + def get_dref(self, event): + if getattr(event, "stage", None) not in ( + EventStage.DREF_APPLICATION, + EventStage.DREF_OPERATIONAL_UPDATE, + EventStage.DREF_FINAL_REPORT, + ): + return None + instance = self._get_stage_instance(event) + if not instance: + return None + + from dref.serializers import EmergencyDrefSerializer + + return EmergencyDrefSerializer(instance, context=self.context).data diff --git a/api/test_views.py b/api/test_views.py index 246ab9a2b..a54da83a1 100644 --- a/api/test_views.py +++ b/api/test_views.py @@ -1,3 +1,4 @@ +import datetime import os import re import uuid @@ -10,19 +11,29 @@ from django.test import override_settings from django.urls import reverse from django.utils import timezone +from django.utils.dateparse import parse_datetime import api.models as models +from api.factories.country import CountryFactory +from api.factories.disaster_type import DisasterTypeFactory from api.factories.event import ( AppealFactory, AppealType, + EventContactFactory, EventFactory, EventFeaturedDocumentFactory, EventLinkFactory, ) from api.factories.field_report import FieldReportFactory -from api.models import Profile, VisibilityChoices +from api.models import AppealStatus, EventStage, Profile, VisibilityChoices from deployments.factories.user import UserFactory -from dref.models import DrefFile +from dref.factories.dref import ( + DrefFactory, + DrefFinalReportFactory, + DrefOperationalUpdateFactory, + DrefSummaryFactory, +) +from dref.models import Dref, DrefFile, DrefSummary from main.test_case import APITestCase from per.factories import OpsLearningFactory @@ -1048,3 +1059,514 @@ def test_ifrc_user_sees_all(self): ] ), ) + + +class EmergencyViewTestCase(APITestCase): + def setUp(self): + super().setUp() + self.disaster_type = DisasterTypeFactory.create( + name="Flood", + ) + + self.event1 = EventFactory.create( + dtype=self.disaster_type, + source=models.Event.EventSource.GDACS, + slug="test1", + parent_event=None, + ) + self.contact = EventContactFactory.create( + event=self.event1, + ) + + self.field_report = FieldReportFactory.create( + event=self.event1, + created_at=datetime.datetime(2024, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc), + updated_at=datetime.datetime(2026, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc), + fr_num=50, + ) + self.event2 = EventFactory.create( + dtype=self.disaster_type, + source=models.Event.EventSource.WHO, + slug="test2", + parent_event=None, + ) + + self.event3 = EventFactory.create( + dtype=self.disaster_type, + source=models.Event.EventSource.APPEAL_ADMIN, + slug="test3", + parent_event=None, + ) + self.appeal2 = AppealFactory.create( + event=self.event3, + dtype=self.disaster_type, + num_beneficiaries=9000, + amount_requested=10000, + amount_funded=1899999, + ) + + def test_retrive_emergency_detail(self): + url = f"/api/v2/emergency/{self.event1.id}/" + response = self.client.get(url) + self.assert_200(response) + self.assertEqual(response.data["id"], self.event1.id) + self.assertEqual(response.data["slug"], self.event1.slug) + self.assertEqual(response.data["name"], self.event1.name) + self.assertEqual(response.data["source"], models.Event.EventSource.GDACS) + + # Stage check + self.assertEqual(response.data["stage"], EventStage.FIELD_REPORT, response.data) + self.assertEqual(response.data["field_report"]["id"], self.field_report.id, response.data) + + +class EmergencyStageTestCase(APITestCase): + """ + Tests for the stage annotation and stage-specific nested serializer + fields (field_report / appeal / dref) on GET /api/v2/emergency// + """ + + def setUp(self): + super().setUp() + self.disaster_type = DisasterTypeFactory.create(name="Flood") + self.country = CountryFactory.create(name="country1", iso3="DEP", iso="DE") + + def _url(self, event): + return f"/api/v2/emergency/{event.id}/" + + def _get(self, event): + return self.client.get(self._url(event)) + + def _approved_dref(self, event): + return DrefFactory.create( + event=event, + status=Dref.Status.APPROVED, + disaster_type=self.disaster_type, + country=self.country, + ) + + def _approved_ops_update(self, dref): + return DrefOperationalUpdateFactory.create( + dref=dref, + status=Dref.Status.APPROVED, + disaster_type=self.disaster_type, + country=self.country, + ) + + def _approved_final_report(self, dref): + return DrefFinalReportFactory.create( + dref=dref, + status=Dref.Status.APPROVED, + disaster_type=self.disaster_type, + country=self.country, + ) + + def _active_emergency_appeal(self, event): + return AppealFactory.create( + event=event, + dtype=self.disaster_type, + status=AppealStatus.ACTIVE, + atype=AppealType.APPEAL, + ) + + def test_stage_is_none(self): + event = EventFactory.create( + dtype=self.disaster_type, + source=models.Event.EventSource.MANUAL_INPUT, + ) + + data = self._get(event).data + + self.assertIsNone(data["stage"]) + self.assertIsNone(data["stage_display"]) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["appeal"]) + self.assertIsNone(data["dref"]) + + def test_stage_field_report(self): + event = EventFactory.create(dtype=self.disaster_type, source=models.Event.EventSource.MANUAL_INPUT) + field_report = FieldReportFactory.create(event=event) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.FIELD_REPORT) + self.assertEqual(data["stage_display"], EventStage(EventStage.FIELD_REPORT).label) + self.assertEqual(data["field_report"]["id"], field_report.id) + self.assertIsNone(data["appeal"]) + self.assertIsNone(data["dref"]) + + def test_stage_field_report_returns_latest_by_updated_at_and_fr_num(self): + """ + When multiple field reports exist, the one ordered by + (-updated_at, -fr_num) should be returned. + """ + event = EventFactory.create(dtype=self.disaster_type) + first_fr = FieldReportFactory.create( + event=event, + dtype=self.disaster_type, + created_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2024, 6, 1, tzinfo=datetime.UTC), + fr_num=1, + ) + + latest_fr = FieldReportFactory.create( + event=event, + dtype=self.disaster_type, + created_at=datetime.datetime(2025, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + fr_num=2, + ) + + data = self._get(event).data + + self.assertIsNotNone(data["field_report"], data) + self.assertEqual(data["field_report"]["id"], latest_fr.id) + self.assertEqual(data["stage"], EventStage.FIELD_REPORT) + self.assertEqual(parse_datetime(data["first_field_report_created_at"]), first_fr.created_at) + self.assertEqual(parse_datetime(data["latest_field_report_created_at"]), latest_fr.created_at) + + def test_stage_dref_application(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + DrefOperationalUpdateFactory.create( + dref=dref, + status=Dref.Status.DRAFT, + disaster_type=self.disaster_type, + country=self.country, + ) + + data = self._get(event).data + + self.assertEqual( + data["stage"], + EventStage.DREF_APPLICATION.value, + ) + + self.assertEqual( + data["stage_display"], + EventStage(EventStage.DREF_APPLICATION).label, + ) + + self.assertEqual(data["dref"]["id"], dref.id) + self.assertIsNone(data["dref"]["operational_update_details"]) + self.assertIsNone(data["dref"]["final_report_details"]) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["appeal"]) + + # After Final report create + dref_final_report = DrefFinalReportFactory.create( + dref=dref, + status=Dref.Status.APPROVED, + disaster_type=self.disaster_type, + country=self.country, + ) + data = self._get(event).data + + self.assertEqual( + data["stage"], + EventStage.DREF_FINAL_REPORT.value, + ) + self.assertEqual( + data["stage_display"], + EventStage(EventStage.DREF_FINAL_REPORT).label, + ) + self.assertEqual(data["dref"]["id"], dref.id) + self.assertIsNone(data["dref"]["operational_update_details"]) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["appeal"]) + self.assertIsNotNone(data["dref"]["final_report_details"]) + self.assertEqual(data["dref"]["final_report_details"]["id"], dref_final_report.id) + + def test_stage_dref_operational_update(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + ops_update = self._approved_ops_update(dref) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_OPERATIONAL_UPDATE) + self.assertEqual(data["dref"]["id"], dref.id) + self.assertEqual(data["dref"]["operational_update_details"]["id"], ops_update.id) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["appeal"]) + + def test_stage_dref_final_report(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + final_report = self._approved_final_report(dref) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_FINAL_REPORT) + self.assertEqual(data["stage_display"], EventStage(EventStage.DREF_FINAL_REPORT).label) + self.assertEqual(data["dref"]["id"], dref.id) + self.assertEqual(data["dref"]["final_report_details"]["id"], final_report.id) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["appeal"]) + + def test_stage_emergency_appeal(self): + event = EventFactory.create( + dtype=self.disaster_type, + source=models.Event.EventSource.MANUAL_INPUT, + ) + appeal = self._active_emergency_appeal(event) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + self.assertEqual(data["stage_display"], EventStage(EventStage.EMERGENCY_APPEAL).label) + self.assertEqual(data["appeal"]["id"], appeal.id) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["dref"]) + + # If Field report is created + first_fr = FieldReportFactory.create( + event=event, + dtype=self.disaster_type, + created_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2024, 6, 1, tzinfo=datetime.UTC), + fr_num=1, + ) + latest_fr = FieldReportFactory.create( + event=event, + dtype=self.disaster_type, + created_at=datetime.datetime(2025, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + fr_num=2, + ) + + data = self._get(event).data + # Emergency appeal stage should take priority over field report stage + # BUT Fields Report dates for latest and first should be present + self.assertEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + + self.assertEqual(parse_datetime(data["first_field_report_created_at"]), first_fr.created_at) + self.assertEqual(parse_datetime(data["latest_field_report_created_at"]), latest_fr.created_at) + + def test_inactive_appeal_does_not_trigger_emergency_appeal_stage(self): + """An appeal that is not ACTIVE should not resolve to EMERGENCY_APPEAL.""" + event = EventFactory.create(dtype=self.disaster_type) + AppealFactory.create( + event=event, + dtype=self.disaster_type, + status=AppealStatus.ARCHIVED, + atype=AppealType.APPEAL, + ) + + data = self._get(event).data + self.assertNotEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + + # CLOSED one also considered but ACTIVE takes over + AppealFactory.create( + event=event, + dtype=self.disaster_type, + status=AppealStatus.CLOSED, + atype=AppealType.APPEAL, + ) + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + self.assertEqual(data["appeal"]["status"], AppealStatus.CLOSED) + + self._active_emergency_appeal(event) + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + + # Active should take priority over closed and archived + self.assertEqual(data["appeal"]["status"], AppealStatus.ACTIVE) + + def test_emergency_appeal_takes_priority_over_entire_dref_chain(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + self._approved_ops_update(dref) + self._approved_final_report(dref) + self._active_emergency_appeal(event) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + + def test_dref_final_report_takes_priority_over_ops_update_and_application(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + self._approved_ops_update(dref) + self._approved_final_report(dref) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_FINAL_REPORT) + + def test_dref_ops_update_takes_priority_over_dref_application(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + self._approved_ops_update(dref) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_OPERATIONAL_UPDATE) + + # Create DRAFT Final report, stage should still be DREF_OPERATIONAL_UPDATE until Final report is APPROVED + DrefFinalReportFactory.create( + dref=dref, + status=Dref.Status.DRAFT, + disaster_type=self.disaster_type, + country=self.country, + ) + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.DREF_OPERATIONAL_UPDATE) + self.assertIsNone(data["appeal"]) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["dref"]["final_report_details"]) + + def test_dref_application_takes_priority_over_field_report(self): + event = EventFactory.create(dtype=self.disaster_type) + FieldReportFactory.create(event=event) + dref = self._approved_dref(event) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_APPLICATION) + + # Create DRAFT ops update, stage should still be DREF Application until ops update is APPROVED + DrefOperationalUpdateFactory.create( + dref=dref, + status=Dref.Status.DRAFT, + disaster_type=self.disaster_type, + country=self.country, + ) + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.DREF_APPLICATION) + self.assertIsNone(data["appeal"]) + self.assertIsNone(data["field_report"]) + + # Create DRAFT Final report, stage should still be DREF Aplication until Final report is APPROVED + DrefFinalReportFactory.create( + dref=dref, + status=Dref.Status.DRAFT, + disaster_type=self.disaster_type, + country=self.country, + ) + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_APPLICATION) + self.assertIsNone(data["appeal"]) + self.assertIsNone(data["field_report"]) + self.assertIsNone(data["dref"]["final_report_details"]) + + def fallback_to_dref_appeal_for_no_approved_dref(self): + """ + If no approved DREF application exists, but an ACTIVE appeal dref type exists, stage should resolve to DREF_APPEAL_ONLY + and not DREF. + """ + event = EventFactory.create(dtype=self.disaster_type) + AppealFactory.create( + event=event, + dtype=self.disaster_type, + status=AppealStatus.ACTIVE, + atype=AppealType.DREF, + ) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.EMERGENCY_APPEAL) + self.assertIsNone(data["dref"]) + self.assertIsNone(data["field_report"]) + self.assertIsNotNone(data["appeal"]) + + # If Field report is created + first_fr = FieldReportFactory.create( + event=event, + dtype=self.disaster_type, + created_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2024, 6, 1, tzinfo=datetime.UTC), + fr_num=1, + ) + latest_fr = FieldReportFactory.create( + event=event, + dtype=self.disaster_type, + created_at=datetime.datetime(2025, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + fr_num=2, + ) + + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.DREF_APPEAL_ONLY) + + self.assertEqual(parse_datetime(data["first_field_report_created_at"]), first_fr.created_at) + self.assertEqual(parse_datetime(data["latest_field_report_created_at"]), latest_fr.created_at) + + def test_field_report_with_timeline_field_reports(self): + event = EventFactory.create(dtype=self.disaster_type) + FieldReportFactory.create_batch( + 3, + event=event, + dtype=self.disaster_type, + ) + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.FIELD_REPORT, data) + self.assertIsNotNone(data["field_report"], data) + self.assertEqual(len(data["timeline_field_reports"]), 3) + + def test_dref_operational_update_with_timeline_ops_updates(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + DrefOperationalUpdateFactory.create_batch( + 3, + dref=dref, + status=Dref.Status.APPROVED, + disaster_type=self.disaster_type, + country=self.country, + ) + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.DREF_OPERATIONAL_UPDATE, data) + + dref_data = data.get("dref") + self.assertIsNotNone(dref_data, data) + + self.assertEqual(len(dref_data["timeline_operational_updates"]), 3) + + # NOTE: if Final report is created, stage should be DREF_FINAL_REPORT and previous operational updates should also show + DrefFinalReportFactory.create( + dref=dref, + status=Dref.Status.APPROVED, + disaster_type=self.disaster_type, + country=self.country, + ) + data = self._get(event).data + self.assertEqual(data["stage"], EventStage.DREF_FINAL_REPORT) + dref_data = data.get("dref") + self.assertIsNotNone(dref_data) + self.assertIsNotNone(dref_data["final_report_details"]) + self.assertEqual(len(dref_data["timeline_operational_updates"]), 3) + + def test_dref_summary_fields_present_when_summary_exists(self): + event = EventFactory.create(dtype=self.disaster_type) + dref = self._approved_dref(event) + DrefSummaryFactory.create( + dref=dref, + status=DrefSummary.SummaryStatus.SUCCESS, + situational_overview="overview text", + operational_strategy="strategy text", + people_centered_approach="approach text", + challenges_identified="challenges text", + lessons_learned="lessons text", + ) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_APPLICATION) + summary = data["dref"]["summary"] + self.assertIsNotNone(summary) + self.assertEqual(summary["status"], DrefSummary.SummaryStatus.SUCCESS) + self.assertEqual(summary["situational_overview"], "overview text") + self.assertEqual(summary["operational_strategy"], "strategy text") + self.assertEqual(summary["people_centered_approach"], "approach text") + self.assertEqual(summary["challenges_identified"], "challenges text") + self.assertEqual(summary["lessons_learned"], "lessons text") + + def test_dref_summary_is_none_when_no_summary_exists(self): + event = EventFactory.create(dtype=self.disaster_type) + self._approved_dref(event) + + data = self._get(event).data + + self.assertEqual(data["stage"], EventStage.DREF_APPLICATION) + self.assertIsNone(data["dref"]["summary"]) diff --git a/assets b/assets index b386998bb..7c5b98dfc 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit b386998bb3efa9dc7145371c63cb42e4c1ceeab9 +Subproject commit 7c5b98dfc773808c7a918040f386cc3b11929a2e diff --git a/deployments/migrations/0096_alter_emergencyproject_event.py b/deployments/migrations/0096_alter_emergencyproject_event.py new file mode 100644 index 000000000..f699854dd --- /dev/null +++ b/deployments/migrations/0096_alter_emergencyproject_event.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.30 on 2026-05-25 12:01 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0232_remove_event_auto_generated_source_event_source_and_more"), + ( + "deployments", + "0095_molnix_appraisal_appraised_person_and_event_participation_constraint", + ), + ] + + operations = [ + migrations.AlterField( + model_name="emergencyproject", + name="event", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="emergency_projects", + to="api.event", + verbose_name="Event", + ), + ), + ] diff --git a/deployments/models.py b/deployments/models.py index ee3ef795b..0fa470b46 100644 --- a/deployments/models.py +++ b/deployments/models.py @@ -817,7 +817,10 @@ class ActivityStatus(models.TextChoices): related_name="+", ) event = models.ForeignKey( - Event, verbose_name=_("Event"), on_delete=models.CASCADE, related_name="+" + Event, + verbose_name=_("Event"), + on_delete=models.CASCADE, + related_name="emergency_projects", ) # this is the current operation districts = models.ManyToManyField( District, diff --git a/deployments/tests.py b/deployments/tests.py index 456f8fac7..af709512d 100644 --- a/deployments/tests.py +++ b/deployments/tests.py @@ -266,6 +266,8 @@ def test_project_update(self): new_country = country.CountryFactory( name="country-2", society_name="society-name-2", + iso="Te", + iso3="frj", ) new_district = district.DistrictFactory( country=new_country, @@ -685,9 +687,24 @@ def setUp(self): disaster_start_date=datetime.datetime(2025, 3, 1), ) - self.country1 = country.CountryFactory(name="Test Country1") - self.country2 = country.CountryFactory(name="Test Country2") - self.country3 = country.CountryFactory(name="Test Country3") + self.country1 = country.CountryFactory.create( + name="Test Country1", + iso3="Ts1", + iso="T1", + society_name="Test Society", + ) + self.country2 = country.CountryFactory.create( + name="Test Country2", + iso3="TS2", + iso="T2", + society_name="Test Society2", + ) + self.country3 = country.CountryFactory.create( + name="Test Country3", + iso3="TS3", + iso="T3", + society_name="Test Society3", + ) self.eru_owner = ERUOwnerFactory( national_society_country=self.country1, diff --git a/docker-compose.yml b/docker-compose.yml index 6ff7c3f57..a9a8faf89 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,8 @@ x-server: &base_server_setup CACHE_TEST_REDIS_URL: ${CACHE_TEST_REDIS_URL:-redis://redis:6379/11} CACHE_MIDDLEWARE_SECONDS: ${CACHE_MIDDLEWARE_SECONDS:-600} ELASTIC_SEARCH_HOST: ${ELASTIC_SEARCH_HOST:-elasticsearch://elasticsearch:9200} + # LLM + USE_DUMMY_LLM_CLIENT: ${USE_DUMMY_LLM_CLIENT:-true} # Appeal API APPEALS_USER: ${APPEALS_USER:-} APPEALS_PASS: ${APPEALS_PASS:-} diff --git a/dref/admin.py b/dref/admin.py index 7b38c4393..390b4c4d9 100644 --- a/dref/admin.py +++ b/dref/admin.py @@ -1,13 +1,14 @@ -from django.contrib import admin +from django.contrib import admin, messages from reversion_compare.admin import CompareVersionAdmin -from lang.admin import TranslationAdmin +from lang.admin import TranslationAdmin, TranslationInlineModelAdmin from .models import ( Dref, DrefFile, DrefFinalReport, DrefOperationalUpdate, + DrefSummary, IdentifiedNeed, NationalSocietyAction, PlannedIntervention, @@ -15,6 +16,7 @@ RiskSecurity, SourceInformation, ) +from .tasks import generate_dref_summary class ReadOnlyMixin: @@ -125,8 +127,31 @@ class SourceInformationAdmin(admin.ModelAdmin): search_fields = ("source_name",) +class DrefSummaryInline(admin.StackedInline, TranslationInlineModelAdmin): + model = DrefSummary + extra = 0 + readonly_fields = ( + "status", + "source_hash", + "created_at", + "updated_at", + ) + fields = ( + "status", + "source_hash", + "situational_overview", + "operational_strategy", + "people_centered_approach", + "challenges_identified", + "lessons_learned", + "created_at", + "updated_at", + ) + + @admin.register(Dref) class DrefAdmin(CompareVersionAdmin, TranslationAdmin, admin.ModelAdmin): + inlines = [DrefSummaryInline] search_fields = ("title", "appeal_code") list_display = ( "title", @@ -149,7 +174,7 @@ class DrefAdmin(CompareVersionAdmin, TranslationAdmin, admin.ModelAdmin): "images", "cover_image", "users", - "field_report", + "event", "supporting_document", "national_society_actions", "needs_identified", @@ -171,7 +196,7 @@ def get_queryset(self, request): "event_map", "cover_image", "country", - "field_report", + "event", "supporting_document", ) .prefetch_related("planned_interventions", "needs_identified", "national_society_actions", "users", "risk_security") @@ -301,3 +326,37 @@ def save_model(self, request, obj, form, change): @admin.register(ProposedAction) class ProposedActionAdmin(ReadOnlyMixin, admin.ModelAdmin): search_fields = ["action"] + + +@admin.register(DrefSummary) +class DrefSummaryAdmin(TranslationAdmin, admin.ModelAdmin): + list_display = ("dref", "status", "source", "source_id", "created_at", "updated_at") + list_filter = ("status",) + search_fields = ("dref__title", "dref__appeal_code") + readonly_fields = ("source_hash", "created_at", "updated_at") + autocomplete_fields = ("dref",) + actions = ["regenerate_summary"] + fields = ( + "dref", + "status", + "source_hash", + "situational_overview", + "operational_strategy", + "people_centered_approach", + "challenges_identified", + "lessons_learned", + "created_at", + "updated_at", + ) + + @admin.action(description="Regenerate summary for selected DREFs") + def regenerate_summary(self, request, queryset): + """Re-trigger summary generation from the DREF's current latest approved source.""" + queryset.filter(status=DrefSummary.SummaryStatus.PROCESSING).update(status=DrefSummary.SummaryStatus.FAILED) + for summary in queryset: + generate_dref_summary.delay(dref_id=summary.dref_id, overwrite=True) + self.message_user( + request, + f"Queued summary regeneration for {queryset.count()} DREF(s).", + messages.SUCCESS, + ) diff --git a/dref/factories/dref.py b/dref/factories/dref.py index ceabd59d0..4cc61d27a 100644 --- a/dref/factories/dref.py +++ b/dref/factories/dref.py @@ -9,6 +9,7 @@ DrefFile, DrefFinalReport, DrefOperationalUpdate, + DrefSummary, IdentifiedNeed, NationalSocietyAction, PlannedIntervention, @@ -198,3 +199,19 @@ class Meta: model = ProposedAction proposed_type = fuzzy.FuzzyChoice(ProposedAction.Action) + + +class DrefSummaryFactory(factory.django.DjangoModelFactory): + class Meta: + model = DrefSummary + + dref = factory.SubFactory(DrefFactory) + source_hash = factory.Sequence(lambda n: f"{n:064d}") + source = DrefSummary.SourceModel.DREF + source_id = factory.SelfAttribute("dref.id") + status = DrefSummary.SummaryStatus.SUCCESS + situational_overview = fuzzy.FuzzyText(length=100) + operational_strategy = fuzzy.FuzzyText(length=100) + people_centered_approach = fuzzy.FuzzyText(length=100) + challenges_identified = fuzzy.FuzzyText(length=100) + lessons_learned = fuzzy.FuzzyText(length=100) diff --git a/dref/filter_set.py b/dref/filter_set.py index 2e27ebf18..cc41bad90 100644 --- a/dref/filter_set.py +++ b/dref/filter_set.py @@ -40,6 +40,7 @@ class BaseDrefFilterSet(filters.FilterSet): queryset=DisasterType.objects.all(), ) appeal_code = filters.CharFilter(field_name="appeal_code", lookup_expr="icontains") + event = filters.NumberFilter(field_name="event", lookup_expr="exact") class CompletedDrefOperationsFilterSet(BaseDrefFilterSet): diff --git a/dref/management/commands/migrate_dref_event.py b/dref/management/commands/migrate_dref_event.py new file mode 100644 index 000000000..74b98d748 --- /dev/null +++ b/dref/management/commands/migrate_dref_event.py @@ -0,0 +1,107 @@ +from django.core.management.base import BaseCommand +from django.db.models import Case, IntegerField, Value, When + +from api.models import Appeal, AppealType +from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate + + +class Command(BaseCommand): + help = "Migrate related Event from Appeal / Dref Final / Operational Update to Dref" + + def handle(self, *args, **options): + self.stdout.write(self.style.NOTICE("Starting migration of events to Dref...")) + + appeal_event_map = dict( + Appeal.objects.filter( + atype=AppealType.DREF, + event__isnull=False, + ) + .exclude(code__isnull=True) + .values_list("code", "event_id") + ) + + final_report_map = dict( + DrefFinalReport.objects.exclude(appeal_code__isnull=True) + .annotate( + approval_priority=Case( + When(status=Dref.Status.APPROVED, then=Value(0)), + default=Value(1), + output_field=IntegerField(), + ) + ) + .order_by("dref_id", "approval_priority") + .distinct("dref_id") + .values_list("dref_id", "appeal_code") + ) + + operational_update_map = dict( + DrefOperationalUpdate.objects.exclude(appeal_code__isnull=True) + .annotate( + approval_priority=Case( + When(status=Dref.Status.APPROVED, then=Value(0)), + default=Value(1), + output_field=IntegerField(), + ) + ) + .order_by("dref_id", "approval_priority", "-created_at") + .distinct("dref_id") + .values_list("dref_id", "appeal_code") + ) + + dref_queryset = Dref.objects.only("id", "title", "appeal_code", "event_id") + self.stdout.write(self.style.NOTICE("\nMigrating Dref Event Mapping")) + + drefs_to_update = [] + for dref in dref_queryset.iterator(): + self.stdout.write(self.style.NOTICE(f"\nDref: {dref.title} (id={dref.id})")) + + appeal_code = None + matched_with = None + + self.stdout.write(self.style.NOTICE("\tChecking Final Report")) + if final_report_map.get(dref.id): + appeal_code = final_report_map[dref.id] + matched_with = "DREF FINAL REPORT" + self.stdout.write(self.style.SUCCESS(f"\t\tFound appeal_code from Final Report ({appeal_code})")) + else: + self.stdout.write(self.style.WARNING("\t\tNo appeal_code in Final Report")) + + if not appeal_code: + self.stdout.write(self.style.NOTICE("\tChecking Operational Update")) + if operational_update_map.get(dref.id): + appeal_code = operational_update_map[dref.id] + matched_with = "DREF OPERATIONAL UPDATE" + self.stdout.write(self.style.SUCCESS(f"\t\tFound appeal_code from Operational Update ({appeal_code})")) + else: + self.stdout.write(self.style.WARNING("\t\tNo appeal_code in Operational Update")) + + if not appeal_code: + self.stdout.write(self.style.NOTICE("\tChecking Dref")) + if dref.appeal_code: + appeal_code = dref.appeal_code + matched_with = "DREF" + self.stdout.write(self.style.SUCCESS(f"\t\tFound appeal_code from Dref ({appeal_code})")) + else: + self.stdout.write(self.style.WARNING("\t\tNo appeal_code in Dref")) + + if not appeal_code: + self.stdout.write(self.style.WARNING("\tNo appeal_code found, skipping...")) + continue + + new_event_id = appeal_event_map.get(appeal_code) + if not new_event_id: + self.stdout.write(self.style.WARNING(f"\tNo matching Appeal found for appeal_code={appeal_code}, skipping...")) + continue + + if dref.event_id and dref.event_id != new_event_id: + self.stdout.write( + self.style.WARNING(f"\tConflict: existing_event_id={dref.event_id}, new_event_id={new_event_id} (skipped)") + ) + continue + + dref.event_id = new_event_id + drefs_to_update.append(dref) + self.stdout.write(self.style.SUCCESS(f"\tUpdating event_id={new_event_id} using {matched_with}")) + + Dref.objects.bulk_update(drefs_to_update, ["event_id"]) + self.stdout.write(self.style.SUCCESS(f"\nSuccessfully updated {len(drefs_to_update)} Dref records\n")) diff --git a/dref/migrations/0089_remove_dref_field_report_dref_event.py b/dref/migrations/0089_remove_dref_field_report_dref_event.py new file mode 100644 index 000000000..db46fffe2 --- /dev/null +++ b/dref/migrations/0089_remove_dref_field_report_dref_event.py @@ -0,0 +1,30 @@ +# Generated by Django 4.2.30 on 2026-05-25 12:01 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0232_remove_event_auto_generated_source_event_source_and_more"), + ("dref", "0088_remove_identifiedneed_title_ar_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="dref", + name="field_report", + ), + migrations.AddField( + model_name="dref", + name="event", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="event_dref", + to="api.event", + verbose_name="event", + ), + ), + ] diff --git a/dref/migrations/0090_drefsummary.py b/dref/migrations/0090_drefsummary.py new file mode 100644 index 000000000..6d9b171ef --- /dev/null +++ b/dref/migrations/0090_drefsummary.py @@ -0,0 +1,54 @@ +# Generated by Django 5.2.14 on 2026-07-03 00:58 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dref', '0089_remove_dref_field_report_dref_event'), + ] + + operations = [ + migrations.CreateModel( + name='DrefSummary', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('source', models.IntegerField(choices=[(100, 'Dref'), (200, 'Dref Operational Update'), (300, 'Dref Final Report')], help_text='The model this summary was generated from.', verbose_name='source')), + ('source_id', models.PositiveBigIntegerField(verbose_name='source id')), + ('source_hash', models.CharField(max_length=64, unique=True)), + ('situational_overview', models.TextField(blank=True, null=True)), + ('situational_overview_en', models.TextField(blank=True, null=True)), + ('situational_overview_es', models.TextField(blank=True, null=True)), + ('situational_overview_fr', models.TextField(blank=True, null=True)), + ('situational_overview_ar', models.TextField(blank=True, null=True)), + ('operational_strategy', models.TextField(blank=True, null=True)), + ('operational_strategy_en', models.TextField(blank=True, null=True)), + ('operational_strategy_es', models.TextField(blank=True, null=True)), + ('operational_strategy_fr', models.TextField(blank=True, null=True)), + ('operational_strategy_ar', models.TextField(blank=True, null=True)), + ('people_centered_approach', models.TextField(blank=True, null=True)), + ('people_centered_approach_en', models.TextField(blank=True, null=True)), + ('people_centered_approach_es', models.TextField(blank=True, null=True)), + ('people_centered_approach_fr', models.TextField(blank=True, null=True)), + ('people_centered_approach_ar', models.TextField(blank=True, null=True)), + ('challenges_identified', models.TextField(blank=True, null=True)), + ('challenges_identified_en', models.TextField(blank=True, null=True)), + ('challenges_identified_es', models.TextField(blank=True, null=True)), + ('challenges_identified_fr', models.TextField(blank=True, null=True)), + ('challenges_identified_ar', models.TextField(blank=True, null=True)), + ('lessons_learned', models.TextField(blank=True, null=True)), + ('lessons_learned_en', models.TextField(blank=True, null=True)), + ('lessons_learned_es', models.TextField(blank=True, null=True)), + ('lessons_learned_fr', models.TextField(blank=True, null=True)), + ('lessons_learned_ar', models.TextField(blank=True, null=True)), + ('status', models.IntegerField(choices=[(100, 'Pending'), (200, 'Processing'), (300, 'Success'), (400, 'Failed')], default=100)), + ('translation_module_original_language', models.CharField(choices=[('en', 'English'), ('es', 'Spanish'), ('fr', 'French'), ('ar', 'Arabic')], default='en', help_text='Language used to create this entity', max_length=2, verbose_name='Entity Original language')), + ('translation_module_skip_auto_translation', models.BooleanField(default=False, help_text='Skip auto translation operation for this entity?', verbose_name='Skip auto translation')), + ('dref', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='summary', to='dref.dref')), + ], + ), + ] diff --git a/dref/models.py b/dref/models.py index dc8ae4228..84f63b7fd 100644 --- a/dref/models.py +++ b/dref/models.py @@ -1,5 +1,6 @@ import copy import os +from typing import Optional import reversion from django.conf import settings @@ -11,7 +12,7 @@ from django.utils.translation import gettext_lazy as _ from pdf2image import convert_from_bytes -from api.models import Country, DisasterType, District, FieldReport +from api.models import Country, DisasterType, District, Event from deployments.models import Sector from main.fields import SecureFileField @@ -295,13 +296,13 @@ class Status(models.IntegerChoices): related_name="modified_by_dref", ) users = models.ManyToManyField(settings.AUTH_USER_MODEL, verbose_name=_("users"), blank=True, related_name="user_dref") - field_report = models.ForeignKey( - FieldReport, - verbose_name=_("field report"), + event = models.ForeignKey[Event]( + Event, + verbose_name=_("event"), on_delete=models.SET_NULL, null=True, blank=True, - related_name="field_report_dref", + related_name="event_dref", ) title = models.CharField(verbose_name=_("title"), max_length=255) title_prefix = models.CharField(verbose_name=_("title prefix"), max_length=255, null=True, blank=True) @@ -745,6 +746,11 @@ class Status(models.IntegerChoices): blank=True, ) + # TYPING + id: int + pk: int + event_id: Optional[int] + class Meta: verbose_name = _("dref") verbose_name_plural = _("drefs") @@ -1227,6 +1233,10 @@ class DrefOperationalUpdate(models.Model): source_information = models.ManyToManyField(SourceInformation, blank=True, verbose_name=_("Source Information")) __budget_file_id = None + # TYPING + id: int + budget_file_id: int | None + class Meta: verbose_name = _("Dref Operational Update") verbose_name_plural = _("Dref Operational Updates") @@ -1651,6 +1661,9 @@ class DrefFinalReport(models.Model): ) __financial_report_id = None + # TYPING + id: int + class Meta: verbose_name = _("Dref Final Report") verbose_name_plural = _("Dref Final Reports") @@ -1694,3 +1707,70 @@ def get_for(user, status=None): if status == Dref.Status.APPROVED: return queryset.filter(status=Dref.Status.APPROVED) return queryset + + +class DrefSummary(models.Model): + + class SummaryStatus(models.IntegerChoices): + PENDING = 100, _("Pending") + PROCESSING = 200, _("Processing") + SUCCESS = 300, _("Success") + FAILED = 400, _("Failed") + + class SourceModel(models.IntegerChoices): + DREF = 100, _("Dref") + DREF_OPERATIONAL_UPDATE = 200, _("Dref Operational Update") + DREF_FINAL_REPORT = 300, _("Dref Final Report") + + dref = models.OneToOneField( + Dref, + on_delete=models.CASCADE, + related_name="summary", + ) + + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + source = models.IntegerField( + verbose_name=_("source"), + help_text=_("The model this summary was generated from."), + choices=SourceModel.choices, + ) + source_id = models.PositiveBigIntegerField( + verbose_name=_("source id"), + ) + + source_hash = models.CharField( + max_length=64, + unique=True, + ) + + situational_overview = models.TextField( + blank=True, + null=True, + ) + + operational_strategy = models.TextField( + blank=True, + null=True, + ) + + people_centered_approach = models.TextField( + blank=True, + null=True, + ) + + challenges_identified = models.TextField( + blank=True, + null=True, + ) + + lessons_learned = models.TextField( + blank=True, + null=True, + ) + + status = models.IntegerField( + choices=SummaryStatus.choices, + default=SummaryStatus.PENDING, + ) diff --git a/dref/serializers.py b/dref/serializers.py index c8f9e9c9d..a055d710a 100644 --- a/dref/serializers.py +++ b/dref/serializers.py @@ -5,13 +5,14 @@ from django.conf import settings from django.contrib.auth.models import User from django.db import models, transaction +from django.db.models.query import Prefetch from django.utils import timezone from django.utils.translation import get_language, gettext from django.utils.translation import gettext_lazy as _ from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from api.models import Appeal +from api.models import Appeal, Event from api.serializers import ( DisasterTypeSerializer, MiniCountrySerializer, @@ -24,6 +25,7 @@ DrefFile, DrefFinalReport, DrefOperationalUpdate, + DrefSummary, IdentifiedNeed, NationalSocietyAction, PlannedIntervention, @@ -247,6 +249,7 @@ class Meta: "operational_update_details", "final_report_details", "country", + "event", "country_details", "has_ops_update", "has_final_report", @@ -365,6 +368,14 @@ class Meta: ] +class DrefApproveSerializer(serializers.Serializer): + event = serializers.PrimaryKeyRelatedField( + queryset=Event.objects.all(), + required=False, + allow_null=True, + ) + + class DrefSerializer(NestedUpdateMixin, NestedCreateMixin, ModelSerializer): SUB_TOTAL_COST = 75000 SURGE_DEPLOYMENT_COST = 10000 @@ -2240,3 +2251,360 @@ class Meta(BaseDref3Serializer.Meta): class DrefFinalReport3Serializer(BaseDref3Serializer): class Meta(BaseDref3Serializer.Meta): model = DrefFinalReport + + +# Flatten the DREF's scalar contact fields into an api.EventContact-shaped list +# for the emergency page. Integrity contact is intentionally excluded. +DREF_EMERGENCY_CONTACT_GROUPS = ( + ("national_society_contact", "National Society Contact"), + ("ifrc_appeal_manager", "IFRC Appeal Manager"), + ("ifrc_project_manager", "IFRC Project Manager"), + ("ifrc_emergency", "IFRC Emergency"), + ("media_contact", "Media Contact"), +) + + +def get_dref_emergency_contacts(obj): + contacts = [] + for prefix, ctype in DREF_EMERGENCY_CONTACT_GROUPS: + name = getattr(obj, f"{prefix}_name", None) + title = getattr(obj, f"{prefix}_title", None) + email = getattr(obj, f"{prefix}_email", None) + phone = getattr(obj, f"{prefix}_phone_number", None) + if not any([name, title, email, phone]): + continue + contacts.append( + { + "id": f"{obj.id}-{prefix}", + "ctype": ctype, + "name": name or "", + "title": title or "", + "email": email or "", + "phone": phone or "", + } + ) + return contacts + + +class EmergencyDrefContactSerializer(serializers.Serializer): + id = serializers.CharField() + ctype = serializers.CharField() + name = serializers.CharField() + title = serializers.CharField() + email = serializers.CharField() + phone = serializers.CharField() + + +# NOTE: This serializer is only used for the emergency page in GO, +# which has a very specific and limited use case. +# It is not intended to be a general-purpose serializer for DREF objects, +# and as such, it does not include all fields or functionality of the other serializers. +# It is designed to provide the necessary data for the emergency page in a format that is easy to consume and display. +class EmergencyDrefFinalReportSerializer(serializers.ModelSerializer): + status_display = serializers.CharField(source="get_status_display", read_only=True) + disaster_category_display = serializers.CharField(source="get_disaster_category_display", read_only=True) + country_details = MiniCountrySerializer(source="country", read_only=True) + district_details = MiniDistrictSerializer(source="district", read_only=True, many=True) + planned_interventions = PlannedInterventionSerializer(many=True, read_only=True) + cover_image_file = DrefFileSerializer(source="cover_image", read_only=True) + proposed_action = ProposedActionSerializer(many=True, required=False) + disaster_type_details = DisasterTypeSerializer(source="disaster_type", read_only=True) + dref_contacts = serializers.SerializerMethodField() + + class Meta: + model = DrefFinalReport + fields = ( + "id", + "title", + "disaster_type_details", + "cover_image_file", + "status", + "status_display", + "disaster_category", + "disaster_category_display", + "event_scope", + "event_description", + "event_date", + "appeal_code", + "glide_code", + "country_details", + "district_details", + "planned_interventions", + "proposed_action", + # Timeframe of operation + "operation_start_date", + "operation_end_date", + "total_dref_allocation", + "government_requested_assistance", + "num_assisted", + "number_of_people_targeted", + "number_of_people_affected", + "total_targeted_population", + "people_targeted_with_early_actions", + "estimated_number_of_affected_male", + "estimated_number_of_affected_female", + "estimated_number_of_affected_girls_under_18", + "estimated_number_of_affected_boys_under_18", + "assisted_num_of_boys_under_18", + "assisted_num_of_girls_under_18", + "assisted_num_of_men", + "assisted_num_of_women", + "women", + "men", + "boys", + "people_assisted", + "date_of_approval", + "created_at", + "modified_at", + "dref_contacts", + ) + + @extend_schema_field(EmergencyDrefContactSerializer(many=True)) + def get_dref_contacts(self, obj): + return get_dref_emergency_contacts(obj) + + +class TimelineEmergencyDrefOperationalUpdateSerializer(serializers.ModelSerializer): + class Meta: + model = DrefOperationalUpdate + fields = ( + "id", + "summary_of_change", + "date_of_approval", + # fallback timeline date when date_of_approval is unset + "modified_at", + "operational_update_number", + "total_targeted_population", + "total_dref_allocation", + ) + + +class EmergencyDrefOperationalUpdateSerializer(serializers.ModelSerializer): + status_display = serializers.CharField(source="get_status_display", read_only=True) + disaster_category_display = serializers.CharField(source="get_disaster_category_display", read_only=True) + country_details = MiniCountrySerializer(source="country", read_only=True) + district_details = MiniDistrictSerializer(source="district", read_only=True, many=True) + planned_interventions = PlannedInterventionSerializer(many=True, read_only=True) + cover_image_file = DrefFileSerializer(source="cover_image", required=False, allow_null=True) + disaster_type_details = DisasterTypeSerializer(source="disaster_type", read_only=True) + dref_contacts = serializers.SerializerMethodField() + + class Meta: + model = DrefOperationalUpdate + fields = ( + "id", + "title", + "disaster_type_details", + "cover_image_file", + "status", + "status_display", + "disaster_category", + "disaster_category_display", + "event_scope", + "event_description", + "operational_update_number", + "event_date", + "appeal_code", + "glide_code", + "country_details", + "district_details", + "planned_interventions", + # Timeframe of operation + "new_operational_start_date", + "new_operational_end_date", + "total_dref_allocation", + "government_requested_assistance", + "women", + "men", + "boys", + "people_assisted", + "number_of_people_targeted", + "number_of_people_affected", + "total_targeted_population", + "people_targeted_with_early_actions", + "estimated_number_of_affected_male", + "estimated_number_of_affected_female", + "estimated_number_of_affected_girls_under_18", + "estimated_number_of_affected_boys_under_18", + "dref_contacts", + ) + + @extend_schema_field(EmergencyDrefContactSerializer(many=True)) + def get_dref_contacts(self, obj): + return get_dref_emergency_contacts(obj) + + +class DrefSummarySerializer(ModelSerializer): + status_display = serializers.CharField(source="get_status_display", read_only=True) + + class Meta: + model = DrefSummary + fields = ( + "id", + "status", + "status_display", + "situational_overview", + "operational_strategy", + "people_centered_approach", + "challenges_identified", + "lessons_learned", + "created_at", + "updated_at", + ) + + +class EmergencyDrefSerializer(serializers.ModelSerializer): + status_display = serializers.CharField(source="get_status_display", read_only=True) + disaster_category_display = serializers.CharField(source="get_disaster_category_display", read_only=True) + country_details = MiniCountrySerializer(source="country", read_only=True) + district_details = MiniDistrictSerializer(source="district", read_only=True, many=True) + planned_interventions = PlannedInterventionSerializer(many=True, read_only=True) + proposed_action = ProposedActionSerializer(many=True, read_only=True) + cover_image_file = DrefFileSerializer(source="cover_image", required=False, allow_null=True) + disaster_type_details = DisasterTypeSerializer(source="disaster_type", read_only=True) + type_of_dref_display = serializers.CharField(source="get_type_of_dref_display", read_only=True) + summary = DrefSummarySerializer(read_only=True) + + # Dref operational update + operational_update_details = serializers.SerializerMethodField() + # Dref Final report + final_report_details = serializers.SerializerMethodField() + + # Timeline of operational updates + timeline_operational_updates = serializers.SerializerMethodField() + + dref_contacts = serializers.SerializerMethodField() + + class Meta: + model = Dref + fields = ( + "id", + "title", + "type_of_dref", + "type_of_dref_display", + "disaster_category", + "disaster_category_display", + "disaster_type_details", + "cover_image_file", + "event_description", + "event_scope", + "status", + "status_display", + "appeal_code", + "glide_code", + "country_details", + "district_details", + "planned_interventions", + "proposed_action", + "emergency_appeal_planned", + "government_requested_assistance", + "did_ns_request_fund", + # Related drefs + "operational_update_details", + "final_report_details", + # Key Figures + "num_affected", + "num_assisted", + "women", + "men", + "boys", + "hazard_date_and_location", + "amount_requested", + "total_cost", + "total_targeted_population", + "people_targeted_with_early_actions", + "estimated_number_of_affected_male", + "estimated_number_of_affected_female", + "estimated_number_of_affected_girls_under_18", + "estimated_number_of_affected_boys_under_18", + "people_assisted", + # Operational timeframe date + "hazard_date", + "end_date", + "total_cost", + "date_of_approval", + # For Response Type + "event_date", + "timeline_operational_updates", + # Contacts + "dref_contacts", + # Summary + "summary", + ) + + @extend_schema_field(TimelineEmergencyDrefOperationalUpdateSerializer(many=True)) + def get_timeline_operational_updates(self, obj): + ops_updates = getattr(obj, "prefetched_timeline_ops_updates", None) + if ops_updates is None: + ops_updates = DrefOperationalUpdate.objects.filter( + dref=obj, + status=Dref.Status.APPROVED, + ).order_by("operational_update_number") + + serializer = TimelineEmergencyDrefOperationalUpdateSerializer( + ops_updates, + many=True, + ) + return serializer.data + + @extend_schema_field(EmergencyDrefContactSerializer(many=True)) + def get_dref_contacts(self, obj): + return get_dref_emergency_contacts(obj) + + @extend_schema_field(EmergencyDrefOperationalUpdateSerializer()) + def get_operational_update_details(self, obj): + if not obj.operational_update_id: + return None + + instance = ( + DrefOperationalUpdate.objects.select_related( + "country", + "disaster_type", + "cover_image", + "cover_image__created_by", + ) + .prefetch_related( + "district", + Prefetch( + "planned_interventions", + queryset=PlannedIntervention.objects.prefetch_related("indicators"), + ), + ) + .get(pk=obj.operational_update_id) + ) + + return EmergencyDrefOperationalUpdateSerializer( + instance, + context=self.context, + ).data + + @extend_schema_field(EmergencyDrefFinalReportSerializer()) + def get_final_report_details(self, obj): + if not obj.final_report_id: + return None + + instance = ( + DrefFinalReport.objects.select_related( + "country", + "disaster_type", + "cover_image", + "cover_image__created_by", + ) + .prefetch_related( + "district", + Prefetch( + "planned_interventions", + queryset=PlannedIntervention.objects.prefetch_related("indicators"), + ), + Prefetch( + "proposed_action", + queryset=ProposedAction.objects.prefetch_related("activities"), + ), + ) + .get(pk=obj.final_report_id) + ) + + return EmergencyDrefFinalReportSerializer( + instance, + context=self.context, + ).data diff --git a/dref/summary.py b/dref/summary.py new file mode 100644 index 000000000..4fae765ca --- /dev/null +++ b/dref/summary.py @@ -0,0 +1,414 @@ +""" +DREF AI summary generation. +""" + +import hashlib +import json +import logging +from typing import Callable, Dict, List, Optional, Union + +import tiktoken +from django.db.models import F + +from api.utils import get_model_name +from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate, DrefSummary +from main.llm import get_dref_summary_llm_client + +logger = logging.getLogger(__name__) + +ENCODING_NAME = "cl100k_base" + +MAX_OUTPUT_CHARS_PER_FIELD = 1500 +MAX_INPUT_TOKENS = 10000 + + +# The models a DrefSummary can be generated from. +DrefSummarySource = Union[Dref, DrefOperationalUpdate, DrefFinalReport] + +SOURCE_BY_MODEL: Dict[type, DrefSummary.SourceModel] = { + Dref: DrefSummary.SourceModel.DREF, + DrefOperationalUpdate: DrefSummary.SourceModel.DREF_OPERATIONAL_UPDATE, + DrefFinalReport: DrefSummary.SourceModel.DREF_FINAL_REPORT, +} + +# DrefSummary fields — order is the iteration order for prompt assembly. +SUMMARY_FIELDS: List[str] = [ + "situational_overview", + "operational_strategy", + "people_centered_approach", + "challenges_identified", + "lessons_learned", +] + +SYSTEM_MESSAGE = ( + "You are an IFRC expert analyst specializing in DREF (Disaster Response Emergency Fund) " + "operations. Analyze the provided DREF data and produce clear, professional humanitarian " + "summaries suitable for IFRC staff and National Society personnel. Use only the information " + "provided in the data; do not invent facts, figures, or details. Where supporting information " + "for a section is absent, return an empty string for that section rather than speculating." +) + +# Section prompt builders + + +def _build_situational_overview_prompt(**kwargs) -> str: + data = {k: v for k, v in kwargs.items() if v not in (None, "", [], {})} + data_json = json.dumps(data, indent=2, ensure_ascii=False, default=str) + return ( + 'Data for "situational_overview" — disaster context, affected areas, ' + f"scale of impact and strategic rationale for the operation:\n{data_json}" + ) + + +def _build_operational_strategy_prompt(**kwargs) -> str: + data = {k: v for k, v in kwargs.items() if v not in (None, "", [], {})} + data_json = json.dumps(data, indent=2, ensure_ascii=False, default=str) + return ( + 'Data for "operational_strategy" — operation objective, response strategy, ' + f"target population, timeframe and budget:\n{data_json}" + ) + + +def _build_people_centered_approach_prompt(**kwargs) -> str: + data = {k: v for k, v in kwargs.items() if v not in (None, "", [], {})} + data_json = json.dumps(data, indent=2, ensure_ascii=False, default=str) + return ( + 'Data for "people_centered_approach" — targeting criteria, beneficiary selection, ' + f"engagement and protection approach:\n{data_json}" + ) + + +def _build_challenges_identified_prompt(**kwargs) -> str: + data = {k: v for k, v in kwargs.items() if v not in (None, "", [], {})} + data_json = json.dumps(data, indent=2, ensure_ascii=False, default=str) + return ( + 'Data for "challenges_identified" — operational challenges, gaps, coordination ' + f"issues and security/safety risks recorded per planned intervention:\n{data_json}" + ) + + +def _build_lessons_learned_prompt(**kwargs) -> str: + data = {k: v for k, v in kwargs.items() if v not in (None, "", [], {})} + data_json = json.dumps(data, indent=2, ensure_ascii=False, default=str) + return ( + 'Data for "lessons_learned" — lessons learnt recorded per planned intervention ' f"(empty if none recorded):\n{data_json}" + ) + + +# Registry +SECTION_PROMPT_BUILDERS: Dict[str, Callable[..., str]] = { + "situational_overview": _build_situational_overview_prompt, + "operational_strategy": _build_operational_strategy_prompt, + "people_centered_approach": _build_people_centered_approach_prompt, + "challenges_identified": _build_challenges_identified_prompt, + "lessons_learned": _build_lessons_learned_prompt, +} + +GLOBAL_PROMPT = ( + "The DREF data above is organised by summary section. Using ONLY that data, write five concise " + "summary sections. Return a single JSON object (and nothing else) with exactly these keys:\n" + "\n" + ' "situational_overview": The disaster situation, affected areas and scale of impact, plus the ' + 'strategic rationale for the operation. Use the data under the "situational_overview" key. ' + "Write a coherent paragraph; include specific figures where available (e.g. people affected, displaced).\n" + ' "operational_strategy": The overall objective and strategic approach of the response. Use the ' + 'data under the "operational_strategy" key. State the target population, timeframe and headline ' + "interventions where available.\n" + ' "people_centered_approach": Who is targeted and how they are selected, engaged and protected. ' + 'Use the data under the "people_centered_approach" key, including the women/men/girls/boys ' + "breakdown where given.\n" + ' "challenges_identified": The challenges, gaps, coordination issues and security/safety risks. ' + 'Use the data under the "challenges_identified" key.\n' + ' "lessons_learned": The lessons learned from this operation. Use the data under the ' + '"lessons_learned" key. If none are recorded, return an empty string.\n' + "\n" + "Requirements:\n" + "- Each value must be plain text (no markdown, no bullet lists, no nested JSON).\n" + "- Each section should be one well-structured paragraph in professional humanitarian language.\n" + "- Be precise with any facts or numbers; never fabricate them.\n" + "- Return ONLY the JSON object, with no surrounding prose or code fences." +) + + +def count_tokens(text: str) -> int: + try: + return len(tiktoken.get_encoding(ENCODING_NAME).encode(text)) + except Exception as e: + logger.error(f"Error counting tokens for DREF summary: {e}") + return len(text) // 4 + + +def _field_val(obj, field_name): + """Return a model field's value, preferring human-readable display for choice fields.""" + v = getattr(obj, field_name, None) + if v in (None, ""): + return None + display = getattr(obj, f"get_{field_name}_display", None) + return display() if callable(display) else v + + +def _extract_fields(obj, field_names: List[str]) -> dict: + return {name: v for name in field_names if (v := _field_val(obj, name)) is not None} + + +# Shared field lists. + +SITUATIONAL_COMMON_FIELDS: List[str] = ["title", "event_description", "date_of_approval"] + +OPERATIONAL_COMMON_FIELDS: List[str] = [ + "operation_objective", + "response_strategy", + "total_targeted_population", + "people_in_need", + "event_date", +] + +PEOPLE_COMMON_FIELDS: List[str] = ["people_assisted", "selection_criteria"] + +DEMOGRAPHIC_FIELDS: List[str] = ["women", "men", "girls", "boys"] + + +class DrefSummaryGenerator: + """Assembles per-section prompts from a source document and produces all summaries.""" + + def __init__(self): + self.client = get_dref_summary_llm_client() + + # Shared helpers — called by multiple extractors + + @staticmethod + def _situational_overview_kwargs(source_doc) -> dict: + """Build situational_overview kwargs — common across all document types. + + ``event_scope`` is omitted for Imminent DREFs since the scope is not + yet known at that stage. + """ + is_imminent = source_doc.type_of_dref == Dref.DrefType.IMMINENT + kwargs = _extract_fields(source_doc, SITUATIONAL_COMMON_FIELDS) + if source_doc.country: + kwargs["country"] = {"name": source_doc.country.name, "iso": source_doc.country.iso} + if source_doc.disaster_type: + kwargs["disaster_type"] = {"name": source_doc.disaster_type.name} + # event_scope is not yet known for Imminent DREFs, so it is excluded there. + if not is_imminent: + kwargs.update(_extract_fields(source_doc, ["event_scope"])) + return kwargs + + @staticmethod + def _challenges_and_lessons_kwargs(source_doc) -> Dict[str, dict]: + """Collect challenges and lessons from ``planned_interventions`` M2M. + + Only called for ``DrefFinalReport`` + """ + + def pi_title(pi): + display = getattr(pi, "get_title_display", None) + return display() if callable(display) else pi.title + + # Order explicitly: planned_interventions has no Meta.ordering, so an + # unordered .all() can return rows in different orders across queries, + # which would change the source hash and trigger needless regeneration. + planned = list(source_doc.planned_interventions.order_by("id")) + return { + "challenges_identified": { + "planned_interventions": [{"title": pi_title(pi), "challenges": pi.challenges} for pi in planned if pi.challenges] + or None, + }, + "lessons_learned": { + "planned_interventions": [ + {"title": pi_title(pi), "lessons_learnt": pi.lessons_learnt} for pi in planned if pi.lessons_learnt + ] + or None, + }, + } + + @classmethod + def _extract_dref_kwargs(cls, dref) -> Dict[str, dict]: + """Dref Application / Assessment / Imminent — three sections only. + + Challenges and lessons are not applicable at the application stage; + they are formally recorded only in the Final Report. + """ + is_imminent = dref.type_of_dref == Dref.DrefType.IMMINENT + operational = _extract_fields(dref, OPERATIONAL_COMMON_FIELDS + ["amount_requested", "end_date"]) + # Normalise to a single "operation_timeframe" key regardless of source field. + timeframe = _field_val(dref, "operation_timeframe_imminent" if is_imminent else "operation_timeframe") + if timeframe is not None: + operational["operation_timeframe"] = timeframe + return { + "situational_overview": cls._situational_overview_kwargs(dref), + "operational_strategy": operational, + "people_centered_approach": _extract_fields(dref, PEOPLE_COMMON_FIELDS), + } + + @classmethod + def _extract_dref_ops_kwargs(cls, ops) -> Dict[str, dict]: + """DrefOperationalUpdate — three sections; different field names from Dref. + + Includes demographic breakdown (women/men/girls/boys) in people_centered_approach. + Challenges and lessons are not generated for Operational Updates. + """ + operational = _extract_fields(ops, OPERATIONAL_COMMON_FIELDS + ["total_dref_allocation"]) + # Ops Update names these differently; normalise to the shared key names. + operational.update( + { + k: v + for k, v in { + "operation_end_date": _field_val(ops, "new_operational_end_date"), + "operation_timeframe": _field_val(ops, "total_operation_timeframe"), + }.items() + if v is not None + } + ) + return { + "situational_overview": cls._situational_overview_kwargs(ops), + "operational_strategy": operational, + "people_centered_approach": _extract_fields(ops, PEOPLE_COMMON_FIELDS + DEMOGRAPHIC_FIELDS), + } + + @classmethod + def _extract_dref_final_kwargs(cls, final) -> Dict[str, dict]: + """DrefFinalReport — all five sections. + + Challenges and lessons come from ``planned_interventions`` M2M via + ``_challenges_and_lessons_kwargs``; this is the only document type + where those two sections are generated. + """ + is_imminent = final.type_of_dref == Dref.DrefType.IMMINENT + operational = _extract_fields(final, OPERATIONAL_COMMON_FIELDS + ["total_dref_allocation", "operation_end_date"]) + # Normalise to a single "operation_timeframe" key regardless of source field. + timeframe = _field_val(final, "total_operation_timeframe_imminent" if is_imminent else "total_operation_timeframe") + if timeframe is not None: + operational["operation_timeframe"] = timeframe + kwargs = { + "situational_overview": cls._situational_overview_kwargs(final), + "operational_strategy": operational, + "people_centered_approach": _extract_fields(final, PEOPLE_COMMON_FIELDS + DEMOGRAPHIC_FIELDS), + } + kwargs.update(cls._challenges_and_lessons_kwargs(final)) + return kwargs + + @classmethod + def get_section_kwargs(cls, source_doc: DrefSummarySource) -> Dict[str, dict]: + """Dispatch to the model-specific extractor. + + Public so callers that need both the hash and the generated summary + for the same ``source_doc`` (see ``compute_source_hash``/``generate_all``) + can compute this once and pass it to both instead of extracting twice. + """ + if isinstance(source_doc, Dref): + return cls._extract_dref_kwargs(source_doc) + if isinstance(source_doc, DrefOperationalUpdate): + return cls._extract_dref_ops_kwargs(source_doc) + if isinstance(source_doc, DrefFinalReport): + return cls._extract_dref_final_kwargs(source_doc) + return {} + + @staticmethod + def get_latest_approved_source(dref: Dref) -> Optional[tuple[DrefSummary.SourceModel, DrefSummarySource]]: + """Most authoritative (source, source object) pair for ``dref``. + + Priority: Final Report > latest Operational Update > Dref itself. + """ + final_report = ( + DrefFinalReport.objects.select_related("country", "disaster_type") + .filter(dref=dref, status=Dref.Status.APPROVED) + .order_by("-created_at") + .first() + ) + if final_report: + return SOURCE_BY_MODEL[DrefFinalReport], final_report + + latest_ops_update = ( + DrefOperationalUpdate.objects.select_related("country", "disaster_type") + .filter(dref=dref, status=Dref.Status.APPROVED) + .order_by(F("operational_update_number").desc(nulls_last=True), "-created_at") + .first() + ) + if latest_ops_update: + return SOURCE_BY_MODEL[DrefOperationalUpdate], latest_ops_update + + if dref.status == Dref.Status.APPROVED: + return SOURCE_BY_MODEL[Dref], dref + + return None + + @classmethod + def build_source_text(cls, source_doc: DrefSummarySource) -> str: + """JSON representation of all section kwargs for inspection and debugging.""" + return json.dumps(cls.get_section_kwargs(source_doc), indent=2, ensure_ascii=False, default=str) + + @classmethod + def compute_source_hash(cls, source_doc: DrefSummarySource, section_kwargs: Optional[Dict[str, dict]] = None) -> str: + """Hash of all source content feeding the summary, for change detection.""" + model_label = get_model_name(type(source_doc)) + if section_kwargs is None: + section_kwargs = cls.get_section_kwargs(source_doc) + payload = {"model": model_label, "id": source_doc.id, "source": section_kwargs} + content = json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str) + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + @staticmethod + def _parse_response(response: str) -> Dict[str, str]: + """Parse the model's JSON object, tolerating ```json code fences.""" + text = response.strip() + if text.startswith("```"): + text = text.split("\n", 1)[1] if "\n" in text else text + text = text.rsplit("```", 1)[0] + data = json.loads(text) + if not isinstance(data, dict): + raise ValueError("Expected a JSON object of summary fields") + return data + + def generate_all(self, source_doc: DrefSummarySource, section_kwargs: Optional[Dict[str, dict]] = None) -> Dict[str, str]: + """Generate every summary for ``source_doc`` in a single LLM call. + + Always returns a value for every ``SUMMARY_FIELDS`` key (empty string + when a section has no content), so callers overwrite stale values + instead of leaving them in place on regeneration. ``section_kwargs`` + can be passed in by a caller that already computed it (e.g. via + ``compute_source_hash``) to avoid re-extracting it from ``source_doc``. + """ + empty_results: Dict[str, str] = {field_name: "" for field_name in SUMMARY_FIELDS} + + if section_kwargs is None: + section_kwargs = self.get_section_kwargs(source_doc) + if not section_kwargs: + logger.info(f"No source content for ({type(source_doc).__name__}) ({source_doc.id}) summary; skipping generation.") + return empty_results + + # Call each builder with only its section's kwargs, then join blocks. + section_blocks = [SECTION_PROMPT_BUILDERS[section](**section_kwargs.get(section, {})) for section in SUMMARY_FIELDS] + user_content = "\n\n".join(section_blocks) + "\n\n" + GLOBAL_PROMPT + + # Respect the token budget: trim the data blocks if the prompt is too long. + if count_tokens(SYSTEM_MESSAGE + user_content) > MAX_INPUT_TOKENS: + logger.warning(f"({type(source_doc).__name__}) ({source_doc.id}) summary prompt too long; truncating source text.") + char_budget = MAX_INPUT_TOKENS * 4 - len(GLOBAL_PROMPT) - len(SYSTEM_MESSAGE) + data_blocks = "\n\n".join(section_blocks) + user_content = data_blocks[: max(char_budget, 0)] + "\n\n" + GLOBAL_PROMPT + + messages = [ + {"role": "system", "content": SYSTEM_MESSAGE}, + {"role": "user", "content": user_content}, + ] + + response = self.client.get_response(messages) + if not response: + raise RuntimeError(f"No LLM response for ({type(source_doc).__name__}) ({source_doc.id}) summary") + + try: + parsed = self._parse_response(response) + except (json.JSONDecodeError, ValueError) as e: + raise RuntimeError(f"Could not parse ({type(source_doc).__name__}) ({source_doc.id}) summary response: {e}") from e + + results = dict(empty_results) + for field_name in SUMMARY_FIELDS: + value = parsed.get(field_name) + if not isinstance(value, str): + continue + summary = value.strip() + if len(summary) > MAX_OUTPUT_CHARS_PER_FIELD: + summary = summary[:MAX_OUTPUT_CHARS_PER_FIELD].rstrip() + results[field_name] = summary + return results diff --git a/dref/tasks.py b/dref/tasks.py index 0b0d08c96..fde4ef745 100644 --- a/dref/tasks.py +++ b/dref/tasks.py @@ -1,17 +1,25 @@ import logging +from datetime import timedelta +from enum import Enum +from typing import Optional from celery import shared_task from django.apps import apps +from django.db import transaction from django.template.loader import render_to_string +from django.utils import timezone from api.utils import get_model_name from lang.tasks import translate_model_fields +from main.lock import RedisLockKey, redis_lock from main.translation import TRANSLATOR_ORIGINAL_LANGUAGE_FIELD_NAME +from main.utils import logger_context from notifications.notification import send_notification from .models import ( Dref, DrefFile, + DrefSummary, IdentifiedNeed, NationalSocietyAction, PlannedIntervention, @@ -21,10 +29,27 @@ RiskSecurity, SourceInformation, ) +from .summary import DrefSummaryGenerator from .utils import get_email_context logger = logging.getLogger(__name__) +# The PROCESSING status has no TTL of its own (see comment in generate_dref_summary +# below), so a row can get stuck PROCESSING forever if a worker dies mid-generation. +# Treat it as stale past this, so a later trigger can take over. +PROCESSING_STALE_AFTER = timedelta(minutes=10) + + +class DrefSummaryGenerationResult(str, Enum): + """Outcome of a ``generate_dref_summary`` run.""" + + SUCCESS = "success" + SOURCE_NOT_FOUND = "source_not_found" + ALREADY_IN_PROGRESS = "already_in_progress" + UP_TO_DATE = "up_to_date" + FAILED = "failed" + SUPERSEDED = "superseded" + @shared_task def send_dref_email(dref_id, users_emails, new_or_updated=""): @@ -57,6 +82,112 @@ def send_dref_email(dref_id, users_emails, new_or_updated=""): ] +@shared_task(soft_time_limit=600, time_limit=630) +def generate_dref_summary(dref_id: int, overwrite: bool = False) -> DrefSummaryGenerationResult: + """Generate and store the AI-assisted summaries for a DREF. + + Always (re)generates from whichever approved source is currently latest + for the DREF (see ``get_latest_approved_source``), rather than a + specific source passed in, so it self-corrects no matter which + approval triggered it or the order concurrent runs execute in. + """ + dref = Dref.objects.select_related("country", "disaster_type").filter(id=dref_id).first() + if not dref: + logger.error("Dref not found for summary generation", extra=logger_context({"dref_id": dref_id})) + return DrefSummaryGenerationResult.SOURCE_NOT_FOUND + + # The Redis lock only needs to guard the brief read-check-mark-PROCESSING + # section below against a concurrent trigger for the same DREF (double- + # approve, admin retrigger, ...); it is released before the LLM call. + # The PROCESSING status written inside the lock is what actually blocks + # a second run for as long as generation takes - unlike the lock, it has + # no TTL, so it still holds even if generation outlives lock_expire. + with redis_lock(key=RedisLockKey.DREF_SUMMARY, id=dref_id) as acquired: + if not acquired: + logger.warning(f"DREF summary generation already in progress for DREF ({dref_id}); skipping.") + return DrefSummaryGenerationResult.ALREADY_IN_PROGRESS + + latest_source = DrefSummaryGenerator.get_latest_approved_source(dref) + if not latest_source: + logger.error(f"No approved source found for DREF ({dref_id}) summary") + return DrefSummaryGenerationResult.SOURCE_NOT_FOUND + source_type, source_obj = latest_source + + section_kwargs = DrefSummaryGenerator.get_section_kwargs(source_obj) + source_hash = DrefSummaryGenerator.compute_source_hash(source_obj, section_kwargs=section_kwargs) + summary_instance: Optional[DrefSummary] = DrefSummary.objects.filter(dref=dref).first() + + if ( + summary_instance + and summary_instance.status == DrefSummary.SummaryStatus.PROCESSING + and timezone.now() - summary_instance.updated_at < PROCESSING_STALE_AFTER + and summary_instance.source == source_type + and summary_instance.source_id == source_obj.id + ): + # Same source already in flight; a different one falls through to regenerate below. + logger.warning(f"DREF summary already in progress for DREF ({dref_id}); skipping.") + return DrefSummaryGenerationResult.ALREADY_IN_PROGRESS + + if ( + summary_instance + and not overwrite + and summary_instance.source_hash == source_hash + and summary_instance.status == DrefSummary.SummaryStatus.SUCCESS + ): + logger.info(f"DREF summary up to date for DREF ({dref_id}); skipping generation.") + return DrefSummaryGenerationResult.UP_TO_DATE + + if summary_instance is None: + summary_instance = DrefSummary(dref=dref) + + try: + summary_instance.source_hash = source_hash + summary_instance.source = source_type + summary_instance.source_id = source_obj.id + summary_instance.status = DrefSummary.SummaryStatus.PROCESSING + summary_instance.save() + except Exception: + logger.warning(f"Failed to mark DREF summary as processing for DREF ({dref_id})", exc_info=True) + if summary_instance.pk: + summary_instance.status = DrefSummary.SummaryStatus.FAILED + summary_instance.save() + return DrefSummaryGenerationResult.FAILED + + # Lock released. The (possibly slow) LLM call runs unlocked; the + # PROCESSING status set above is what a concurrent trigger checks. + own_marker = { + "pk": summary_instance.pk, + "source": source_type, + "source_id": source_obj.id, + "status": DrefSummary.SummaryStatus.PROCESSING, + } + try: + logger.info(f"Generating DREF summaries for DREF ({dref_id}) from ({source_type.label}) ({source_obj.id})") + results = DrefSummaryGenerator().generate_all(source_obj, section_kwargs=section_kwargs) + with transaction.atomic(): + latest_summary_instance = DrefSummary.objects.select_for_update().filter(**own_marker).first() + if latest_summary_instance is None: + logger.warning(f"DREF summary run for DREF ({dref_id}) was superseded by a newer trigger; discarding result.") + return DrefSummaryGenerationResult.SUPERSEDED + for field_name, value in results.items(): + setattr(latest_summary_instance, field_name, value) + latest_summary_instance.status = DrefSummary.SummaryStatus.SUCCESS + latest_summary_instance.save() + logger.info(f"Successfully generated DREF summaries for DREF ({dref_id})") + transaction.on_commit(lambda: translate_model_fields.delay(get_model_name(DrefSummary), latest_summary_instance.pk)) + return DrefSummaryGenerationResult.SUCCESS + except Exception: + with transaction.atomic(): + latest_summary_instance = DrefSummary.objects.select_for_update().filter(**own_marker).first() + if latest_summary_instance is None: + logger.warning(f"DREF summary run for DREF ({dref_id}) failed but was already superseded; leaving it as is.") + else: + latest_summary_instance.status = DrefSummary.SummaryStatus.FAILED + latest_summary_instance.save() + logger.warning(f"DREF summary generation failed for DREF ({dref_id})", exc_info=True) + return DrefSummaryGenerationResult.FAILED + + @shared_task def process_dref_translation(model_name, instance_pk): """ diff --git a/dref/test_summary.py b/dref/test_summary.py new file mode 100644 index 000000000..f291d47bc --- /dev/null +++ b/dref/test_summary.py @@ -0,0 +1,327 @@ +from datetime import date + +from django.test import TestCase, override_settings + +from dref.factories.dref import ( + DrefFactory, + DrefFinalReportFactory, + DrefOperationalUpdateFactory, + PlannedInterventionFactory, +) +from dref.models import Dref, DrefSummary +from dref.summary import SUMMARY_FIELDS, DrefSummaryGenerator +from dref.tasks import DrefSummaryGenerationResult, generate_dref_summary +from main.llm import ( + AzureOpenAiChat, + DummyDrefSummaryLLMClient, + get_dref_summary_llm_client, +) + + +class DrefSummaryLLMClientTest(TestCase): + def test_uses_dummy_client_during_tests_by_default(self): + # settings.TESTING is always true under the test runner, so the + # factory should return the dummy client even though + # USE_DUMMY_LLM_CLIENT itself defaults to False. + self.assertIsInstance(get_dref_summary_llm_client(), DummyDrefSummaryLLMClient) + + @override_settings( + TESTING=False, + USE_DUMMY_LLM_CLIENT=False, + AZURE_OPENAI_ENDPOINT="https://example.com", + AZURE_OPENAI_API_KEY="fake-key", + AZURE_OPENAI_DEPLOYMENT_NAME="fake-deployment", + ) + def test_uses_real_client_when_flag_off_and_not_testing(self): + self.assertIsInstance(get_dref_summary_llm_client(), AzureOpenAiChat) + + @override_settings(TESTING=False, USE_DUMMY_LLM_CLIENT=True) + def test_uses_dummy_client_when_flag_on(self): + self.assertIsInstance(get_dref_summary_llm_client(), DummyDrefSummaryLLMClient) + + +class DrefSummaryGeneratorTest(TestCase): + def test_generate_all_succeeds_end_to_end_for_dref(self): + dref = DrefFactory.create(title="Test Dref") + + results = DrefSummaryGenerator().generate_all(dref) + + self.assertEqual(set(results.keys()), set(SUMMARY_FIELDS)) + self.assertIn("DUMMY RESPONSE", results["situational_overview"]) + + def test_generate_all_succeeds_end_to_end_for_dref_operational_update(self): + ops_update = DrefOperationalUpdateFactory.create(dref=DrefFactory.create(), title="Test Ops Update") + + results = DrefSummaryGenerator().generate_all(ops_update) + + self.assertEqual(set(results.keys()), set(SUMMARY_FIELDS)) + self.assertIn("DUMMY RESPONSE", results["situational_overview"]) + + def test_generate_all_succeeds_end_to_end_for_dref_final_report(self): + planned_intervention = PlannedInterventionFactory.create(challenges="Some challenge", lessons_learnt="Some lesson") + final_report = DrefFinalReportFactory.create(title="Test Final Report", planned_interventions=[planned_intervention]) + + results = DrefSummaryGenerator().generate_all(final_report) + + self.assertEqual(set(results.keys()), set(SUMMARY_FIELDS)) + self.assertIn("DUMMY RESPONSE", results["situational_overview"]) + self.assertIn("DUMMY RESPONSE", results["challenges_identified"]) + self.assertIn("DUMMY RESPONSE", results["lessons_learned"]) + + def test_get_section_kwargs_for_dref(self): + dref = DrefFactory.create( + title="Test Dref", + event_description="Severe flooding", + type_of_dref=Dref.DrefType.RESPONSE, + event_scope="Affected 3 districts", + operation_objective="Provide shelter", + response_strategy="Cash and shelter support", + total_targeted_population=5000, + people_in_need=8000, + people_assisted="5000 people", + selection_criteria="Most vulnerable households", + ) + + kwargs = DrefSummaryGenerator.get_section_kwargs(dref) + + self.assertEqual(set(kwargs.keys()), {"situational_overview", "operational_strategy", "people_centered_approach"}) + self.assertEqual(kwargs["situational_overview"]["title"], "Test Dref") + self.assertEqual(kwargs["situational_overview"]["event_description"], "Severe flooding") + self.assertEqual(kwargs["situational_overview"]["event_scope"], "Affected 3 districts") + self.assertEqual(kwargs["operational_strategy"]["operation_objective"], "Provide shelter") + self.assertEqual(kwargs["operational_strategy"]["response_strategy"], "Cash and shelter support") + self.assertEqual(kwargs["operational_strategy"]["total_targeted_population"], 5000) + self.assertEqual(kwargs["operational_strategy"]["people_in_need"], 8000) + self.assertEqual(kwargs["people_centered_approach"]["people_assisted"], "5000 people") + self.assertEqual(kwargs["people_centered_approach"]["selection_criteria"], "Most vulnerable households") + self.assertNotIn("women", kwargs["people_centered_approach"]) + + def test_get_section_kwargs_for_dref_operational_update(self): + ops_update = DrefOperationalUpdateFactory.create( + dref=DrefFactory.create(), + title="Test Ops Update", + event_description="Flooding continues", + operation_objective="Extend shelter support", + response_strategy="Extended cash support", + total_dref_allocation=100000, + people_assisted="6000 people", + selection_criteria="Vulnerable households", + women=1000, + men=900, + girls=500, + boys=600, + new_operational_end_date=date(2025, 6, 1), + total_operation_timeframe=6, + ) + + kwargs = DrefSummaryGenerator.get_section_kwargs(ops_update) + + self.assertEqual(set(kwargs.keys()), {"situational_overview", "operational_strategy", "people_centered_approach"}) + self.assertEqual(kwargs["situational_overview"]["title"], "Test Ops Update") + self.assertEqual(kwargs["situational_overview"]["event_description"], "Flooding continues") + self.assertEqual(kwargs["operational_strategy"]["operation_objective"], "Extend shelter support") + self.assertEqual(kwargs["operational_strategy"]["total_dref_allocation"], 100000) + self.assertEqual(kwargs["operational_strategy"]["operation_end_date"], date(2025, 6, 1)) + self.assertEqual(kwargs["operational_strategy"]["operation_timeframe"], 6) + self.assertEqual(kwargs["people_centered_approach"]["people_assisted"], "6000 people") + self.assertEqual(kwargs["people_centered_approach"]["women"], 1000) + self.assertEqual(kwargs["people_centered_approach"]["men"], 900) + self.assertEqual(kwargs["people_centered_approach"]["girls"], 500) + self.assertEqual(kwargs["people_centered_approach"]["boys"], 600) + + def test_get_section_kwargs_for_dref_final_report(self): + planned_intervention = PlannedInterventionFactory.create( + challenges="Access constraints due to weather", + lessons_learnt="Earlier pre-positioning of stock helps", + ) + final_report = DrefFinalReportFactory.create( + title="Test Final Report", + event_description="Response concluded", + operation_objective="Deliver relief items", + response_strategy="Direct distribution", + total_dref_allocation=150000, + people_assisted="7000 people", + selection_criteria="Most affected households", + women=1200, + men=1100, + girls=600, + boys=700, + planned_interventions=[planned_intervention], + ) + + kwargs = DrefSummaryGenerator.get_section_kwargs(final_report) + + self.assertEqual( + set(kwargs.keys()), + { + "situational_overview", + "operational_strategy", + "people_centered_approach", + "challenges_identified", + "lessons_learned", + }, + ) + self.assertEqual(kwargs["situational_overview"]["title"], "Test Final Report") + self.assertEqual(kwargs["operational_strategy"]["total_dref_allocation"], 150000) + self.assertEqual(kwargs["people_centered_approach"]["women"], 1200) + + challenges = kwargs["challenges_identified"]["planned_interventions"] + self.assertEqual(len(challenges), 1) + self.assertEqual(challenges[0]["challenges"], "Access constraints due to weather") + + lessons = kwargs["lessons_learned"]["planned_interventions"] + self.assertEqual(len(lessons), 1) + self.assertEqual(lessons[0]["lessons_learnt"], "Earlier pre-positioning of stock helps") + + def test_get_section_kwargs_excludes_event_scope_for_imminent_dref(self): + imminent_dref = DrefFactory.create(type_of_dref=Dref.DrefType.IMMINENT, event_scope="scope text") + self.assertNotIn("event_scope", DrefSummaryGenerator.get_section_kwargs(imminent_dref)["situational_overview"]) + + response_dref = DrefFactory.create(type_of_dref=Dref.DrefType.RESPONSE, event_scope="scope text") + self.assertIn("event_scope", DrefSummaryGenerator.get_section_kwargs(response_dref)["situational_overview"]) + + def test_compute_source_hash_changes_with_content_and_is_deterministic(self): + dref = DrefFactory.create(title="Original Title", type_of_dref=Dref.DrefType.RESPONSE) + + hash_a = DrefSummaryGenerator.compute_source_hash(dref) + hash_b = DrefSummaryGenerator.compute_source_hash(dref) + self.assertEqual(hash_a, hash_b) + + dref.title = "Changed Title" + dref.save(update_fields=["title"]) + hash_c = DrefSummaryGenerator.compute_source_hash(dref) + self.assertNotEqual(hash_a, hash_c) + + def test_compute_source_hash_uses_provided_section_kwargs(self): + dref = DrefFactory.create(title="Test Dref", type_of_dref=Dref.DrefType.RESPONSE) + default_hash = DrefSummaryGenerator.compute_source_hash(dref) + + actual_kwargs = DrefSummaryGenerator.get_section_kwargs(dref) + self.assertEqual(default_hash, DrefSummaryGenerator.compute_source_hash(dref, section_kwargs=actual_kwargs)) + + custom_kwargs = {"situational_overview": {"title": "overridden"}} + self.assertNotEqual(default_hash, DrefSummaryGenerator.compute_source_hash(dref, section_kwargs=custom_kwargs)) + + def test_get_latest_approved_source_returns_none_when_dref_not_approved(self): + dref = DrefFactory.create(status=Dref.Status.FINALIZED) + self.assertIsNone(DrefSummaryGenerator.get_latest_approved_source(dref)) + + def test_get_latest_approved_source_follows_priority_order(self): + dref = DrefFactory.create(status=Dref.Status.APPROVED) + self.assertEqual(DrefSummaryGenerator.get_latest_approved_source(dref), (DrefSummary.SourceModel.DREF, dref)) + + # Unapproved updates don't count, so the Dref itself still wins. + DrefOperationalUpdateFactory.create(dref=dref, status=Dref.Status.FINALIZED, operational_update_number=1) + self.assertEqual(DrefSummaryGenerator.get_latest_approved_source(dref), (DrefSummary.SourceModel.DREF, dref)) + + # An approved Operational Update supersedes the Dref; the higher + # operational_update_number wins, regardless of insertion order. + DrefOperationalUpdateFactory.create(dref=dref, status=Dref.Status.APPROVED, operational_update_number=1) + ops_update_2 = DrefOperationalUpdateFactory.create(dref=dref, status=Dref.Status.APPROVED, operational_update_number=2) + self.assertEqual( + DrefSummaryGenerator.get_latest_approved_source(dref), + (DrefSummary.SourceModel.DREF_OPERATIONAL_UPDATE, ops_update_2), + ) + + # An approved Final Report supersedes everything else. + final_report = DrefFinalReportFactory.create(dref=dref, status=Dref.Status.APPROVED) + self.assertEqual( + DrefSummaryGenerator.get_latest_approved_source(dref), + (DrefSummary.SourceModel.DREF_FINAL_REPORT, final_report), + ) + + def test_returns_source_not_found_when_dref_missing_or_nothing_approved(self): + self.assertEqual(generate_dref_summary(dref_id=999999), DrefSummaryGenerationResult.SOURCE_NOT_FOUND) + + dref = DrefFactory.create(status=Dref.Status.FINALIZED) + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.SOURCE_NOT_FOUND) + + def test_generates_from_dref_and_is_up_to_date_on_rerun(self): + dref = DrefFactory.create(status=Dref.Status.APPROVED) + + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.SUCCESS) + summary = DrefSummary.objects.get(dref=dref) + self.assertEqual(summary.source, DrefSummary.SourceModel.DREF) + self.assertEqual(summary.source_id, dref.id) + + # Nothing changed since - re-running should not regenerate. + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.UP_TO_DATE) + + def test_retrigger_with_overwrite_regenerates_even_when_up_to_date(self): + dref = DrefFactory.create(status=Dref.Status.APPROVED) + + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.SUCCESS) + summary = DrefSummary.objects.get(dref=dref) + first_hash = summary.source_hash + first_updated_at = summary.updated_at + + # Nothing changed since, so a plain retrigger would be UP_TO_DATE ... + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.UP_TO_DATE) + + # ... but overwrite=True must bypass that check and regenerate anyway. + self.assertEqual(generate_dref_summary(dref_id=dref.id, overwrite=True), DrefSummaryGenerationResult.SUCCESS) + + self.assertEqual(DrefSummary.objects.filter(dref=dref).count(), 1) + summary.refresh_from_db() + self.assertEqual(summary.status, DrefSummary.SummaryStatus.SUCCESS) + self.assertEqual(summary.source, DrefSummary.SourceModel.DREF) + self.assertEqual(summary.source_id, dref.id) + self.assertEqual(summary.source_hash, first_hash) + self.assertGreater(summary.updated_at, first_updated_at) + + def test_processing_in_flight_skips_same_source_but_regenerates_for_a_newer_one(self): + dref = DrefFactory.create(status=Dref.Status.APPROVED) + # Simulate a run already in flight for the Dref itself. + DrefSummary.objects.create( + dref=dref, + source=DrefSummary.SourceModel.DREF, + source_id=dref.id, + source_hash="in-flight-hash", + status=DrefSummary.SummaryStatus.PROCESSING, + ) + + # A duplicate trigger for the same in-flight source is dropped. + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.ALREADY_IN_PROGRESS) + + # A newer approval lands while that run is still (supposedly) processing. + ops_update = DrefOperationalUpdateFactory.create(dref=dref, status=Dref.Status.APPROVED, operational_update_number=1) + + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.SUCCESS) + summary = DrefSummary.objects.get(dref=dref) + self.assertEqual(summary.source, DrefSummary.SourceModel.DREF_OPERATIONAL_UPDATE) + self.assertEqual(summary.source_id, ops_update.id) + + def test_generate_all_succeeds_for_newer_ops_update_while_older_one_in_flight(self): + """Two Operational Updates on one Dref: a run in flight for the older + one must not block generation once a newer one is approved.""" + dref = DrefFactory.create(status=Dref.Status.APPROVED) + ops_update_1 = DrefOperationalUpdateFactory.create(dref=dref, status=Dref.Status.APPROVED, operational_update_number=1) + DrefSummary.objects.create( + dref=dref, + source=DrefSummary.SourceModel.DREF_OPERATIONAL_UPDATE, + source_id=ops_update_1.id, + source_hash="in-flight-hash", + status=DrefSummary.SummaryStatus.PROCESSING, + ) + + # A duplicate trigger for the same in-flight source (ops_update_1) is dropped. + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.ALREADY_IN_PROGRESS) + + # A second, newer Operational Update is approved while that run is still (supposedly) processing. + ops_update_2 = DrefOperationalUpdateFactory.create(dref=dref, status=Dref.Status.APPROVED, operational_update_number=2) + + # The in-flight record is for a now-stale source (ops_update_1), so this + # trigger falls through and regenerates from ops_update_2 instead of + # being blocked as "already in progress". + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.SUCCESS) + + # Exactly one DrefSummary row for the Dref - updated in place, not duplicated. + self.assertEqual(DrefSummary.objects.filter(dref=dref).count(), 1) + summary = DrefSummary.objects.get(dref=dref) + self.assertEqual(summary.source, DrefSummary.SourceModel.DREF_OPERATIONAL_UPDATE) + self.assertEqual(summary.source_id, ops_update_2.id) + self.assertEqual(summary.status, DrefSummary.SummaryStatus.SUCCESS) + self.assertNotEqual(summary.source_hash, "in-flight-hash") + + # Nothing changed since - re-running should not regenerate. + self.assertEqual(generate_dref_summary(dref_id=dref.id), DrefSummaryGenerationResult.UP_TO_DATE) diff --git a/dref/test_views.py b/dref/test_views.py index 3c4a221d0..a0071523d 100644 --- a/dref/test_views.py +++ b/dref/test_views.py @@ -10,7 +10,7 @@ from django.core import management from rest_framework import status -from api.models import Country, DisasterType, District, Region, RegionName +from api.models import Country, DisasterType, District, Event, Region, RegionName from api.utils import get_model_name from deployments.factories.project import SectorFactory from deployments.factories.user import UserFactory @@ -30,6 +30,7 @@ ProposedAction, ) from dref.tasks import send_dref_email +from lang.serializers import TranslatedModelSerializerMixin from main.test_case import APITestCase @@ -2664,6 +2665,79 @@ def test_dref_final_report_finalize(self, mock_translation): self.assert_200(response) self.assertEqual(response.data["status"], Dref.Status.FINALIZED) + @patch.object( + TranslatedModelSerializerMixin, + "trigger_field_translation", + ) + def test_create_event_from_dref(self, mock_trigger_translation): + region = Region.objects.create(name=RegionName.ASIA_PACIFIC) + country = Country.objects.create(name="Test countrynpl", region=region) + district = District.objects.create(name="test district", country=country) + disaster_type = DisasterType.objects.create(name="test disaster") + dref = DrefFactory.create( + title="Test Title", + disaster_type=disaster_type, + event_description="Test event description", + event_date="2021-10-10", + glide_code="GLIDE123", + created_by=self.user, + country=country, + status=Dref.Status.FINALIZED, + type_of_dref=Dref.DrefType.ASSESSMENT, + ) + dref.district.set([district]) + url = f"/api/v2/dref/{dref.id}/approve/" + self.authenticate(self.root_user) + response = self.client.post(url) + self.assert_200(response) + + dref.refresh_from_db() + + dref_event_id = response.data["event"] + event_instance = Event.objects.get(id=dref_event_id) + + # Translation triggered + mock_trigger_translation.assert_called_once() + + translated_event = mock_trigger_translation.call_args.args[0] + + self.assertEqual( + translated_event.pk, + event_instance.pk, + ) + + self.assertEqual( + { + event_instance.name, + event_instance.dtype.id, + event_instance.summary, + event_instance.disaster_start_date.date(), + event_instance.glide, + event_instance.source, + }, + { + dref.title, + dref.disaster_type.id, + dref.event_description, + dref.event_date, + dref.glide_code, + Event.EventSource.DREF, + }, + ) + self.assertEqual( + list(event_instance.regions.values_list("id", flat=True)), + [dref.country.region.id], + ) + self.assertEqual( + list(event_instance.countries.values_list("id", flat=True)), + [dref.country.id], + ) + self.assertEqual( + list(event_instance.districts.values_list("id", flat=True)), + [district.id], + ) + self.assertTrue(event_instance.auto_generated) + User = get_user_model() diff --git a/dref/translation.py b/dref/translation.py index 1c3b29d50..58a7fc986 100644 --- a/dref/translation.py +++ b/dref/translation.py @@ -5,6 +5,7 @@ DrefFile, DrefFinalReport, DrefOperationalUpdate, + DrefSummary, IdentifiedNeed, NationalSocietyAction, PlannedIntervention, @@ -116,6 +117,17 @@ class DrefOperationalUpdateTO(TranslationOptions): ) +@register(DrefSummary) +class DrefSummaryTO(TranslationOptions): + fields = ( + "situational_overview", + "operational_strategy", + "people_centered_approach", + "challenges_identified", + "lessons_learned", + ) + + @register(IdentifiedNeed) class IdentifiedNeedTO(TranslationOptions): fields = ("description",) diff --git a/dref/utils.py b/dref/utils.py index 70db6822c..3baa13521 100644 --- a/dref/utils.py +++ b/dref/utils.py @@ -1,7 +1,9 @@ from django.conf import settings from django.contrib.postgres.aggregates import ArrayAgg from django.db import models +from django.utils import timezone +from api.models import Event from dref.models import Dref, DrefFinalReport, DrefOperationalUpdate @@ -53,3 +55,33 @@ def get_dref_users(): ) ) return dref_users_list + + +def create_event_from_dref(dref: Dref) -> Event: + create_kwargs = dict( + name=dref.title, + dtype=dref.disaster_type, + summary=dref.event_description or dref.event_scope or "", + disaster_start_date=dref.event_date or dref.hazard_date, + glide=dref.glide_code or "", + auto_generated=True, + source=Event.EventSource.DREF, + ) + + # Dref.DisasterCategory and api.AlertLevel share the same 0/1/2 indices, so + # the value maps directly with no remap. + if dref.disaster_category is not None: + create_kwargs["ifrc_severity_level"] = dref.disaster_category + create_kwargs["ifrc_severity_level_update_date"] = dref.date_of_approval or timezone.now() + + event = Event.objects.create(**create_kwargs) + + country = getattr(dref, "country", None) + if country: + event.countries.add(dref.country) + + event.districts.add(*dref.district.all()) + region = getattr(country, "region", None) + if region: + event.regions.add(region) + return event diff --git a/dref/views.py b/dref/views.py index 7975881db..7dfe58e34 100644 --- a/dref/views.py +++ b/dref/views.py @@ -38,6 +38,7 @@ AddDrefUserSerializer, CompletedDrefOperationsSerializer, Dref3Serializer, + DrefApproveSerializer, DrefFileInputSerializer, DrefFileSerializer, DrefFinalReport3Serializer, @@ -49,7 +50,9 @@ DrefShareUserSerializer, MiniDrefSerializer, ) -from dref.tasks import process_dref_translation +from dref.tasks import generate_dref_summary, process_dref_translation +from dref.utils import create_event_from_dref +from lang.serializers import TranslatedModelSerializerMixin from main.permissions import DenyGuestUserPermission logger = logging.getLogger(__name__) @@ -90,23 +93,52 @@ def get_queryset(self): ) return filter_dref_queryset_by_user_access(user, queryset) - @extend_schema(request=None, responses=DrefSerializer) + @extend_schema( + request=DrefApproveSerializer, + responses=DrefSerializer, + ) @action( detail=True, url_path="approve", methods=["post"], - permission_classes=[permissions.IsAuthenticated, ApproveDrefPermission, DenyGuestUserPermission], + permission_classes=[ + permissions.IsAuthenticated, + ApproveDrefPermission, + DenyGuestUserPermission, + ], ) def get_approved(self, request, pk=None, version=None): - dref = self.get_object() + dref: Dref = self.get_object() + if dref.status == Dref.Status.APPROVED: raise serializers.ValidationError(gettext("This Dref has already been approved.")) + if dref.status != Dref.Status.FINALIZED: raise serializers.ValidationError(gettext("Must be finalized before it can be approved")) + + serializer = DrefApproveSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + event = serializer.validated_data.get("event", None) + + if dref.event and event and dref.event != event: + raise serializers.ValidationError({"event": gettext("This Dref is already attached to an event.")}) + + # NOTE: If the Dref is not attached to an event, + # attaching it to the provided event or create a new one if not provided. + if not dref.event: + if event: + dref.event = event + else: + event = create_event_from_dref(dref) + dref.event = event + # Translate the emergency instance + TranslatedModelSerializerMixin.trigger_field_translation(event) + dref.status = Dref.Status.APPROVED - dref.save(update_fields=["status"]) - serializer = DrefSerializer(dref, context={"request": request}) - return response.Response(serializer.data) + dref.save(update_fields=["event", "status"]) + transaction.on_commit(lambda: generate_dref_summary.delay(dref_id=dref.id)) + return response.Response(DrefSerializer(dref, context={"request": request}).data) @extend_schema(request=None, responses=DrefSerializer) @action( @@ -200,7 +232,9 @@ def get_approved(self, request, pk=None, version=None): raise serializers.ValidationError(gettext("Must be finalized before it can be approved.")) operational_update.status = Dref.Status.APPROVED - operational_update.save(update_fields=["status"]) + operational_update.date_of_approval = timezone.now().date() + operational_update.save(update_fields=["status", "date_of_approval"]) + transaction.on_commit(lambda: generate_dref_summary.delay(dref_id=operational_update.dref_id)) serializer = DrefOperationalUpdateSerializer(operational_update, context={"request": request}) return response.Response(serializer.data) @@ -269,6 +303,7 @@ def get_approved(self, request, pk=None, version=None): final_report.dref.is_active = False final_report.date_of_approval = timezone.now().date() final_report.dref.save(update_fields=["is_active", "date_of_approval"]) + transaction.on_commit(lambda: generate_dref_summary.delay(dref_id=final_report.dref_id)) serializer = DrefFinalReportSerializer(final_report, context={"request": request}) return response.Response(serializer.data) @@ -335,7 +370,22 @@ class CompletedDrefOperationsViewSet(viewsets.ReadOnlyModelViewSet): DenyGuestUserPermission, ] filterset_class = CompletedDrefOperationsFilterSet - queryset = DrefFinalReport.objects.filter(status=Dref.Status.APPROVED).order_by("-created_at").distinct() + queryset = ( + DrefFinalReport.objects.filter(status=Dref.Status.APPROVED) + .select_related("country", "dref", "dref__country") + .prefetch_related( + # MiniDrefSerializer.operational_update_details reads this prefetched + # attr; without it DRF silently drops the field. + Prefetch( + "dref__drefoperationalupdate_set", + queryset=DrefOperationalUpdate.objects.select_related("country").order_by("-created_at"), + to_attr="prefetched_operational_updates", + ), + "dref__dreffinalreport__country", + ) + .order_by("-created_at") + .distinct() + ) def get_queryset(self): user = self.request.user diff --git a/main/llm.py b/main/llm.py new file mode 100644 index 000000000..46860e10a --- /dev/null +++ b/main/llm.py @@ -0,0 +1,183 @@ +import json +import logging +import re +from typing import Dict, List, Optional + +from django.conf import settings +from django.utils.functional import cached_property +from openai import AzureOpenAI + +logger = logging.getLogger(__name__) + +# A chat completion prompt: the ``[{"role": ..., "content": ...}, ...]`` list +# every real call site builds and passes in. +Message = Dict[str, str] +Messages = List[Message] + + +PLACEHOLDER_TEXT = ( + "This is placeholder text standing in for a real AI-generated summary. It is returned by " + "the dummy LLM client Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt " + "ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco " + "laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in " + "voluptate velit esse cillum dolore eu fugiat nulla pariatur." +) + + +class BaseLLMClient: + def get_response(self, messages: Messages) -> Optional[str]: + raise NotImplementedError + + @staticmethod + def _last_user_message(messages: Messages) -> str: + return next((message["content"] for message in reversed(messages) if message["role"] == "user"), "") + + def _fake_content(self) -> str: + """Placeholder text every dummy response uses in place of real LLM output.""" + return f"DUMMY RESPONSE, {PLACEHOLDER_TEXT}" + + +class AzureOpenAiChat(BaseLLMClient): + """The real Azure OpenAI-backed client, shared by every feature.""" + + def __init__(self, temperature: float = 0.5): + self.temperature = temperature + + if settings.TESTING: + # Tests never hit Azure, so a missing/placeholder config shouldn't fail construction. + return + + if not settings.AZURE_OPENAI_ENDPOINT or not settings.AZURE_OPENAI_API_KEY or not settings.AZURE_OPENAI_DEPLOYMENT_NAME: + raise Exception("Azure OpenAI configuration missing") + + @cached_property + def client(self): + return AzureOpenAI( + azure_endpoint=settings.AZURE_OPENAI_ENDPOINT, + api_key=settings.AZURE_OPENAI_API_KEY, + api_version="2023-05-15", + ) + + def get_response(self, messages: Messages) -> Optional[str]: + if settings.USE_DUMMY_LLM_CLIENT or settings.TESTING: + logger.info(f"{type(self).__name__}: dummy mode is on; returning a fake response instead of calling Azure OpenAI.") + return self._fake_content() + + try: + response = self.client.chat.completions.create( + model=settings.AZURE_OPENAI_DEPLOYMENT_NAME, + messages=messages, + temperature=self.temperature, + ) + return response.choices[0].message.content + except Exception as e: + logger.error(f"Error while generating LLM response: {e}", exc_info=True) + return None + + +class BaseDummyLLMClient(BaseLLMClient): + """Summaries generated for local development/testing, without hitting Azure OpenAI. Each subclass implements + ``_fake_response()`` to return a JSON object with the expected fields for that feature + """ + + def get_response(self, messages: Messages) -> Optional[str]: + return self._fake_response(messages) + + def _fake_response(self, messages: Messages) -> str: + raise NotImplementedError + + +class DummyDrefSummaryLLMClient(BaseDummyLLMClient): + """Fake response for DREF summary generation: a JSON object with all of + DREF's summary fields, each set to the same fake content, so + ``dref.summary.DrefSummaryGenerator`` parses/saves it unchanged. + """ + + def _fake_response(self, messages: Messages) -> str: + from dref.summary import SUMMARY_FIELDS + + logger.info("DummyDrefSummaryLLMClient: generating fake response for DREF summary fields") + fake_content = self._fake_content() + payload: Dict[str, str] = {field_name: fake_content for field_name in SUMMARY_FIELDS} + return json.dumps(payload) + + +class DummyOpsLearningLLMClient(BaseDummyLLMClient): + """Fake response for PER Ops Learning summaries.""" + + _EXCERPT_ID_RE = re.compile(r"^(\d+)\. In ", re.MULTILINE) + + def _prompt_excerpt_ids(self, user_content: str) -> List[int]: + return [int(excerpt_id) for excerpt_id in self._EXCERPT_ID_RE.findall(user_content)] + + def _fake_response(self, messages: Messages) -> str: + from per.models import OpsLearning + from per.ops_learning_summary import OpsLearningSummaryTask + + logger.info("DummyOpsLearningLLMClient: generating fake response for Ops Learning summary") + user_content = self._last_user_message(messages) + fake_content = self._fake_content() + excerpt_ids = self._prompt_excerpt_ids(user_content) + logger.info(f"DummyOpsLearningLLMClient: found {len(excerpt_ids)} excerpt id(s) in the prompt") + + all_excerpts_id = ", ".join(str(excerpt_id) for excerpt_id in excerpt_ids) + + payload: Dict[str, Dict[str, str]] + if OpsLearningSummaryTask.component_prompt in user_content: + logger.info("DummyOpsLearningLLMClient: detected the component prompt shape.") + excerpts = list( + OpsLearning.objects.filter(id__in=excerpt_ids, is_validated=True, per_component_validated__isnull=False) + .prefetch_related("per_component_validated") + .distinct() + ) + component = excerpts[0].per_component_validated.first() if excerpts else None + subtype = component.title if component else "Logistics" + excerpts_id = ", ".join(str(excerpt.id) for excerpt in excerpts) or all_excerpts_id + logger.info( + f"DummyOpsLearningLLMClient: {len(excerpts)} excerpt(s) matched a validated component; " + f"using subtype '{subtype}'." + ) + payload = {"0": {"type": "component", "subtype": subtype, "excerpts id": excerpts_id, "content": fake_content}} + + elif OpsLearningSummaryTask.sector_prompt in user_content: + logger.info("DummyOpsLearningLLMClient: detected the sector prompt shape.") + excerpts = list( + OpsLearning.objects.filter(id__in=excerpt_ids, is_validated=True, sector_validated__isnull=False) + .prefetch_related("sector_validated") + .distinct() + ) + sector = excerpts[0].sector_validated.first() if excerpts else None + subtype = sector.title if sector else "Shelter" + excerpts_id = ", ".join(str(excerpt.id) for excerpt in excerpts) or all_excerpts_id + logger.info( + f"DummyOpsLearningLLMClient: {len(excerpts)} excerpt(s) matched a validated sector; " + f"using subtype '{subtype}'." + ) + payload = {"0": {"type": "sector", "subtype": subtype, "excerpts id": excerpts_id, "content": fake_content}} + else: + logger.info( + f"DummyOpsLearningLLMClient: detected the primary prompt shape; " + f"returning 3 fake findings with excerpts id(s) [{all_excerpts_id}]." + ) + payload = { + str(i): { + "title": f"Dummy Finding {i + 1}", + "excerpts id": all_excerpts_id, + "content": fake_content, + "confidence level": "1/5", + } + for i in range(3) + } + return json.dumps(payload) + + +def get_dref_summary_llm_client() -> BaseLLMClient: + if settings.USE_DUMMY_LLM_CLIENT or settings.TESTING: + return DummyDrefSummaryLLMClient() + return AzureOpenAiChat(temperature=0.5) + + +def get_ops_learning_llm_client() -> BaseLLMClient: + if settings.USE_DUMMY_LLM_CLIENT or settings.TESTING: + return DummyOpsLearningLLMClient() + return AzureOpenAiChat(temperature=0.7) diff --git a/main/lock.py b/main/lock.py index 3d1fd2de1..9de5b7aaf 100644 --- a/main/lock.py +++ b/main/lock.py @@ -20,6 +20,7 @@ class RedisLockKey(models.TextChoices): OPERATION_LEARNING_SUMMARY = _BASE + "-operation-learning-summary-{0}" OPERATION_LEARNING_SUMMARY_EXPORT = _BASE + "-operation-learning-summary-export-{0}" MODEL_TRANSLATION = _BASE + "-{model_name}-translation-{id}" + DREF_SUMMARY = _BASE + "-dref-summary-{0}" @contextmanager diff --git a/main/settings.py b/main/settings.py index a500bef8c..0927e30ef 100644 --- a/main/settings.py +++ b/main/settings.py @@ -144,8 +144,10 @@ NS_INITIATIVES_API_TOKEN=(str, None), # OpenAi Azure AZURE_OPENAI_ENDPOINT=(str, None), - AZURE_OPENAI_KEY=(str, None), + AZURE_OPENAI_API_KEY=(str, None), AZURE_OPENAI_DEPLOYMENT_NAME=(str, None), + # Use a fake LLM client instead of calling Azure OpenAI + USE_DUMMY_LLM_CLIENT=(bool, False), # ReliefWeb appname RELIEF_WEB_APP_NAME=(str, None), # PowerBI @@ -866,8 +868,9 @@ def decode_base64(env_key, fallback_env_key): JWT_EXPIRE_TIMESTAMP_DAYS = env("JWT_EXPIRE_TIMESTAMP_DAYS") AZURE_OPENAI_ENDPOINT = env("AZURE_OPENAI_ENDPOINT") -AZURE_OPENAI_KEY = env("AZURE_OPENAI_KEY") +AZURE_OPENAI_API_KEY = env("AZURE_OPENAI_API_KEY") AZURE_OPENAI_DEPLOYMENT_NAME = env("AZURE_OPENAI_DEPLOYMENT_NAME") +USE_DUMMY_LLM_CLIENT = env("USE_DUMMY_LLM_CLIENT") OIDC_ENABLE = env("OIDC_ENABLE") OIDC_RSA_PRIVATE_KEY = None diff --git a/main/urls.py b/main/urls.py index ddf055962..0fcb30b25 100644 --- a/main/urls.py +++ b/main/urls.py @@ -91,6 +91,9 @@ router.register(r"admin2", api_views.Admin2Viewset, basename="admin2") router.register(r"district", api_views.DistrictViewset, basename="district") router.register(r"district_rmd", api_views.DistrictRMDViewset, basename="district_rmd") + +router.register(r"emergency", api_views.EmergencyViewset, basename="emergency") + router.register(r"domainwhitelist", registration_views.DomainWhitelistViewset) router.register(r"eru", deployment_views.ERUViewset, basename="eru") router.register( @@ -269,6 +272,7 @@ url(r"^api/v2/per-options/", per_views.PerOptionsView.as_view()), url(r"^api/v2/export-per/(?P\d+)/", per_views.ExportPerView.as_view()), url(r"^api/v2/local-units-options/", local_units_views.LocalUnitOptionsView.as_view()), + # NOTE: This should be removed as DefaultRouter should cover this. url(r"^api/v2/event/(?P\d+)", api_views.EventViewset.as_view({"get": "retrieve"})), url(r"^api/v2/event/(?P[-\w]+)", api_views.EventViewset.as_view({"get": "retrieve"}, lookup_field="slug")), url(r"^api/v2/delegation-office/(?P\d+)", DelegationOfficeDetailAPIView.as_view()), diff --git a/per/models.py b/per/models.py index 4cd7e7488..3b792c426 100644 --- a/per/models.py +++ b/per/models.py @@ -705,6 +705,9 @@ class OpsLearning(models.Model): modified_at = models.DateTimeField(verbose_name=_("modified_at"), auto_now=True) created_at = models.DateTimeField(verbose_name=_("created at"), auto_now_add=True) + # TYPING + id: int + class Meta: ordering = ("learning",) verbose_name = _("Operational Learning") diff --git a/per/ops_learning_summary.py b/per/ops_learning_summary.py index d2d3d6cd4..5a5d150ed 100644 --- a/per/ops_learning_summary.py +++ b/per/ops_learning_summary.py @@ -5,17 +5,15 @@ import pandas as pd import tiktoken -from django.conf import settings from django.db import transaction from django.db.models import F -from django.utils.functional import cached_property -from openai import AzureOpenAI from api.logger import logger from api.models import Country from api.utils import get_model_name from deployments.models import SectorTag from lang.tasks import translate_model_fields +from main.llm import get_ops_learning_llm_client from per.cache import OpslearningSummaryCacheHelper from per.models import ( FormComponent, @@ -29,24 +27,6 @@ ) -class AzureOpenAiChat: - - @cached_property - def client(self): - return AzureOpenAI( - azure_endpoint=settings.AZURE_OPENAI_ENDPOINT, api_key=settings.AZURE_OPENAI_KEY, api_version="2023-05-15" - ) - - def get_response(self, message): - try: - response = self.client.chat.completions.create( - model=settings.AZURE_OPENAI_DEPLOYMENT_NAME, messages=message, temperature=0.7 - ) - return response.choices[0].message.content - except Exception as e: - logger.error(f"Error while generating response: {e}", exc_info=True) - - class OpsLearningSummaryTask: PROMPT_DATA_LENGTH_LIMIT = 5000 @@ -815,9 +795,9 @@ def _summarize(prompt, type: OpsLearningPromptResponseCache.PromptType, system_m logger.warning("The length of the prompt might be too long.") return "{}" - # Using Azure OpenAI to summarize the prompt - client = AzureOpenAiChat() - response = client.get_response(message=messages) + # Using Azure OpenAI (or the dummy, per settings.USE_DUMMY_LLM_CLIENT) to summarize the prompt + client = get_ops_learning_llm_client() + response = client.get_response(messages) return response def _validate_format(summary, MAX_RETRIES=3):