feat: system comments for p1, p2 areas

apps/internal/migrations/0029_auto_generated_sql.py

@@ -0,0 +1,9 @@
+# Generated by Django
+from django.db import migrations
+from apps.internal.helpers import safe_run_sql
+sql_files = [
+    'fyle-integrations-db-migrations/intacct/functions/delete_workspace.sql'
+]
+class Migration(migrations.Migration):
+    dependencies = [('internal', '0028_auto_generated_sql')]
+    operations = safe_run_sql(sql_files)

apps/internal/tasks.py

@@ -4,14 +4,18 @@
 
 from django.db.models import Q
 from django_q.models import OrmQ, Schedule
+
+from fyle_accounting_library.system_comments.models import SystemComment
 from fyle_accounting_library.fyle_platform.enums import ExpenseImportSourceEnum
 
 from apps.tasks.models import TaskLog
 from apps.fyle.models import ExpenseGroup
 from apps.workspaces.actions import export_to_intacct
 from apps.workspaces.models import LastExportDetail, Workspace
+from apps.workspaces.system_comments import add_system_comment
 from workers.helpers import RoutingKeyEnum, WorkerActionEnum, publish_to_rabbitmq
 from apps.fyle.actions import post_accounting_export_summary, update_failed_expenses
+from apps.workspaces.enums import SystemCommentEntityTypeEnum, SystemCommentIntentEnum, SystemCommentReasonEnum, SystemCommentSourceEnum
 
 logger = logging.getLogger(__name__)
 logger.level = logging.INFO
@@ -36,6 +40,7 @@ def retrigger_stuck_exports() -> None:
     Re-triggers export stuck exports by identifying failed export attempts
     and retrying them.
     """
+    system_comments = []
     prod_workspace_ids = Workspace.objects.filter(
         ~Q(name__icontains='fyle for') & ~Q(name__iendswith='test')
     ).values_list('id', flat=True)
@@ -66,6 +71,7 @@ def retrigger_stuck_exports() -> None:
         for expense_group in expense_groups:
             expenses.extend(expense_group.expenses.all())
         workspace_ids_list = list(workspace_ids)
+        task_logs_dict = {tl.expense_group_id: tl for tl in task_logs}
         task_logs.update(status='FAILED', updated_at=datetime.now(), re_attempt_export=True)
         for workspace_id in workspace_ids_list:
             errored_expenses = [expense for expense in expenses if expense.workspace_id == workspace_id]
@@ -83,10 +89,30 @@ def retrigger_stuck_exports() -> None:
                 export_expense_group_ids = list(expense_groups.filter(workspace_id=workspace_id).values_list('id', flat=True))
                 if export_expense_group_ids and len(export_expense_group_ids) < 200:
                     logger.info('Re-triggering export for expense group %s since no 1 hour schedule for workspace  %s', export_expense_group_ids, workspace_id)
+
+                    for expense_group_id in export_expense_group_ids:
+                        expense_group = expense_groups.filter(id=expense_group_id, workspace_id=workspace_id).first()
+                        task_log = task_logs_dict.get(expense_group_id)
+                        if expense_group and task_log:
+                            stuck_duration_seconds = (datetime.now(timezone.utc) - task_log.updated_at.replace(tzinfo=timezone.utc)).total_seconds()
+                            add_system_comment(
+                                system_comments=system_comments,
+                                source=SystemCommentSourceEnum.RETRIGGER_STUCK_EXPORTS,
+                                intent=SystemCommentIntentEnum.EXPORT_RETRIGGERED,
+                                entity_type=SystemCommentEntityTypeEnum.EXPENSE_GROUP,
+                                workspace_id=workspace_id,
+                                entity_id=expense_group_id,
+                                reason=SystemCommentReasonEnum.EXPORT_RETRIGGERED_STUCK,
+                                info={'stuck_duration_seconds': stuck_duration_seconds}
+                            )
+
                     export_to_intacct(workspace_id=workspace_id, expense_group_ids=export_expense_group_ids, triggered_by=ExpenseImportSourceEnum.INTERNAL)
                 else:
                     logger.info('Skipping export for workspace %s since it has more than 200 expense groups', workspace_id)
 
+        if system_comments:
+            SystemComment.bulk_create_comments(system_comments)
+
 
 def pause_and_resume_export_schedules() -> None:
     """

apps/sage_intacct/models.py

@@ -147,6 +147,7 @@ def get_project_id_or_none(expense_group: ExpenseGroup, lineitem: Expense, gener
             entity_type=SystemCommentEntityTypeEnum.EXPENSE,
             workspace_id=expense_group.workspace_id,
             entity_id=lineitem.id,
+            persist_without_export=False,
             reason=SystemCommentReasonEnum.DEFAULT_PROJECT_APPLIED,
             info={'default_project_id': general_mappings.default_project_id, 'default_project_name': general_mappings.default_project_name}
         )
@@ -200,7 +201,8 @@ def get_department_id_or_none(expense_group: ExpenseGroup, lineitem: Expense, ge
                 workspace_id=expense_group.workspace_id,
                 entity_id=lineitem.id,
                 reason=SystemCommentReasonEnum.EMPLOYEE_DEPARTMENT_APPLIED,
-                info={'employee_department_id': employee_department, 'employee_department_name': employee_department}
+                info={'employee_department_id': employee_department, 'employee_department_name': employee_department},
+                persist_without_export=False
             )
             return employee_department
 
@@ -214,7 +216,8 @@ def get_department_id_or_none(expense_group: ExpenseGroup, lineitem: Expense, ge
             workspace_id=expense_group.workspace_id,
             entity_id=lineitem.id,
             reason=SystemCommentReasonEnum.DEFAULT_DEPARTMENT_APPLIED,
-            info={'default_department_id': general_mappings.default_department_id, 'default_department_name': general_mappings.default_department_name}
+            info={'default_department_id': general_mappings.default_department_id, 'default_department_name': general_mappings.default_department_name},
+            persist_without_export=False
         )
         return general_mappings.default_department_id
 
@@ -266,7 +269,8 @@ def get_location_id_or_none(expense_group: ExpenseGroup, lineitem: Expense, gene
                 workspace_id=expense_group.workspace_id,
                 entity_id=lineitem.id,
                 reason=SystemCommentReasonEnum.EMPLOYEE_LOCATION_APPLIED,
-                info={'employee_location_id': employee_location, 'employee_location_name': employee_location}
+                info={'employee_location_id': employee_location, 'employee_location_name': employee_location},
+                persist_without_export=False
             )
             return employee_location
 
@@ -280,7 +284,8 @@ def get_location_id_or_none(expense_group: ExpenseGroup, lineitem: Expense, gene
             workspace_id=expense_group.workspace_id,
             entity_id=lineitem.id,
             reason=SystemCommentReasonEnum.DEFAULT_LOCATION_APPLIED,
-            info={'default_location_id': general_mappings.default_location_id, 'default_location_name': general_mappings.default_location_name}
+            info={'default_location_id': general_mappings.default_location_id, 'default_location_name': general_mappings.default_location_name},
+            persist_without_export=False
         )
         return general_mappings.default_location_id
 
@@ -377,6 +382,7 @@ def get_item_id_or_none(expense_group: ExpenseGroup, lineitem: Expense, general_
             workspace_id=expense_group.workspace_id,
             entity_id=lineitem.id,
             reason=SystemCommentReasonEnum.DEFAULT_ITEM_APPLIED,
+            persist_without_export=False,
             info={'default_item_id': general_mappings.default_item_id, 'default_item_name': general_mappings.default_item_name}
         )
         return general_mappings.default_item_id
@@ -533,7 +539,8 @@ def get_class_id_or_none(expense_group: ExpenseGroup, lineitem: Expense, general
             workspace_id=expense_group.workspace_id,
             entity_id=lineitem.id,
             reason=SystemCommentReasonEnum.DEFAULT_CLASS_APPLIED,
-            info={'default_class_id': general_mappings.default_class_id, 'default_class_name': general_mappings.default_class_name}
+            info={'default_class_id': general_mappings.default_class_id, 'default_class_name': general_mappings.default_class_name},
+            persist_without_export=False
         )
         return general_mappings.default_class_id
 
@@ -909,7 +916,8 @@ def get_ccc_account_id(general_mappings: GeneralMapping, expense: Expense, descr
             workspace_id=general_mappings.workspace_id,
             entity_id=expense.id,
             reason=SystemCommentReasonEnum.EMPLOYEE_CCC_ACCOUNT_APPLIED,
-            info={'employee_ccc_account_id': employee_mapping.destination_card_account.destination_id, 'employee_ccc_account_name': employee_mapping.destination_card_account.display_name}
+            info={'employee_ccc_account_id': employee_mapping.destination_card_account.destination_id, 'employee_ccc_account_name': employee_mapping.destination_card_account.display_name},
+            persist_without_export=False
         )
 
         return employee_mapping.destination_card_account.destination_id
@@ -922,7 +930,8 @@ def get_ccc_account_id(general_mappings: GeneralMapping, expense: Expense, descr
         workspace_id=general_mappings.workspace_id,
         entity_id=expense.id,
         reason=SystemCommentReasonEnum.DEFAULT_CREDIT_CARD_APPLIED,
-        info={'default_charge_card_id': general_mappings.default_charge_card_id, 'default_charge_card_name': general_mappings.default_charge_card_name}
+        info={'default_charge_card_id': general_mappings.default_charge_card_id, 'default_charge_card_name': general_mappings.default_charge_card_name},
+        persist_without_export=False
     )
 
     return general_mappings.default_charge_card_id
@@ -1009,7 +1018,8 @@ def create_bill(expense_group: ExpenseGroup, supdoc_id: str = None, system_comme
                     workspace_id=expense_group.workspace_id,
                     entity_id=expense_group.id,
                     reason=SystemCommentReasonEnum.DEFAULT_CCC_VENDOR_APPLIED,
-                    info={'default_ccc_vendor_id': general_mappings.default_ccc_vendor_id, 'default_ccc_vendor_name': general_mappings.default_ccc_vendor_name}
+                    info={'default_ccc_vendor_id': general_mappings.default_ccc_vendor_id, 'default_ccc_vendor_name': general_mappings.default_ccc_vendor_name},
+                    persist_without_export=False
                 )
 
         bill_object, _ = Bill.objects.update_or_create(
@@ -1126,6 +1136,23 @@ def create_bill_lineitems(expense_group: ExpenseGroup, configuration: Configurat
                 allocation_dimensions = set(allocation_detail.keys())
                 user_defined_dimensions = [user_defined_dimension for user_defined_dimension in user_defined_dimensions if list(user_defined_dimension.keys())[0] not in allocation_dimensions]
 
+            if lineitem.billable and not (dimensions_values['customer_id'] and dimensions_values['item_id']):
+                missing_fields = []
+                if not dimensions_values['customer_id']:
+                    missing_fields.append('customer_id')
+                if not dimensions_values['item_id']:
+                    missing_fields.append('item_id')
+                add_system_comment(
+                    system_comments=system_comments,
+                    source=SystemCommentSourceEnum.CREATE_BILL_LINEITEMS,
+                    intent=SystemCommentIntentEnum.BILLABLE_DISABLED,
+                    entity_type=SystemCommentEntityTypeEnum.EXPENSE,
+                    workspace_id=expense_group.workspace_id,
+                    entity_id=lineitem.id,
+                    reason=SystemCommentReasonEnum.BILLABLE_SET_TO_FALSE_MISSING_DIMENSIONS,
+                    info={'missing_fields': missing_fields, 'original_billable': lineitem.billable}
+                )
+
             bill_lineitem_object, _ = BillLineitem.objects.update_or_create(
                 bill=bill,
                 expense_id=lineitem.id,
@@ -1341,9 +1368,27 @@ def create_expense_report_lineitems(expense_group: ExpenseGroup, configuration:
                         workspace_id=expense_group.workspace_id,
                         entity_id=lineitem.id,
                         reason=SystemCommentReasonEnum.CREDIT_CARD_MISC_VENDOR_APPLIED,
-                        info={'original_merchant': merchant, 'vendor_used': 'Credit Card Misc'}
+                        info={'original_merchant': merchant, 'vendor_used': 'Credit Card Misc'},
+                        persist_without_export=False
                     )
 
+            if lineitem.billable and not (customer_id and item_id):
+                missing_fields = []
+                if not customer_id:
+                    missing_fields.append('customer_id')
+                if not item_id:
+                    missing_fields.append('item_id')
+                add_system_comment(
+                    system_comments=system_comments,
+                    source=SystemCommentSourceEnum.CREATE_EXPENSE_REPORT_LINEITEMS,
+                    intent=SystemCommentIntentEnum.BILLABLE_DISABLED,
+                    entity_type=SystemCommentEntityTypeEnum.EXPENSE,
+                    workspace_id=expense_group.workspace_id,
+                    entity_id=lineitem.id,
+                    reason=SystemCommentReasonEnum.BILLABLE_SET_TO_FALSE_MISSING_DIMENSIONS,
+                    info={'missing_fields': missing_fields, 'original_billable': lineitem.billable}
+                )
+
             expense_report_lineitem_object, _ = ExpenseReportLineitem.objects.update_or_create(
                 expense_report=expense_report,
                 expense_id=lineitem.id,
@@ -1523,7 +1568,8 @@ def create_journal_entry_lineitems(expense_group: ExpenseGroup, configuration: C
                             workspace_id=expense_group.workspace_id,
                             entity_id=lineitem.id,
                             reason=SystemCommentReasonEnum.CREDIT_CARD_MISC_VENDOR_APPLIED,
-                            info={'original_merchant': lineitem.vendor, 'vendor_used': 'Credit Card Misc'}
+                            info={'original_merchant': lineitem.vendor, 'vendor_used': 'Credit Card Misc'},
+                            persist_without_export=False
                         )
 
             else:
@@ -1547,7 +1593,8 @@ def create_journal_entry_lineitems(expense_group: ExpenseGroup, configuration: C
                             workspace_id=expense_group.workspace_id,
                             entity_id=lineitem.id,
                             reason=SystemCommentReasonEnum.CREDIT_CARD_MISC_VENDOR_APPLIED,
-                            info={'original_merchant': merchant, 'vendor_used': 'Credit Card Misc'}
+                            info={'original_merchant': merchant, 'vendor_used': 'Credit Card Misc'},
+                            persist_without_export=False
                         )
                     else:
                         raise ValueErrorWithResponse(message='Something Went Wrong', response='Credit Card Misc vendor not found')
@@ -1568,6 +1615,23 @@ def create_journal_entry_lineitems(expense_group: ExpenseGroup, configuration: C
 
             allocation_id, _ = get_allocation_id_or_none(expense_group=expense_group, lineitem=lineitem)
 
+            if lineitem.billable and not (customer_id and item_id):
+                missing_fields = []
+                if not customer_id:
+                    missing_fields.append('customer_id')
+                if not item_id:
+                    missing_fields.append('item_id')
+                add_system_comment(
+                    system_comments=system_comments,
+                    source=SystemCommentSourceEnum.CREATE_JOURNAL_ENTRY_LINEITEMS,
+                    intent=SystemCommentIntentEnum.BILLABLE_DISABLED,
+                    entity_type=SystemCommentEntityTypeEnum.EXPENSE,
+                    workspace_id=expense_group.workspace_id,
+                    entity_id=lineitem.id,
+                    reason=SystemCommentReasonEnum.BILLABLE_SET_TO_FALSE_MISSING_DIMENSIONS,
+                    info={'missing_fields': missing_fields, 'original_billable': lineitem.billable}
+                )
+
             journal_entry_lineitem_object, _ = JournalEntryLineitem.objects.update_or_create(
                 journal_entry=journal_entry,
                 expense_id=lineitem.id,
@@ -1656,7 +1720,8 @@ def create_charge_card_transaction(expense_group: ExpenseGroup, vendor_id: str =
                     workspace_id=expense_group.workspace_id,
                     entity_id=expense_group.id,
                     reason=SystemCommentReasonEnum.CREDIT_CARD_MISC_VENDOR_APPLIED,
-                    info={'original_merchant': merchant, 'vendor_used': 'Credit Card Misc'}
+                    info={'original_merchant': merchant, 'vendor_used': 'Credit Card Misc'},
+                    persist_without_export=False
                 )
             else:
                 raise ValueErrorWithResponse(message='Something Went Wrong', response='Credit Card Misc vendor not found')
@@ -1792,7 +1857,25 @@ def create_charge_card_transaction_lineitems(expense_group: ExpenseGroup, config
                     workspace_id=expense_group.workspace_id,
                     entity_id=lineitem.id,
                     reason=SystemCommentReasonEnum.CREDIT_CARD_MISC_VENDOR_APPLIED,
-                    info={'original_merchant': lineitem.vendor, 'vendor_used': 'Credit Card Misc'}
+                    info={'original_merchant': lineitem.vendor, 'vendor_used': 'Credit Card Misc'},
+                    persist_without_export=False
+                )
+
+            if lineitem.billable and not (customer_id and item_id):
+                missing_fields = []
+                if not customer_id:
+                    missing_fields.append('customer_id')
+                if not item_id:
+                    missing_fields.append('item_id')
+                add_system_comment(
+                    system_comments=system_comments,
+                    source=SystemCommentSourceEnum.CREATE_CHARGE_CARD_TRANSACTION_LINEITEMS,
+                    intent=SystemCommentIntentEnum.BILLABLE_DISABLED,
+                    entity_type=SystemCommentEntityTypeEnum.EXPENSE,
+                    workspace_id=expense_group.workspace_id,
+                    entity_id=lineitem.id,
+                    reason=SystemCommentReasonEnum.BILLABLE_SET_TO_FALSE_MISSING_DIMENSIONS,
+                    info={'missing_fields': missing_fields, 'original_billable': lineitem.billable}
                 )
 
             charge_card_transaction_lineitem_object, _ = ChargeCardTransactionLineitem.objects.update_or_create(