diff --git a/dtable_events/automations/actions.py b/dtable_events/automations/actions.py index b959bf4f..8ec1d644 100644 --- a/dtable_events/automations/actions.py +++ b/dtable_events/automations/actions.py @@ -30,6 +30,7 @@ from dtable_events.utils.dtable_server_api import DTableServerAPI from dtable_events.utils.dtable_web_api import DTableWebAPI from dtable_events.utils.dtable_db_api import DTableDBAPI, RowsQueryError, Request429Error +from dtable_events.utils.row_converter import convert_row_with_sql_row from dtable_events.notification_rules.utils import get_nickname_by_usernames from dtable_events.utils.sql_generator import filter2sql, BaseSQLGenerator, ColumnFilterInvalidError, \ has_user_filter, is_user_filter @@ -212,12 +213,15 @@ def send_selected_collaborator_notis(self, row_data): if user not in notify_users: notify_users.append(user) elif self.action_type == 'update_record': - converted_row = self.auto_rule.get_convert_sql_row() - row_id = converted_row['_id'] + sql_row = self.auto_rule.get_sql_row() + row_id = sql_row['_id'] for column_name, value in row_data.items(): if column_name not in notify_column_names: continue - old_value = converted_row.get(column_name) or [] + column = next(filter(lambda column: column['name'] == column_name, self.auto_rule.table_info['columns']), None) + if not column: + continue + old_value = sql_row.get(column['key']) or [] for user in (set(value) - set(old_value)): if user not in notify_users: notify_users.append(user) @@ -1603,7 +1607,8 @@ class LinkRecordsAction(BaseAction): ColumnTypes.COLLABORATOR: "is_exactly", ColumnTypes.EMAIL: "is", ColumnTypes.RATE: "equal", - ColumnTypes.AUTO_NUMBER: "is" + ColumnTypes.AUTO_NUMBER: "is", + ColumnTypes.DEPARTMENT_SINGLE_SELECT: "is" } VALID_COLUMN_TYPES = [ @@ -3381,7 +3386,6 @@ def __init__(self, data, db_session, raw_trigger, raw_actions, options, metadata self._sql_query_count = 0 self._convert_sql_row = None - self._convert_sql_query_count = 0 self._sql_query_max = 3 @@ -3486,22 +3490,9 @@ def related_users_dict(self): return self._related_users_dict def get_convert_sql_row(self): - if self._convert_sql_row is not None or self._convert_sql_query_count >= self._sql_query_max: + if self._convert_sql_row is not None: return self._convert_sql_row - if not self.data: - return None - if 'row_id' not in self.data: - return None - row_id = self.data['row_id'] - while not self._convert_sql_row and self._convert_sql_query_count < self._sql_query_max: - sql = f"SELECT * FROM `{self.table_info['name']}` WHERE _id='{row_id}'" - sql_rows, _ = self.query(sql, convert=True) - if not sql_rows: - self._convert_sql_query_count += 1 - logger.warning('auto-rule %s query dtable %s table %s convert row %s not found, query count %s', self.rule_id, self.dtable_uuid, self.table_id, row_id, self._convert_sql_query_count) - time.sleep(0.1) - continue - self._convert_sql_row = sql_rows[0] + self._convert_sql_row = convert_row_with_sql_row(self.get_sql_row(), self.table_info['columns'], self.db_session) return self._convert_sql_row def get_sql_row(self): diff --git a/dtable_events/notification_rules/notification_rules_utils.py b/dtable_events/notification_rules/notification_rules_utils.py index 471a8907..98e4636a 100644 --- a/dtable_events/notification_rules/notification_rules_utils.py +++ b/dtable_events/notification_rules/notification_rules_utils.py @@ -17,7 +17,7 @@ from dtable_events.utils.dtable_server_api import DTableServerAPI from dtable_events.utils.dtable_web_api import DTableWebAPI from dtable_events.utils.dtable_db_api import DTableDBAPI -from dtable_events.notification_rules.message_formatters import create_formatter_params, formatter_map +from dtable_events.utils.message_formatters import create_formatter_params, formatter_map logger = logging.getLogger(__name__) @@ -187,24 +187,29 @@ def convert_zero_in_value(value): return value +def fill_cell_with_sql_row(value, column, db_session, convert_to_html=False): + column_type = column['type'] + formatter_class = formatter_map.get(column_type) + if not formatter_class: + return None + params = create_formatter_params(column_type, value=value, db_session=db_session, convert_to_html=convert_to_html) + if value is None: + message = formatter_class(column).format_empty_message() + return message + try: + message = formatter_class(column).format_message(**params) + return message + except Exception as e: + logger.exception('value %s for filling column %s error %s', value, column) + return '' + + def fill_msg_blanks_with_sql_row(msg, column_blanks, col_name_dict, row, db_session, convert_to_html=False): for blank in column_blanks: - value = row.get(col_name_dict[blank]['key']) - column_type = col_name_dict[blank]['type'] - formatter_class = formatter_map.get(column_type) - if not formatter_class: - continue - params = create_formatter_params(column_type, value=value, db_session=db_session, convert_to_html=convert_to_html) - if value is None: - message = formatter_class(col_name_dict[blank]).format_empty_message() - msg = msg.replace('{' + blank + '}', str(message)) + cell_message = fill_cell_with_sql_row(row.get(col_name_dict[blank]['key']), col_name_dict[blank], db_session, convert_to_html=False) + if cell_message is None: continue - try: - message = formatter_class(col_name_dict[blank]).format_message(**params) - msg = msg.replace('{' + blank + '}', str(message)) - except Exception as e: - logger.exception(e) - msg = msg.replace('{' + blank + '}', '') + msg = msg.replace('{' + blank + '}', str(cell_message)) return msg diff --git a/dtable_events/notification_rules/message_formatters.py b/dtable_events/utils/message_formatters.py similarity index 99% rename from dtable_events/notification_rules/message_formatters.py rename to dtable_events/utils/message_formatters.py index e1a45a69..137af27d 100644 --- a/dtable_events/notification_rules/message_formatters.py +++ b/dtable_events/utils/message_formatters.py @@ -288,7 +288,7 @@ def format_message(self, value): except Exception as e: logger.warning('parse value: %s to datetime error: %s', value, e) return self.format_empty_message() - value = datetime_obj.strftime('%Y-%m-%d %H:%M') + value = datetime_obj.strftime('%Y-%m-%d %H:%M:%S') return value @@ -436,11 +436,11 @@ def format_message(self, value): minutes = (abs(value) % 3600) // 60 seconds = abs(value) % 60 if duration_format == 'h:mm': - value = '%s%d:%2d' % (prefix, hours, minutes) + value = '%s%d:%02d' % (prefix, hours, minutes) elif duration_format == 'h:mm:ss': value = '%s%d:%02d:%02d' % (prefix, hours, minutes, seconds) else: - value = '%s%d:%2d' % (prefix, hours, minutes) + value = '%s%d:%02d' % (prefix, hours, minutes) return value diff --git a/dtable_events/utils/row_converter.py b/dtable_events/utils/row_converter.py new file mode 100644 index 00000000..7717c923 --- /dev/null +++ b/dtable_events/utils/row_converter.py @@ -0,0 +1,153 @@ +import logging +import re +from datetime import datetime + +from dateutil import parser + +from dtable_events.utils.constants import FormulaResultType, ColumnTypes +from dtable_events.notification_rules.notification_rules_utils import fill_cell_with_sql_row +from dtable_events.utils.message_formatters import FormulaMessageFormatter + +logger = logging.getLogger(__name__) + + +class BaseCellConverter: + + def __init__(self, column): + self.column = column + + def get_column_data(self): + return self.column.get('data') or {} + + def convert(self, value): + return value + + +class SingleSelectCellConverter(BaseCellConverter): + + def convert(self, value): + column_data = self.get_column_data() + options = column_data.get('options') or [] + option = next(filter(lambda option: option.get('id') == value, options), None) + return option.get('name') or None + + +class MultiSelectCellConverter(BaseCellConverter): + + def convert(self, value): + column_data = self.get_column_data() + options = column_data.get('options') or [] + result = [] + if not isinstance(value, list): + return '' + for item in value: + item_option = next(filter(lambda option: option.get('id') == item, options), None) + if item_option: + result.append(item_option['name']) + return result + + +class LongTextCellConverter(BaseCellConverter): + + def convert(self, value): + if isinstance(value, str): + return value + elif isinstance(value, dict): + return value.get('text') or '' + return '' + + +class DateCellConverter(BaseCellConverter): + + def format_text_to_date(self, text, format='YYYY-MM-DD'): + if not isinstance(text, str): + return None + is_all_number = re.match(r'^\d+$', text) + if is_all_number: + date_obj = datetime.fromtimestamp(int(text) / 1000) + else: + date_obj = parser.parse(text) + if 'HH:mm' not in format: + return datetime.strftime(date_obj, '%Y-%m-%d') + return datetime.strftime(date_obj, '%Y-%m-%d %H:%M') + + def convert(self, value): + value = re.sub(r'([+-]\d{2}:?\d{2})|Z$', '', value) + column_data = self.get_column_data() + format = column_data.get('format') or 'YYYY-MM-DD' + return self.format_text_to_date(value, format) + + +class FormulaCellConverter(BaseCellConverter): + + def convert(self, value, db_session): + column_data = self.get_column_data() + if not column_data: + return None + result_type = column_data.get('result_type') + if result_type in [ + FormulaResultType.NUMBER, + FormulaResultType.DATE, + ]: + return FormulaMessageFormatter(self.column).format_message(value, None) + if isinstance(value, bool): + return 'true' if bool else 'false' + if result_type == FormulaResultType.ARRAY: + array_type = column_data.get('array_type') + array_data = column_data.get('array_data') + if not array_type: + return '' + if array_type in [ + ColumnTypes.COLLABORATOR, + ColumnTypes.CREATOR, + ColumnTypes.LAST_MODIFIER + ]: + return value + if array_type not in [ + ColumnTypes.IMAGE, + ColumnTypes.FILE, + ColumnTypes.MULTIPLE_SELECT, + ColumnTypes.COLLABORATOR + ] and isinstance(value, list): + return FormulaMessageFormatter(self.column).format_message(value, db_session) + # if find bugs in fill_cell_with_sql_row, do not modify message-formatter, fix in this file + return fill_cell_with_sql_row(value, {'type': array_type, 'data': array_data}, db_session) + return value + + +def convert_row_with_sql_row(row, columns, db_session): + result = {} + if '_id' in row: + result['_id'] = row['_id'] + if '_ctime' in row: + result['_ctime'] = row['_ctime'] + if '_mtime' in row: + result['_mtime'] = row['_mtime'] + for column in columns: + value = row.get(column['key']) + if not value: + continue + column_name = column['name'] + if not column: + continue + try: + if column['type'] == ColumnTypes.SINGLE_SELECT: + result[column_name] = SingleSelectCellConverter(column).convert(value) + elif column['type'] == ColumnTypes.MULTIPLE_SELECT: + result[column_name] = MultiSelectCellConverter(column).convert(value) + elif column['type'] == ColumnTypes.DATE: + result[column_name] = DateCellConverter(column).convert(value) + elif column['type'] == ColumnTypes.LONG_TEXT: + result[column_name] = LongTextCellConverter(column).convert(value) + elif column['type'] in [ + ColumnTypes.FORMULA, + ColumnTypes.LINK_FORMULA + ]: + result[column_name] = FormulaCellConverter(column).convert(value, db_session) + else: + result[column_name] = BaseCellConverter(column).convert(value) + except Exception as e: + logger.warning('convert row column %s with value %s', column, value) + result[column_name] = value + + return result