diff --git a/target_postgres/postgres.py b/target_postgres/postgres.py index 97d9ccf0..223c0d55 100644 --- a/target_postgres/postgres.py +++ b/target_postgres/postgres.py @@ -1,5 +1,6 @@ from copy import deepcopy import csv +from functools import lru_cache import io import json import logging @@ -19,6 +20,14 @@ RESERVED_NULL_DEFAULT = 'NULL' +@lru_cache(maxsize=128) +def _format_datetime(value): + """ + Format a datetime value. This is only called from the + PostgresTarget.serialize_table_record_datetime_value + but this non-method version allows caching + """ + return arrow.get(value).format('YYYY-MM-DD HH:mm:ss.SSSSZZ') def _update_schema_0_to_1(table_metadata, table_schema): """ @@ -546,7 +555,7 @@ def serialize_table_record_null_value(self, remote_schema, streamed_schema, fiel return value def serialize_table_record_datetime_value(self, remote_schema, streamed_schema, field, value): - return arrow.get(value).format('YYYY-MM-DD HH:mm:ss.SSSSZZ') + return _format_datetime(value) def persist_csv_rows(self, cur, diff --git a/target_postgres/sql_base.py b/target_postgres/sql_base.py index 9d879526..2064005d 100644 --- a/target_postgres/sql_base.py +++ b/target_postgres/sql_base.py @@ -13,6 +13,8 @@ # from copy import deepcopy +from functools import lru_cache, partial +import pickle import time import singer @@ -628,16 +630,23 @@ def log_message(msg): return self._get_table_schema(connection, table_name) - def _serialize_table_record_field_name(self, remote_schema, path, value_json_schema): + def _serialize_table_record_field_name(self, remote_schema, path, value_json_schema_tuple): """ Returns the appropriate remote field (column) name for `path`. :param remote_schema: TABLE_SCHEMA(remote) :param path: (string, ...) - :value_json_schema: dict, JSON Schema + :value_json_schema: tuple, JSON Schema :return: string """ + # rebuild the dict that needs to be passed further down the call stack + if len(value_json_schema_tuple) == 1: + value_json_schema = { 'type': value_json_schema_tuple[0] } + else: + value_json_schema = {'type': value_json_schema_tuple[0], + 'format': value_json_schema_tuple[1]} + simple_json_schema = json_schema.simple_type(value_json_schema) mapping = self._get_mapping(remote_schema, @@ -725,9 +734,15 @@ def _serialize_table_records( default_row = dict([(field, NULL_DEFAULT) for field in remote_fields]) paths = streamed_schema['schema']['properties'].keys() + + ## create a partial function with only hashable args so we can use lru_cache on it + _cached_field_name = partial(self._serialize_table_record_field_name, remote_schema) + cached_field_name = lru_cache(maxsize=None)(_cached_field_name) + for record in records: - row = deepcopy(default_row) + ## pickling/unpickling is much faster than deepcopy + row = pickle.loads(pickle.dumps(default_row)) for path in paths: json_schema_string_type, value = record.get(path, (None, None)) @@ -747,18 +762,14 @@ def _serialize_table_records( and value is not None: value = self.serialize_table_record_datetime_value(remote_schema, streamed_schema, path, value) - value_json_schema = {'type': json_schema.STRING, - 'format': json_schema.DATE_TIME_FORMAT} + value_json_schema_tuple = (json_schema.STRING, json_schema.DATE_TIME_FORMAT) + field_name = cached_field_name(path, value_json_schema_tuple) else: - value_json_schema = {'type': json_schema_string_type} + field_name = cached_field_name(path, (json_schema_string_type,)) ## Serialize NULL default value value = self.serialize_table_record_null_value(remote_schema, streamed_schema, path, value) - field_name = self._serialize_table_record_field_name(remote_schema, - path, - value_json_schema) - ## `field_name` is unset if row[field_name] == NULL_DEFAULT: row[field_name] = value