From d1671cdbea09c1e46f3c4b9dc805cba50c48950a Mon Sep 17 00:00:00 2001 From: Rudolf Cardinal Date: Thu, 6 Mar 2025 09:01:49 +0000 Subject: [PATCH 1/3] Fix datetime conversion for spreadsheet export --- cardinal_pythonlib/excel.py | 44 ++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/cardinal_pythonlib/excel.py b/cardinal_pythonlib/excel.py index 9762880..53fcd49 100644 --- a/cardinal_pythonlib/excel.py +++ b/cardinal_pythonlib/excel.py @@ -26,6 +26,10 @@ """ +# ============================================================================= +# Imports +# ============================================================================= + import datetime import io from typing import Any @@ -36,7 +40,18 @@ from pendulum.datetime import DateTime from semantic_version import Version -from cardinal_pythonlib.datetimefunc import pendulum_to_datetime + +# ============================================================================= +# Constants +# ============================================================================= + +# ISO 8601, e.g. 2013-07-24T20:04:07+0100) +ISO8601_STRFTIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z" + + +# ============================================================================= +# Conversion functions +# ============================================================================= def excel_to_bytes(wb: Workbook) -> bytes: @@ -63,9 +78,28 @@ def convert_for_openpyxl(x: Any) -> Any: Returns: the same thing, or a more suitable value! + + 2025-03-06 update: We were doing this: + if isinstance(x, DateTime): + return pendulum_to_datetime(x) + However, conversion of pendulum.datetime.Datetime to datetime.datetime is + insufficient, because you can still end up with this error from + openpyxl/utils/datetime.py, line 97, in to_excel: + days = (dt - epoch).days + TypeError: can't subtract offset-naive and offset-aware datetimes + The "epoch" variable does NOT have a timezone attribute. So we need to + ensure that what we produce here doesn't, either. In principle, there are + three alternatives: (a) convert to a standard timezone (UTC), making things + slightly and silently unhappier for those working outside UTC; (b) strip + timezone information, causing errors if datetime values are subtracted; or + (c) convert to a standard textual representation, including timezone + information, preserving all data but letting the user sort out the meaning. + Since ``convert_for_pyexcel_ods3`` was already converting + pendulum.datetime.DateTime and datetime.datetime values to a standard + string, via strftime, let's do that too. """ - if isinstance(x, DateTime): - return pendulum_to_datetime(x) + if isinstance(x, (DateTime, datetime.datetime)): + return x.strftime(ISO8601_STRFTIME_FORMAT) elif isinstance(x, (Version, uuid.UUID)): return str(x) else: @@ -90,10 +124,8 @@ def convert_for_pyexcel_ods3(x: Any) -> Any: Returns: the same thing, or a more suitable value! """ - if isinstance(x, (DateTime, datetime.datetime)): - # ISO 8601, e.g. 2013-07-24T20:04:07+0100) - return x.strftime("%Y-%m-%dT%H:%M:%S%z") + return x.strftime(ISO8601_STRFTIME_FORMAT) elif x is None: return "" elif isinstance(x, (Version, uuid.UUID)): From d79a4f94df52e8bbbaca4f94180ee6471195c8cb Mon Sep 17 00:00:00 2001 From: Rudolf Cardinal Date: Thu, 6 Mar 2025 09:29:32 +0000 Subject: [PATCH 2/3] Fix spreadsheet export --- cardinal_pythonlib/excel.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/cardinal_pythonlib/excel.py b/cardinal_pythonlib/excel.py index 53fcd49..efbf7d3 100644 --- a/cardinal_pythonlib/excel.py +++ b/cardinal_pythonlib/excel.py @@ -83,8 +83,8 @@ def convert_for_openpyxl(x: Any) -> Any: if isinstance(x, DateTime): return pendulum_to_datetime(x) However, conversion of pendulum.datetime.Datetime to datetime.datetime is - insufficient, because you can still end up with this error from - openpyxl/utils/datetime.py, line 97, in to_excel: + insufficient, because with openpyxl==3.0.7 you can still end up with this + error from openpyxl/utils/datetime.py, line 97, in to_excel: days = (dt - epoch).days TypeError: can't subtract offset-naive and offset-aware datetimes The "epoch" variable does NOT have a timezone attribute. So we need to @@ -96,7 +96,9 @@ def convert_for_openpyxl(x: Any) -> Any: information, preserving all data but letting the user sort out the meaning. Since ``convert_for_pyexcel_ods3`` was already converting pendulum.datetime.DateTime and datetime.datetime values to a standard - string, via strftime, let's do that too. + string, via strftime, let's do that too. Note that this also anticipates + the deprecation of timezone-aware dates from openpyxl==3.0.7 + (https://foss.heptapod.net/openpyxl/openpyxl/-/issues/1645). """ if isinstance(x, (DateTime, datetime.datetime)): return x.strftime(ISO8601_STRFTIME_FORMAT) @@ -117,12 +119,19 @@ def convert_for_pyexcel_ods3(x: Any) -> Any: - ``None`` - :class:`numpy.float64` - :class:`uuid.UUID` + - subclasses of `str` Args: x: a data value Returns: the same thing, or a more suitable value! + + 2025-03-06 update: With pyexcel-ods3==0.6.0, we were getting a KeyError + from pyexcel_ods3/odsw.py, in ODSSheetWriter.write_row. It does this: + value_type = service.ODS_WRITE_FORMAT_COVERSION[type(cell)] + and we had a cell that looked like 'aq' but had the type , a subclass of str. """ if isinstance(x, (DateTime, datetime.datetime)): return x.strftime(ISO8601_STRFTIME_FORMAT) @@ -132,5 +141,7 @@ def convert_for_pyexcel_ods3(x: Any) -> Any: return str(x) elif isinstance(x, float64): return float(x) + elif isinstance(x, str): + return str(x) else: return x From 9bf4330a42af863cfb2e6dc7193ec38743be0466 Mon Sep 17 00:00:00 2001 From: Rudolf Cardinal Date: Thu, 6 Mar 2025 09:52:23 +0000 Subject: [PATCH 3/3] fix docstrings --- cardinal_pythonlib/excel.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/cardinal_pythonlib/excel.py b/cardinal_pythonlib/excel.py index efbf7d3..45c489c 100644 --- a/cardinal_pythonlib/excel.py +++ b/cardinal_pythonlib/excel.py @@ -80,13 +80,21 @@ def convert_for_openpyxl(x: Any) -> Any: the same thing, or a more suitable value! 2025-03-06 update: We were doing this: + + .. code-block:: python + if isinstance(x, DateTime): return pendulum_to_datetime(x) + However, conversion of pendulum.datetime.Datetime to datetime.datetime is insufficient, because with openpyxl==3.0.7 you can still end up with this error from openpyxl/utils/datetime.py, line 97, in to_excel: - days = (dt - epoch).days - TypeError: can't subtract offset-naive and offset-aware datetimes + + .. code-block:: python + + days = (dt - epoch).days + # TypeError: can't subtract offset-naive and offset-aware datetimes + The "epoch" variable does NOT have a timezone attribute. So we need to ensure that what we produce here doesn't, either. In principle, there are three alternatives: (a) convert to a standard timezone (UTC), making things @@ -129,7 +137,11 @@ def convert_for_pyexcel_ods3(x: Any) -> Any: 2025-03-06 update: With pyexcel-ods3==0.6.0, we were getting a KeyError from pyexcel_ods3/odsw.py, in ODSSheetWriter.write_row. It does this: + + .. code-block:: python + value_type = service.ODS_WRITE_FORMAT_COVERSION[type(cell)] + and we had a cell that looked like 'aq' but had the type , a subclass of str. """