From b5fff8a2058a715df38168bbbf4176337d8f4a4e Mon Sep 17 00:00:00 2001 From: Ryan-McClanahan Date: Tue, 2 Dec 2025 13:44:23 -0800 Subject: [PATCH 1/6] Added val_as_str option to the data insight tool --- trace/widgets/data_insight_tool.py | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/trace/widgets/data_insight_tool.py b/trace/widgets/data_insight_tool.py index 4b22b0b3..afb7a685 100644 --- a/trace/widgets/data_insight_tool.py +++ b/trace/widgets/data_insight_tool.py @@ -29,6 +29,7 @@ QMessageBox, QPushButton, QVBoxLayout, + QCheckBox, ) from pydm.widgets.archiver_time_plot import ( @@ -100,6 +101,7 @@ def __init__(self, parent: QObject = None) -> None: self.unit = None self.description = None self.caget_thread = None + self._val_as_str = False self.network_manager = QNetworkAccessManager() self.network_manager.finished.connect(self.recieve_archive_reply) @@ -122,6 +124,9 @@ def data(self, index: QModelIndex, role: Qt.ItemDataRole = Qt.DisplayRole) -> st return None elif role == Qt.DisplayRole: val = self.df.iat[index.row(), index.column()] + if index.column() == 1 and self.val_as_str: + characters = map(chr, list(val)) + val = "".join(characters) return str(val) return None @@ -348,6 +353,20 @@ def export_data(self, file_path: Path, extension: str) -> None: with file_path.open("w") as file: json.dump(export_dict, file, indent=2) + @property + def val_as_str(self) -> bool: + """weather or not to show the value column as a string or raw data""" + return self._val_as_str + + @val_as_str.setter + def val_as_str(self, val_as_str:bool): + """set the value column to display as string or raw data""" + if val_as_str != self._val_as_str: + self._val_as_str = val_as_str + start = self.index(0,1) + end = self.index(self.rowCount(),1) + self.dataChanged.emit(start, end) + class DataInsightTool(QWidget): """The Data Insight Tool is a standalone widget that allows users to display @@ -408,6 +427,11 @@ def layout_init(self) -> None: self.meta_data_label = QLabel() self.metadata_layout.addWidget(self.meta_data_label, alignment=Qt.AlignLeft) + self.val_as_str_checkbox = QCheckBox() + self.val_as_str_checkbox.setText("Value As String") + self.metadata_layout.addWidget(self.val_as_str_checkbox) + self.val_as_str_checkbox.toggled.connect(self.set_val_as_str) + self.refresh_button = QPushButton("Refresh Data") self.metadata_layout.addWidget(self.refresh_button, alignment=Qt.AlignRight) self.main_layout.addLayout(self.metadata_layout) @@ -445,6 +469,15 @@ def combobox_to_curve(self, combobox_ind: int) -> ArchivePlotCurveItem: if combobox_ind < 0 or self.pv_select_box.count() <= combobox_ind: combobox_ind = self.pv_select_box.currentIndex() return self.plot.curveAtIndex(combobox_ind) + + def set_val_as_str(self) -> None: + """set the val_as_str flag on the self.data_vis_model based off of the self.val_as_str_checkbox + then emit the dataChanged signal from the data_vis_model for the value column""" + if self.val_as_str_checkbox.isChecked(): + self.data_vis_model.val_as_str = True + else: + self.data_vis_model.val_as_str = False + @Slot() def update_pv_select_box(self) -> None: From df1484c62b4ba32af9324fd7776586a6c247dd61 Mon Sep 17 00:00:00 2001 From: Ryan-McClanahan Date: Tue, 2 Dec 2025 15:16:28 -0800 Subject: [PATCH 2/6] when value as string is checked the export will export the values as strings --- trace/widgets/data_insight_tool.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/trace/widgets/data_insight_tool.py b/trace/widgets/data_insight_tool.py index afb7a685..ff48ea95 100644 --- a/trace/widgets/data_insight_tool.py +++ b/trace/widgets/data_insight_tool.py @@ -52,6 +52,22 @@ logger.setLevel("DEBUG") handler.setLevel("DEBUG") +def list_to_ascii(val:list[int])->str: + """""" + characters = map(chr, list(val)) + string = "".join(characters) + # string = string.encode("ascii", "ignore").decode("ascii") + string = string.replace('\u0000', "") + print(f"({string})") + return string + +def list_to_ascii_row(row): + """""" + row["Value"] = list_to_ascii(row["Value"]) + # row["Value"] = row["Value"].str.encode('ascii', 'ignore').str.decode('ascii') + print(f"({row["Value"]})") + return row + class CAGetThread(QThread): """Thread for making a CA get request to the given address. This is used @@ -125,8 +141,7 @@ def data(self, index: QModelIndex, role: Qt.ItemDataRole = Qt.DisplayRole) -> st elif role == Qt.DisplayRole: val = self.df.iat[index.row(), index.column()] if index.column() == 1 and self.val_as_str: - characters = map(chr, list(val)) - val = "".join(characters) + val = list_to_ascii(val) return str(val) return None @@ -334,9 +349,12 @@ def export_data(self, file_path: Path, extension: str) -> None: header_dict = {"Address": self.address, "Unit": self.unit, "Description": self.description} - export_df = self.df.copy() + export_df = self.df.copy(deep=True) export_df["Datetime"] = export_df["Datetime"].astype("int64") / 1e9 + if self.val_as_str: + export_df = export_df.apply(list_to_ascii_row, axis=1) + if extension == ".csv": file_header = "".join([f"{k}: {v}\n" for k, v in header_dict.items()]) with file_path.open("w") as file: @@ -385,6 +403,7 @@ def __init__(self, parent: QObject, plot: PyDMArchiverTimePlot = None) -> None: self.data_vis_model.reply_recieved.connect(self.loading_label.hide) self.data_vis_model.description_changed.connect(self.set_meta_data) self.export_button.clicked.connect(self.export_data_to_file) + self.val_as_str_checkbox.toggled.connect(self.set_val_as_str) self.pv_select_box.currentIndexChanged.connect(self.get_data) self.refresh_button.clicked.connect(self.get_data) @@ -430,7 +449,6 @@ def layout_init(self) -> None: self.val_as_str_checkbox = QCheckBox() self.val_as_str_checkbox.setText("Value As String") self.metadata_layout.addWidget(self.val_as_str_checkbox) - self.val_as_str_checkbox.toggled.connect(self.set_val_as_str) self.refresh_button = QPushButton("Refresh Data") self.metadata_layout.addWidget(self.refresh_button, alignment=Qt.AlignRight) From c4a050676e4afc9747a37260c1a2980fcd9384d0 Mon Sep 17 00:00:00 2001 From: Ryan-McClanahan Date: Tue, 2 Dec 2025 15:16:52 -0800 Subject: [PATCH 3/6] cleaned up --- trace/widgets/data_insight_tool.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/trace/widgets/data_insight_tool.py b/trace/widgets/data_insight_tool.py index ff48ea95..c950cb95 100644 --- a/trace/widgets/data_insight_tool.py +++ b/trace/widgets/data_insight_tool.py @@ -56,16 +56,12 @@ def list_to_ascii(val:list[int])->str: """""" characters = map(chr, list(val)) string = "".join(characters) - # string = string.encode("ascii", "ignore").decode("ascii") string = string.replace('\u0000', "") - print(f"({string})") return string def list_to_ascii_row(row): """""" row["Value"] = list_to_ascii(row["Value"]) - # row["Value"] = row["Value"].str.encode('ascii', 'ignore').str.decode('ascii') - print(f"({row["Value"]})") return row From 2c46da10d35d99eddc91c58ebd6a252873f14d4c Mon Sep 17 00:00:00 2001 From: Ryan-McClanahan Date: Tue, 2 Dec 2025 15:19:26 -0800 Subject: [PATCH 4/6] docstrings --- trace/widgets/data_insight_tool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trace/widgets/data_insight_tool.py b/trace/widgets/data_insight_tool.py index c950cb95..18a225a4 100644 --- a/trace/widgets/data_insight_tool.py +++ b/trace/widgets/data_insight_tool.py @@ -53,14 +53,14 @@ handler.setLevel("DEBUG") def list_to_ascii(val:list[int])->str: - """""" + """converts a list of integers into an ascii string""" characters = map(chr, list(val)) string = "".join(characters) string = string.replace('\u0000', "") return string def list_to_ascii_row(row): - """""" + """converts the Value column of a dataframe row from list to ascii string""" row["Value"] = list_to_ascii(row["Value"]) return row From 1a2eb02c2ca587563d2e144f63b0b653d131c5a1 Mon Sep 17 00:00:00 2001 From: Ryan-McClanahan Date: Tue, 2 Dec 2025 15:21:07 -0800 Subject: [PATCH 5/6] formatting --- trace/widgets/data_insight_tool.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/trace/widgets/data_insight_tool.py b/trace/widgets/data_insight_tool.py index 18a225a4..218eb23e 100644 --- a/trace/widgets/data_insight_tool.py +++ b/trace/widgets/data_insight_tool.py @@ -23,13 +23,13 @@ from qtpy.QtWidgets import ( QLabel, QWidget, + QCheckBox, QComboBox, QFileDialog, QHBoxLayout, QMessageBox, QPushButton, QVBoxLayout, - QCheckBox, ) from pydm.widgets.archiver_time_plot import ( @@ -52,13 +52,15 @@ logger.setLevel("DEBUG") handler.setLevel("DEBUG") -def list_to_ascii(val:list[int])->str: + +def list_to_ascii(val: list[int]) -> str: """converts a list of integers into an ascii string""" characters = map(chr, list(val)) string = "".join(characters) - string = string.replace('\u0000', "") + string = string.replace("\u0000", "") return string + def list_to_ascii_row(row): """converts the Value column of a dataframe row from list to ascii string""" row["Value"] = list_to_ascii(row["Value"]) @@ -371,14 +373,14 @@ def export_data(self, file_path: Path, extension: str) -> None: def val_as_str(self) -> bool: """weather or not to show the value column as a string or raw data""" return self._val_as_str - + @val_as_str.setter - def val_as_str(self, val_as_str:bool): + def val_as_str(self, val_as_str: bool): """set the value column to display as string or raw data""" if val_as_str != self._val_as_str: self._val_as_str = val_as_str - start = self.index(0,1) - end = self.index(self.rowCount(),1) + start = self.index(0, 1) + end = self.index(self.rowCount(), 1) self.dataChanged.emit(start, end) @@ -483,7 +485,7 @@ def combobox_to_curve(self, combobox_ind: int) -> ArchivePlotCurveItem: if combobox_ind < 0 or self.pv_select_box.count() <= combobox_ind: combobox_ind = self.pv_select_box.currentIndex() return self.plot.curveAtIndex(combobox_ind) - + def set_val_as_str(self) -> None: """set the val_as_str flag on the self.data_vis_model based off of the self.val_as_str_checkbox then emit the dataChanged signal from the data_vis_model for the value column""" @@ -491,7 +493,6 @@ def set_val_as_str(self) -> None: self.data_vis_model.val_as_str = True else: self.data_vis_model.val_as_str = False - @Slot() def update_pv_select_box(self) -> None: From a29ec20ac349fcc36501b0a4c1b93a4977b9b636 Mon Sep 17 00:00:00 2001 From: Zach Domke Date: Thu, 19 Mar 2026 10:01:26 -0700 Subject: [PATCH 6/6] ENH: Avoid trying to convert non-list values to str --- trace/widgets/data_insight_tool.py | 90 ++++++++++++++++-------------- 1 file changed, 49 insertions(+), 41 deletions(-) diff --git a/trace/widgets/data_insight_tool.py b/trace/widgets/data_insight_tool.py index c3afa4f1..5b9fae23 100644 --- a/trace/widgets/data_insight_tool.py +++ b/trace/widgets/data_insight_tool.py @@ -53,20 +53,6 @@ handler.setLevel("DEBUG") -def list_to_ascii(val: list[int]) -> str: - """converts a list of integers into an ascii string""" - characters = map(chr, list(val)) - string = "".join(characters) - string = string.replace("\u0000", "") - return string - - -def list_to_ascii_row(row): - """converts the Value column of a dataframe row from list to ascii string""" - row["Value"] = list_to_ascii(row["Value"]) - return row - - class CAGetThread(QThread): """Thread for making a CA get request to the given address. This is used to get the description of the curve. @@ -116,7 +102,7 @@ def __init__(self, parent: QObject = None) -> None: self.unit = None self.description = None self.caget_thread = None - self._val_as_str = False + self._decode_as_string = False self.network_manager = QNetworkAccessManager() self.network_manager.finished.connect(self.recieve_archive_reply) @@ -139,8 +125,8 @@ def data(self, index: QModelIndex, role: Qt.ItemDataRole = Qt.DisplayRole) -> st return None elif role == Qt.DisplayRole: val = self.df.iat[index.row(), index.column()] - if index.column() == 1 and self.val_as_str: - val = list_to_ascii(val) + if index.column() == 1 and self.decode_as_string: + val = self.list_to_ascii(val) return str(val) return None @@ -149,6 +135,20 @@ def headerData(self, section: int, orientation: Qt.Orientation, role: Qt.ItemDat if orientation == Qt.Horizontal and role == Qt.DisplayRole: return self.df.columns[section] + @property + def decode_as_string(self) -> bool: + """weather or not to show the value column as a string or raw data""" + return self._decode_as_string + + @decode_as_string.setter + def decode_as_string(self, decode_as_string: bool): + """set the value column to display as string or raw data""" + if decode_as_string != self._decode_as_string: + self._decode_as_string = decode_as_string + start = self.index(0, 1) + end = self.index(self.rowCount(), 1) + self.dataChanged.emit(start, end) + def set_description(self, description: str) -> None: """Set the description of the curve. This is called when the CAGetThread emits a result_ready signal. @@ -285,7 +285,6 @@ def recieve_archive_reply(self, reply: QNetworkReply) -> None: reply : QNetworkReply Reply to the network request made in request_archive_data """ - self.reply_recieved.emit() if reply.error() == QNetworkReply.NoError: bytes_str = reply.readAll() try: @@ -298,6 +297,7 @@ def recieve_archive_reply(self, reply: QNetworkReply) -> None: f"Request for data from archiver failed, request url: {reply.url()} retrieved header: " f"{reply.header(QNetworkRequest.ContentTypeHeader)} error: {reply.error()}" ) + self.reply_recieved.emit() reply.deleteLater() def set_archive_data(self, data_dict: dict) -> None: @@ -354,8 +354,8 @@ def export_data(self, file_path: Path, extension: str) -> None: export_df = self.df.copy(deep=True) export_df["Datetime"] = export_df["Datetime"].astype("int64") / 1e9 - if self.val_as_str: - export_df = export_df.apply(list_to_ascii_row, axis=1) + if self.decode_as_string: + export_df["Value"] = export_df["Value"].apply(self.list_to_ascii) if extension == ".csv": file_header = "".join([f"{k}: {v}\n" for k, v in header_dict.items()]) @@ -373,19 +373,19 @@ def export_data(self, file_path: Path, extension: str) -> None: with file_path.open("w") as file: json.dump(export_dict, file, indent=2) - @property - def val_as_str(self) -> bool: - """weather or not to show the value column as a string or raw data""" - return self._val_as_str + def has_waveform_data(self) -> bool: + """Return True if any value in the Value column is a list or numpy array""" + return bool(self.df["Value"].apply(lambda v: isinstance(v, (list, np.ndarray))).any()) - @val_as_str.setter - def val_as_str(self, val_as_str: bool): - """set the value column to display as string or raw data""" - if val_as_str != self._val_as_str: - self._val_as_str = val_as_str - start = self.index(0, 1) - end = self.index(self.rowCount(), 1) - self.dataChanged.emit(start, end) + @staticmethod + def list_to_ascii(val: list[int]) -> str: + """Convert a list of integers into an ASCII string, ignoring null characters""" + if not isinstance(val, (list, np.ndarray)): + return str(val) + characters = map(chr, list(val)) + string = "".join(characters) + string = string.replace("\u0000", "") + return string class DataInsightTool(QWidget): @@ -405,9 +405,10 @@ def __init__(self, parent: QObject, plot: PyDMArchiverTimePlot = None) -> None: self.unopened = True self.data_vis_model.reply_recieved.connect(self.loading_label.hide) + self.data_vis_model.reply_recieved.connect(self.update_decode_as_string_visibility) self.data_vis_model.description_changed.connect(self.set_meta_data) self.export_button.clicked.connect(self.export_data_to_file) - self.val_as_str_checkbox.toggled.connect(self.set_val_as_str) + self.decode_as_string_checkbox.toggled.connect(self.set_decode_as_string) self.pv_select_box.currentIndexChanged.connect(self.get_data) self.refresh_button.clicked.connect(self.get_data) @@ -450,9 +451,10 @@ def layout_init(self) -> None: self.meta_data_label = QLabel() self.metadata_layout.addWidget(self.meta_data_label, alignment=Qt.AlignLeft) - self.val_as_str_checkbox = QCheckBox() - self.val_as_str_checkbox.setText("Value As String") - self.metadata_layout.addWidget(self.val_as_str_checkbox) + self.decode_as_string_checkbox = QCheckBox() + self.decode_as_string_checkbox.setText("Decode As String") + self.decode_as_string_checkbox.hide() + self.metadata_layout.addWidget(self.decode_as_string_checkbox) self.refresh_button = QPushButton("Refresh Data") self.metadata_layout.addWidget(self.refresh_button, alignment=Qt.AlignRight) @@ -498,13 +500,18 @@ def combobox_to_curve(self, combobox_ind: int) -> ArchivePlotCurveItem: combobox_ind = self.pv_select_box.currentIndex() return self.plot.curveAtIndex(combobox_ind) - def set_val_as_str(self) -> None: - """set the val_as_str flag on the self.data_vis_model based off of the self.val_as_str_checkbox + def set_decode_as_string(self) -> None: + """set the decode_as_string flag on the self.data_vis_model based off of the self.decode_as_string_checkbox then emit the dataChanged signal from the data_vis_model for the value column""" - if self.val_as_str_checkbox.isChecked(): - self.data_vis_model.val_as_str = True + if self.decode_as_string_checkbox.isChecked(): + self.data_vis_model.decode_as_string = True else: - self.data_vis_model.val_as_str = False + self.data_vis_model.decode_as_string = False + + @Slot() + def update_decode_as_string_visibility(self) -> None: + """Show the decode_as_string_checkbox only when the model's Value column contains arrays""" + self.decode_as_string_checkbox.setVisible(self.data_vis_model.has_waveform_data()) @Slot() def update_pv_select_box(self) -> None: @@ -557,6 +564,7 @@ def get_data(self, combobox_index: int = -1) -> None: curve_item = self.combobox_to_curve(combobox_index) x_range = self.plot.getXAxis().range + self.decode_as_string_checkbox.hide() self.data_vis_model.set_all_data(curve_item, x_range) self.set_meta_data() self.loading_label.show()