From daca7de5431486269c915e04187dace0ffb1ffb0 Mon Sep 17 00:00:00 2001 From: Jasper-Harvey0 Date: Thu, 8 May 2025 12:44:50 +1000 Subject: [PATCH 1/3] First pass on new waveform acquire function --- src/fixate/drivers/dso/agilent_mso_x.py | 93 +++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/src/fixate/drivers/dso/agilent_mso_x.py b/src/fixate/drivers/dso/agilent_mso_x.py index 50971f2c..a0dddeaf 100644 --- a/src/fixate/drivers/dso/agilent_mso_x.py +++ b/src/fixate/drivers/dso/agilent_mso_x.py @@ -697,6 +697,99 @@ def retrieve_waveform_data(self): values = struct.unpack("%dB" % len(valid_bytes), valid_bytes) return values + def get_channel_data(self, channel): + """ + Get all channel data currently displayed on the screen. + Assumes the data is already captured on the scope, and acquisition is stopped. + + returns: (time_series, data) + time_series: A list of times in Seconds corresponding to each datapoint. + data: A list of the data on the screen. + + It is up to the user to know what units the data is in. + This differs from waveform_values() in that the waveform must already be acquired. This is + useful when capturing data via a trigger. + """ + data = [] + time_series = [] + + # Check the channel is actually on: + on = int(self.instrument.query(":CHANnel" + str(channel) + ":DISPlay?")) + if not on: + # Channel is off + raise ValueError("Requested channel is off.") + + # Check if there is actually data to acquire: + data_available = int( + self.instrument.query( + ":WAVeform:SOURce CHANnel" + str(channel) + ";POINTs?" + ) + ) + if data_available == 0: + # No data is available + # Setting a channel to be a waveform source turns it on, so we need to turn it off now: + self.instrument.write(":CHANnel" + str(channel) + ":DISPlay OFF") + raise ValueError("No data is available") + + # Now we know the channel is on and has some data in it. + # Get information about the waveform: + preamble = self.waveform_preamble() + + acq_type = int(preamble["acquire"]) # 0= NORMAL, 1 = PEAK DETECT, 2 = AVERAGE + x_inc = float(preamble["x_increment"]) # Time difference between data points. + x_origin = float(preamble["x_origin"]) # Always the first data point in memory + x_reference = float( + preamble["x_reference"] + ) # Specifies the data point associated with x-origin + y_inc = float( + preamble["y_increment"] + ) # Y INCrement, Voltage difference between data points + y_origin = float(preamble["y_origin"]) # Y ORIGin, Voltage at center screen + y_reference = float( + preamble["y_reference"] + ) # Y REFerence, Specifies the data point where y-origin occurs, always zero + + ## This can also be done when pulling pre-ambles (pre[1]) or may be known ahead of time, but since the script is supposed to find everything, it is done now. + if ( + acq_type == "AVER" or acq_type == "HRES" + ): # Don't need to check for both types of mnemonics like this: if ACQ_TYPE == "AVER" or ACQ_TYPE == "AVERage": becasue the scope ALWAYS returns the short form + points_mode = ( + "NORMal" # Use for Average and High Resoultion acquisition Types. + ) + ## If the :WAVeform:POINts:MODE is RAW, and the Acquisition Type is Average, the number of points available is 0. If :WAVeform:POINts:MODE is MAX, it may or may not return 0 points. + ## If the :WAVeform:POINts:MODE is RAW, and the Acquisition Type is High Resolution, then the effect is (mostly) the same as if the Acq. Type was Normal (no box-car averaging). + ## Note: if you use :SINGle to acquire the waveform in AVERage Acq. Type, no average is performed, and RAW works. + else: + points_mode = "RAW" # Use for Acq. Type NORMal or PEAK + + # Setup for data export: + # 16 bit word format... or BYTE for 8 bit format: + self.instrument.write(":WAVeform:FORMat WORD") + ## WORD format especially recommended for Average and High Res. Acq. Types, which can produce more than 8 bits of resolution. + # Explicitly set this to avoid confusion - only applies to WORD FORMat + self.instrument.write(":WAVeform:BYTeorder LSBFirst") + # Explicitly set this to avoid confusion + self.instrument.write(":WAVeform:UNSigned 0") + # This command sets the points mode to MAX AND ensures that the maximum # of points to be transferred is set, though they must still be on screen + self.instrument.write(":WAVeform:POINts MAX") + # Since the ":WAVeform:POINts MAX" command above also changes the :POINts:MODE to MAXimum, which may or may not be a good thing, so change it to what is needed next. + self.instrument.write(":WAVeform:POINts:MODE " + str(points_mode)) + + # Now get the data: + raw_data = self.instrument.query_binary_values( + ":WAVeform:SOURce CHANnel" + str(1) + ";DATA?", "h", False + ) + # Scale it based on the preamble data we got earlier: + data = [(((i - y_reference) * y_inc) + y_origin) for i in raw_data] + + # Create initial space for time data: + available_points = int(self.instrument.query_ascii_values(":WAV:POIN?")[0]) + time_series = [i for i in range(0, available_points)] + # Scale and shift as needed: + time_series = [(((i - x_reference) * x_inc) + x_origin) for i in time_series] + + return time_series, data + def digitize(self, signals): signals = [self.validate_signal(sig) for sig in signals] self.write(":DIG {}".format(",".join(signals))) From 293490dff9db99f2b535de35b2b9d1e26afcf2a6 Mon Sep 17 00:00:00 2001 From: Jasper-Harvey0 Date: Fri, 23 May 2025 12:51:38 +1000 Subject: [PATCH 2/3] Draft version of logging to json. --- src/fixate/reporting/json.py | 302 +++++++++++++++++++++++++++++++++++ src/fixate/sequencer.py | 4 +- 2 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 src/fixate/reporting/json.py diff --git a/src/fixate/reporting/json.py b/src/fixate/reporting/json.py new file mode 100644 index 00000000..87005c86 --- /dev/null +++ b/src/fixate/reporting/json.py @@ -0,0 +1,302 @@ +from datetime import datetime +import logging +import sys +import os +import time +import json + +from pydantic.dataclasses import dataclass, Field +from pydantic.json import pydantic_encoder +from pubsub import pub + +from queue import Queue +from fixate.core.common import TestClass +from fixate.core.checks import CheckResult +import fixate +import fixate.config + +logger = logging.getLogger() + +""" +Example of a log file: +Note that this is not currently exactly what the code outputs. It's more of a guide of what we want +to get to. + +{ + "serial_number" : "1234567890", + "start_time_millis": 1747868933064, + "end_time_millis": 1747868933064, + "outcome" : "FAIL", + "outcome_details" : [] + "module_name" : "test_module", + "part_number" : "123456", + "tests" : [ + { + "test_name": "TestCrowbars", + "test_list" : "MainTestList", + "measurements" : [ + { + "name": "measure r41 resistance", + "outcome": "PASS", + "validators": [ + "9 <= x <= 11" + ], + "units": { + "name": "ohm", + "code": "OHM", + "suffix": "\u2126" + }, + "measured_value": 9.989233575589664 + }, + { + "name": "measure r43 resistance", + "outcome": "FAIL", + "validators": [ + "x < 119" + ], + "units": { + "name": "ohm", + "code": "OHM", + "suffix": "\u2126" + }, + "measured_value": 900 + }, + { + "name": "log the value of something", + "outcome": "PASS", + "validators": [ ], + "units": { + "name": "string", + "code": "STRING", + "suffix": "" + }, + "measured_value": "On chk log, we might be able to dynamically populate the type information to put into the above fields" + }, + ], + "parameters": {"testclassargs": ["things","stuff"]} + "start_time_millis": 1747868933064, + "end_time_millis": 1747868933064, + } + ], + "instruments" : [ + { + "type" : "DMM", # We may need to add a type attribute to the drivers? + "name" : "FLUKE BLAH BLAH", + "serial" : "1234", + }, + { + "type" : "DSO", # We may need to add a type attribute to the drivers? + "name" : "Keysight BLAH BLAH", + "serial" : "1234", + } + ] +} + +""" + + +class TestClassImp(TestClass): + """ + Minimum implementation of the Test class so that it can be used for parameter extraction from the + actual implemented test classes + """ + + def test(self): + pass + + +""" +The log schema is defined by the following dataclasses +""" + + +@dataclass +class InstrumentLog: + """ + Log entry for the instruments + """ + + serial = "" + + +@dataclass +class TestLog: + """ + Log of data for a test list + """ + + measurements: list[CheckResult] = Field( + default_factory=list + ) # Class to store all the checks that get done in the testList + parent_test_list: str = "" # The parent test list that the test belongs to + test_name: str = "" # The name of the test class + description: str = "" # The test description + description_long: str = "" # The test description + test_index: str = "" + outcome: str = "" + args: list[str] = Field(default_factory=list) # args + start_time_millis: int = 0 + end_time_millis: int = 0 + + +@dataclass +class LogFile: + """ + Logfile for a test run + """ + + tests: list[TestLog] = Field(default_factory=list) + instruments: list[InstrumentLog] = Field(default_factory=list) + serial_number: str = "" + outcome: str = "" + start_time_millis: int = 0 + end_time_millis: int = 0 + part_number: int = "" + module_name: int = "" + + +class JSONWriter: + def __init__(self): + self.log_queue = Queue() + self.json_writer = None + + self.log_file_path = fixate.config.LOG_DIRECTORY + + self.log_path = "" + self.exception = None + + self._topics = [ + (self.test_start, "Test_Start"), + (self.test_comparison, "Check"), + (self.test_exception, "Test_Exception"), + (self.test_complete, "Test_Complete"), # Finish up the log file here + (self.sequence_update, "Sequence_Update"), + (self.sequence_complete, "Sequence_Complete"), + (self.user_wait_start, "UI_block_start"), + (self.user_wait_end, "UI_block_end"), + (self.driver_open, "driver_open"), # Log the instruments here + ] + + self.logFile = LogFile() + + def install(self): + + for callback, topic in self._topics: + pub.subscribe(callback, topic) + + def uninstall(self): + for callback, topic in self._topics: + pub.unsubscribe(callback, topic) + + def ensure_alive(self): + pass + + def sequence_update(self, status): + logger.info("Sequence update") + if status in ["Running"]: + self.logFile.start_time_millis = time.perf_counter() + # Get the module name and test name etc + test_module = sys.modules["module.loaded_tests"] + module_name = os.path.basename(test_module.__file__).split(".")[0] + self.logFile.module_name = module_name + logger.info(f"module = {module_name}") + + # Get the serial number + serial = fixate.config.RESOURCES["SEQUENCER"].context_data["serial_number"] + logger.info(f"serial = {serial}") + self.logFile.serial_number = serial + + # The part number is not necessarily in the context data yet... + + def sequence_complete( + self, status, passed, failed, error, skipped, sequence_status + ): + + self.logFile.end_time_millis = time.perf_counter() + self.logFile.outcome = status + logger.info(f"stoptime = {self.logFile.end_time_millis}") + self.save_file() + + def test_start(self, data, test_index): + """ + :param data: + the test class that is being started + :param test_index: + the test index in the sequencer + """ + logger.info("Test Start") + logger.info(data) + logger.info(test_index) + + new_test_log = TestLog() + new_test_log.description = data.test_desc + new_test_log.description_long = data.test_desc_long + new_test_log.start_time_millis = time.perf_counter() + new_test_log.test_name = data.__class__.__name__ + new_test_log.test_index = test_index + new_test_log.args = self.extract_test_parameters(data) + self.logFile.tests.append(new_test_log) + + def test_exception(self, exception, test_index): + logger.info("test exception") + logger.info(exception) + logger.info(test_index) + + def test_comparison( + self, passes: bool, chk: CheckResult, chk_cnt: int, context: str + ): + logger.info("Test comparison") + logger.info(f"passes {passes}") + logger.info(f"results {chk}") + logger.info(f"count {chk_cnt}") + logger.info(f"context {context}") + + self.logFile.tests[-1].measurements.append(chk) + + def test_complete(self, data, test_index, status): + logger.info("Test complete") + logger.info(data) + logger.info(test_index) + logger.info(status) + + self.logFile.tests[-1].outcome = status + self.logFile.tests[-1].end_time_millis = time.perf_counter() + + def user_wait_start(self, *args, **kwargs): + pass + + def user_wait_end(self, *args, **kwargs): + pass + + def driver_open(self, instr_type, identity): + logger.info("Driver Open") + logger.info(instr_type) + logger.info(identity) + + @staticmethod + def extract_test_parameters(test_cls): + """ + :param test_cls: + The class to extract parameters from + :return: + the keys and values in the form in alphabetical order on the parameter names and zipped as + [(param_name, param_value)] + """ + comp = TestClassImp() + keys = sorted(set(test_cls.__dict__) - set(comp.__dict__)) + return [(key, test_cls.__dict__[key]) for key in keys] + + def save_file(self): + """ + Dumps the logfile to a .json file + """ + now = datetime.now() + json_data = json.dumps(self.logFile, indent=4, default=pydantic_encoder) + with open( + os.path.join( + self.log_file_path, + "test_log_" + now.strftime("%Y%m%d_%H%M%S") + ".json", + ), + "w", + ) as f: + f.write(json_data) diff --git a/src/fixate/sequencer.py b/src/fixate/sequencer.py index 69d3db52..f258ff28 100644 --- a/src/fixate/sequencer.py +++ b/src/fixate/sequencer.py @@ -6,7 +6,7 @@ from fixate.core.exceptions import SequenceAbort, CheckFail from fixate.core.ui import user_retry_abort_fail from fixate.core.checks import CheckResult -from fixate.reporting import CSVWriter +from fixate.reporting import JSONWriter STATUS_STATES = ["Idle", "Running", "Paused", "Finished", "Restart", "Aborted"] @@ -110,7 +110,7 @@ def __init__(self): self.context = ContextStack() self.context_data = {} self.end_status = "N/A" - self.reporting_service = CSVWriter() + self.reporting_service = JSONWriter() # Sequencer behaviour. Don't ask the user when things to wrong, just marks tests as failed. # This does not change the behaviour of tests that call out to the user. They will still block as required. From fd44970f8ed7d75578638c98293c82f940b5db94 Mon Sep 17 00:00:00 2001 From: Jasper-Harvey0 Date: Fri, 23 May 2025 13:31:53 +1000 Subject: [PATCH 3/3] Revert "First pass on new waveform acquire function" This reverts commit daca7de5431486269c915e04187dace0ffb1ffb0. --- src/fixate/drivers/dso/agilent_mso_x.py | 93 ------------------------- 1 file changed, 93 deletions(-) diff --git a/src/fixate/drivers/dso/agilent_mso_x.py b/src/fixate/drivers/dso/agilent_mso_x.py index a0dddeaf..50971f2c 100644 --- a/src/fixate/drivers/dso/agilent_mso_x.py +++ b/src/fixate/drivers/dso/agilent_mso_x.py @@ -697,99 +697,6 @@ def retrieve_waveform_data(self): values = struct.unpack("%dB" % len(valid_bytes), valid_bytes) return values - def get_channel_data(self, channel): - """ - Get all channel data currently displayed on the screen. - Assumes the data is already captured on the scope, and acquisition is stopped. - - returns: (time_series, data) - time_series: A list of times in Seconds corresponding to each datapoint. - data: A list of the data on the screen. - - It is up to the user to know what units the data is in. - This differs from waveform_values() in that the waveform must already be acquired. This is - useful when capturing data via a trigger. - """ - data = [] - time_series = [] - - # Check the channel is actually on: - on = int(self.instrument.query(":CHANnel" + str(channel) + ":DISPlay?")) - if not on: - # Channel is off - raise ValueError("Requested channel is off.") - - # Check if there is actually data to acquire: - data_available = int( - self.instrument.query( - ":WAVeform:SOURce CHANnel" + str(channel) + ";POINTs?" - ) - ) - if data_available == 0: - # No data is available - # Setting a channel to be a waveform source turns it on, so we need to turn it off now: - self.instrument.write(":CHANnel" + str(channel) + ":DISPlay OFF") - raise ValueError("No data is available") - - # Now we know the channel is on and has some data in it. - # Get information about the waveform: - preamble = self.waveform_preamble() - - acq_type = int(preamble["acquire"]) # 0= NORMAL, 1 = PEAK DETECT, 2 = AVERAGE - x_inc = float(preamble["x_increment"]) # Time difference between data points. - x_origin = float(preamble["x_origin"]) # Always the first data point in memory - x_reference = float( - preamble["x_reference"] - ) # Specifies the data point associated with x-origin - y_inc = float( - preamble["y_increment"] - ) # Y INCrement, Voltage difference between data points - y_origin = float(preamble["y_origin"]) # Y ORIGin, Voltage at center screen - y_reference = float( - preamble["y_reference"] - ) # Y REFerence, Specifies the data point where y-origin occurs, always zero - - ## This can also be done when pulling pre-ambles (pre[1]) or may be known ahead of time, but since the script is supposed to find everything, it is done now. - if ( - acq_type == "AVER" or acq_type == "HRES" - ): # Don't need to check for both types of mnemonics like this: if ACQ_TYPE == "AVER" or ACQ_TYPE == "AVERage": becasue the scope ALWAYS returns the short form - points_mode = ( - "NORMal" # Use for Average and High Resoultion acquisition Types. - ) - ## If the :WAVeform:POINts:MODE is RAW, and the Acquisition Type is Average, the number of points available is 0. If :WAVeform:POINts:MODE is MAX, it may or may not return 0 points. - ## If the :WAVeform:POINts:MODE is RAW, and the Acquisition Type is High Resolution, then the effect is (mostly) the same as if the Acq. Type was Normal (no box-car averaging). - ## Note: if you use :SINGle to acquire the waveform in AVERage Acq. Type, no average is performed, and RAW works. - else: - points_mode = "RAW" # Use for Acq. Type NORMal or PEAK - - # Setup for data export: - # 16 bit word format... or BYTE for 8 bit format: - self.instrument.write(":WAVeform:FORMat WORD") - ## WORD format especially recommended for Average and High Res. Acq. Types, which can produce more than 8 bits of resolution. - # Explicitly set this to avoid confusion - only applies to WORD FORMat - self.instrument.write(":WAVeform:BYTeorder LSBFirst") - # Explicitly set this to avoid confusion - self.instrument.write(":WAVeform:UNSigned 0") - # This command sets the points mode to MAX AND ensures that the maximum # of points to be transferred is set, though they must still be on screen - self.instrument.write(":WAVeform:POINts MAX") - # Since the ":WAVeform:POINts MAX" command above also changes the :POINts:MODE to MAXimum, which may or may not be a good thing, so change it to what is needed next. - self.instrument.write(":WAVeform:POINts:MODE " + str(points_mode)) - - # Now get the data: - raw_data = self.instrument.query_binary_values( - ":WAVeform:SOURce CHANnel" + str(1) + ";DATA?", "h", False - ) - # Scale it based on the preamble data we got earlier: - data = [(((i - y_reference) * y_inc) + y_origin) for i in raw_data] - - # Create initial space for time data: - available_points = int(self.instrument.query_ascii_values(":WAV:POIN?")[0]) - time_series = [i for i in range(0, available_points)] - # Scale and shift as needed: - time_series = [(((i - x_reference) * x_inc) + x_origin) for i in time_series] - - return time_series, data - def digitize(self, signals): signals = [self.validate_signal(sig) for sig in signals] self.write(":DIG {}".format(",".join(signals)))