From 79213a3d259ef27b95e9970cbb2ada5e4aead7ce Mon Sep 17 00:00:00 2001 From: micah johnson Date: Sat, 24 May 2025 08:23:21 -0600 Subject: [PATCH 1/4] Added more logging, added angle to report card --- study_lyte/depth.py | 10 +++++++++- study_lyte/profile.py | 22 ++++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/study_lyte/depth.py b/study_lyte/depth.py index 55fc8de..8772653 100644 --- a/study_lyte/depth.py +++ b/study_lyte/depth.py @@ -28,6 +28,9 @@ def get_depth_from_acceleration(acceleration_df: pd.DataFrame) -> pd.DataFrame: # Convert from g's to m/s2 g = -9.81 acc = acceleration_df[acceleration_columns].mul(g) + # from study_lyte.plotting import plot_ts + # ax = plot_ts(acc, show=False) + # ax = plot_ts(acceleration_df[acceleration_columns].mul(9.81), show=True, ax=ax) # Calculate position position_vec = {} @@ -176,7 +179,7 @@ def velocity(self): @property def velocity_range(self): - """min, max of the absulute probe velocity during motion""" + """min, max of the absolute probe velocity during motion""" if self._velocity_range is None: minimum = np.min(self.velocity.iloc[self.start_idx:self.stop_idx].abs()) self._velocity_range = SimpleNamespace(min=minimum, max=self.max_velocity) @@ -229,6 +232,7 @@ def has_upward_motion(self): coarse = data.groupby(data.index // 200).first() else: coarse = data + # loop and find any values greater than the current value for i, v in enumerate(coarse): upward = np.any(coarse.iloc[i:] > v + 5) @@ -243,6 +247,7 @@ class BarometerDepth(DepthTimeseries): def __init__(self, *args, angle=None, **kwargs): super().__init__(*args, **kwargs) self.angle = angle + @property def depth(self): if self._depth is None: @@ -259,11 +264,14 @@ def depth(self): class AccelerometerDepth(DepthTimeseries): + @property def depth(self): if self._depth is None: valid = ~np.isnan(self.raw) self._depth = get_depth_from_acceleration(self.raw[valid])[self.raw.name] + # Flatten out the depth at the end self._depth.iloc[self.stop_idx:] = self._depth.iloc[self.stop_idx] + self._depth = self._depth - self._depth.iloc[self.origin] return self._depth diff --git a/study_lyte/profile.py b/study_lyte/profile.py index 89f1509..46faaed 100644 --- a/study_lyte/profile.py +++ b/study_lyte/profile.py @@ -13,6 +13,9 @@ from .logging import setup_log from .calibrations import Calibrations import logging + +from .plotting import plot_ts + setup_log() LOG = logging.getLogger('study_lyte.profile') @@ -360,6 +363,9 @@ def acceleration(self): if self.motion_detect_name != Sensor.UNAVAILABLE: # Remove gravity self._acceleration = get_neutral_bias_at_border(self.raw[self.motion_detect_name]) + # from study_lyte.plotting import plot_ts + # ax = plot_ts(self._acceleration, show=False) + # ax = plot_ts(self.raw[self.motion_detect_name], ax=ax, show=True) else: self._acceleration = Sensor.UNAVAILABLE return self._acceleration @@ -390,6 +396,7 @@ def barometer(self): baro = baro.set_index('time')['baro'] if self.accelerometer != Sensor.UNAVAILABLE: + # TODO: WHATS GOING ON HERE? idx = abs(self.accelerometer.depth - -1).argmin() else: idx = self.start.index @@ -405,6 +412,7 @@ def depth(self): if self.motion_detect_name != Sensor.UNAVAILABLE and self.depth_method != 'barometer': # User requested fused if self.depth_method == 'fused': + LOG.info("Using fused sensors to compute depth.") depth = self.fuse_depths(self.accelerometer.depth.values.copy(), self.barometer.depth.values.copy(), error=self.error.index) @@ -419,11 +427,16 @@ def depth(self): else: self._depth = pd.Series(data=depth, index=self.raw['time']) + # User requested accelerometer elif self.depth_method == 'accelerometer': + LOG.info("Using accelerometer alone to compute depth.") self._depth = self.accelerometer.depth else: + LOG.info("Using barometer alone to compute depth.") self._depth = self.barometer.depth + + # Assign positions of each event detected self.assign_event_depths() return self._depth @@ -590,6 +603,8 @@ def report_card(self): profile_string += msg.format('Snow Depth', f'{self.distance_through_snow:0.1f} cm') profile_string += msg.format('Ground Strike:', 'True' if self.ground.time is not None else 'False') profile_string += msg.format('Upward Motion:', "True" if self.has_upward_motion else "False") + if self.angle != Sensor.UNAVAILABLE: + profile_string += msg.format('Angle:', int(self.angle)) profile_string += msg.format('Errors:', f'@ {self.error.time:0.1f} s' if self.error.time is not None else 'None') profile_string += '-' * (len(header)-2) + '\n' @@ -629,7 +644,10 @@ def fuse_depths(cls, acc_depth, baro_depth, error=None): sensor_diff = abs(acc_bottom) - abs(baro_bottom) delta = 0.572 * abs(acc_bottom) + 0.308 * abs(baro_bottom) + 0.264 * sensor_diff + 8.916 # delta = (acc_bottom * (5 - scale) + baro_bottom * scale) / 5 - avg = (avg / avg_bottom) * -1*delta + avg = (avg / avg_bottom) * -1 * delta + # from study_lyte.plotting import plot_ts + # ax = plot_ts(avg, show=True) + return avg @property @@ -639,7 +657,7 @@ def angle(self): """ if self._angle is None and self.acceleration_names != Sensor.UNAVAILABLE: if 'Y-Axis' in self.acceleration_names: - data = self.raw[self.acceleration_names].iloc[0:self.start.index].mean(axis=0) + data = self.raw[self.acceleration_names].iloc[0:self.start.index + 1].mean(axis=0) magn = data.pow(2).sum()**0.5 self._angle = np.arccos(abs(data['Y-Axis']) / magn) * 180 / np.pi else: From 2e50c2239aebb970fe3ca9f87d9e27106da3ea82 Mon Sep 17 00:00:00 2001 From: micah johnson Date: Sat, 24 May 2025 08:55:57 -0600 Subject: [PATCH 2/4] Added in date based options for calibrations. Original tests plus pass --- study_lyte/calibrations.py | 40 +++++++++++++++++++++++++++++++++--- tests/data/calibrations.json | 15 ++++++++++++-- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/study_lyte/calibrations.py b/study_lyte/calibrations.py index 5aae964..db57c36 100644 --- a/study_lyte/calibrations.py +++ b/study_lyte/calibrations.py @@ -5,16 +5,26 @@ import logging from dataclasses import dataclass from typing import List +from datetime import datetime + setup_log() LOG = logging.getLogger('study_lyte.calibrations') +class MissingMeasurementDateException(Exception): + """ + Exception to raise when a probe has multiple calibrations but the date has + not been specified. + """ + pass + @dataclass() class Calibration: """Small class to make accessing calibration data a bit more convenient""" serial: str calibration: dict[str, List[float]] + date: datetime = None class Calibrations: @@ -26,15 +36,39 @@ def __init__(self, filename:Path): with open(filename, mode='r') as fp: self._info = json.load(fp) - def from_serial(self, serial:str) -> Calibration: + def from_serial(self, serial:str, date=None) -> Calibration: """ Build data object from the calibration result """ - cal = self._info.get(serial) - if cal is None: + calibrations = self._info.get(serial) + cal = None + + if calibrations is None: LOG.warning(f"No Calibration found for serial {serial}, using default") cal = self._info['default'] serial = 'UNKNOWN' else: + # Single calibration, returned as a dict + if isinstance(calibrations, dict): + cal = calibrations + + # Account for multiple calibrations + elif isinstance(calibrations, list): + # Check the date is provided + if date is None and len(calibrations) > 1: + raise MissingMeasurementDateException("Multiple calibrations found, but no date provided") + else: + # Find the calibration that matches the date + for c in calibrations: + if c['date'] >= date: + cal = c['calibration'] + + # No matches were found, date is too early + if cal is None: + LOG.warning(f"All available calibrations for {serial} are not available before {date}, using default") + cal = self._info['default'] + serial = 'UNKNOWN' + + if cal is not None: LOG.info(f"Calibration found ({serial})!") result = Calibration(serial=serial, calibration=cal) diff --git a/tests/data/calibrations.json b/tests/data/calibrations.json index ee29cfa..1b0609c 100644 --- a/tests/data/calibrations.json +++ b/tests/data/calibrations.json @@ -1,6 +1,17 @@ { - "252813070A020004":{"Sensor1":[0, 0, -10, 409], - "comment": "Test"}, + "252813070A020004":{ + "Sensor1":[0, 0, -10, 409], + "comment": "Test"}, + + "252813070A020005":[ + {"date": "2024-01-01", + "Sensor1":[0, 0, -10, 200], + "comment": "Date Test"}, + + {"date": "2025-05-01", + "Sensor1":[0, 0, -10, 600], + "comment": "Date Test2"} + ], "default":{"Sensor1":[0, 0, -1, 4096], "comment": "default"} From 16b42de9fb636a36d28479106a46061a1e5afecf Mon Sep 17 00:00:00 2001 From: micah johnson Date: Sat, 24 May 2025 09:06:55 -0600 Subject: [PATCH 3/4] Working date based calibrations --- study_lyte/calibrations.py | 10 ++++++---- tests/test_calibration.py | 23 ++++++++++++++++++++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/study_lyte/calibrations.py b/study_lyte/calibrations.py index db57c36..d1b8b72 100644 --- a/study_lyte/calibrations.py +++ b/study_lyte/calibrations.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from typing import List from datetime import datetime +import pandas as pd setup_log() @@ -42,7 +43,6 @@ def from_serial(self, serial:str, date=None) -> Calibration: cal = None if calibrations is None: - LOG.warning(f"No Calibration found for serial {serial}, using default") cal = self._info['default'] serial = 'UNKNOWN' @@ -59,8 +59,8 @@ def from_serial(self, serial:str, date=None) -> Calibration: else: # Find the calibration that matches the date for c in calibrations: - if c['date'] >= date: - cal = c['calibration'] + if date >= pd.to_datetime(c['date']): + cal = c # No matches were found, date is too early if cal is None: @@ -68,8 +68,10 @@ def from_serial(self, serial:str, date=None) -> Calibration: cal = self._info['default'] serial = 'UNKNOWN' - if cal is not None: + if cal is not None and serial != 'UNKNOWN': LOG.info(f"Calibration found ({serial})!") + else: + LOG.warning(f"No calibration found for {serial}, using default") result = Calibration(serial=serial, calibration=cal) return result diff --git a/tests/test_calibration.py b/tests/test_calibration.py index baa6389..fd07d5e 100644 --- a/tests/test_calibration.py +++ b/tests/test_calibration.py @@ -1,7 +1,8 @@ import pytest from os.path import join from pathlib import Path -from study_lyte.calibrations import Calibrations +from study_lyte.calibrations import Calibrations, MissingMeasurementDateException +import pandas as pd class TestCalibrations: @@ -14,7 +15,10 @@ def calibrations(self, calibration_json): return Calibrations(calibration_json) @pytest.mark.parametrize("serial, expected", [ + # Test actual calibration ("252813070A020004", -10), + + # Test no calibration found ("NONSENSE", -1), ]) def test_attributes(self, calibrations, serial, expected): @@ -22,4 +26,21 @@ def test_attributes(self, calibrations, serial, expected): result = calibrations.from_serial(serial) assert result.calibration['Sensor1'][2] == expected + @pytest.mark.parametrize("serial, date, expected", [ + # Test valid multi calibration between date 1 and date 2 + ("252813070A020005", "2024-02-01", 200), + # Test valid multi calibration with exact match on date 2 + ("252813070A020005", "2025-05-01", 600), + # Test single calibration with a date provided. + ("252813070A020004", "2025-01-01", 409), + ]) + def test_date_based(self, calibrations, serial, date, expected): + """""" + dt = pd.to_datetime(date) + result = calibrations.from_serial(serial, date=dt) + assert result.calibration['Sensor1'][3] == expected + def test_missing_measurement_date_exception(self, calibrations): + """ Confirm this raises an exception when no date is provided """ + with pytest.raises(MissingMeasurementDateException): + calibrations.from_serial("252813070A020005", date=None) From e8d3b0edbb4795cea26ac4182cf567d72e30c11f Mon Sep 17 00:00:00 2001 From: micah johnson Date: Sat, 24 May 2025 09:25:16 -0600 Subject: [PATCH 4/4] Added in the ability to recieve or not recevie profile dates into the profile set calibration --- study_lyte/calibrations.py | 2 +- study_lyte/profile.py | 2 +- tests/data/angled_measurement.csv | 2 +- tests/test_profile.py | 3 +++ 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/study_lyte/calibrations.py b/study_lyte/calibrations.py index d1b8b72..94175a0 100644 --- a/study_lyte/calibrations.py +++ b/study_lyte/calibrations.py @@ -37,7 +37,7 @@ def __init__(self, filename:Path): with open(filename, mode='r') as fp: self._info = json.load(fp) - def from_serial(self, serial:str, date=None) -> Calibration: + def from_serial(self, serial:str, date: datetime=None) -> Calibration: """ Build data object from the calibration result """ calibrations = self._info.get(serial) cal = None diff --git a/study_lyte/profile.py b/study_lyte/profile.py index 46faaed..2ecf495 100644 --- a/study_lyte/profile.py +++ b/study_lyte/profile.py @@ -96,7 +96,7 @@ def set_calibration(self, ext_calibrations:Calibrations): Args: ext_calibrations: External collection of calibrations """ - cal = ext_calibrations.from_serial(self.serial_number) + cal = ext_calibrations.from_serial(self.serial_number, date=self.datetime) self._calibration = cal.calibration @property diff --git a/tests/data/angled_measurement.csv b/tests/data/angled_measurement.csv index b430fb4..849cd2d 100644 --- a/tests/data/angled_measurement.csv +++ b/tests/data/angled_measurement.csv @@ -6,7 +6,7 @@ MODEL NUMBER = 3 SAMPLE RATE = 16000 ZPFO = 50 ACC. Range = 16 -Serial Num. = 252813070A020004 +Serial Num. = 252813070A020005 time,Sensor1,Sensor2,Sensor3,Sensor4,depth,X-Axis,Y-Axis,Z-Axis 0.0,3451,6,1216,3665,-0.5615234375,-0.49348,-0.7920499999999999,-0.12045 6.250247868332343e-05,3438,8,1212,3666,-0.5806001142951156,-0.49361417545537245,-0.7920525373676652,-0.1206102717929017 diff --git a/tests/test_profile.py b/tests/test_profile.py index 172c361..ee91afa 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -233,6 +233,9 @@ def test_serial_number(self, profile,filename, depth_method, expected): ("open_air.csv", 'fused', -10), # Test serial number not found ("mores_20230119.csv", 'fused', -1), + # Tests the date based calibration serial in the profile + ("angled_measurement.csv", 'fused', -10), + ]) def test_set_calibration(self, data_dir, profile,filename, depth_method, expected): p = Path(join(data_dir,'calibrations.json'))