diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 2c2c431..89a7a6b 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -16,15 +16,16 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 + with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | sudo apt-get update - python -m pip install --upgrade pip - python -m pip install pytest - python setup.py install + python3 -m pip install --upgrade pip + python3 -m pip install -e ".[dev]" + - name: test run: | diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e248dda --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,57 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "study-lyte" +version = "0.7.2" +description = "Analysis software for the Lyte probe, a digital penetrometer for studying snow" +keywords = ["snow penetrometer", "smart probe", "digital penetrometer", 'lyte probe', "avalanches", "snow densiy"] +readme = "README.rst" +requires-python = ">=3.8" +classifiers = [ + 'Development Status :: 2 - Pre-Alpha', + 'Intended Audience :: Developers', + 'Natural Language :: English', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10' +] +dependencies = [ "pandas > 2.0.0", "pandas < 3.0.0", + "scipy", "shapely"] + +[project.optional-dependencies] +dev = [ + "pytest", + "pytest-cov", + "matplotlib", + "jupyterlab", + "twine" +] + +docs = [ "nbsphinx>=0.8.12", + "sphinx-gallery>=0.9.0", + "nbconvert>=7.2.9", + "Sphinx>=5.0.0,<6.0.0", + "pandoc>=1.0.2", + "sphinxcontrib-apidoc>=0.3.0", + "ipython>=7.23.1" + ] + +all = ["study_lyte[dev,docs]"] + +[project.license] +file = "LICENSE" + +[project.urls] +Homepage = "https://adventuredata.com/" +Documentation = "https://study-lyte.readthedocs.io" +Repository = "https://github.com/AdventureData/study_lyte" +Issues = "https://github.com/AdventureData/study_lyte/issues" + +[tool.setuptools] +include-package-data = false + +[tool.setuptools.packages.find] +include = ["study_lyte*"] +exclude = ["docs*", "tests*"] diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index ba72e91..0000000 --- a/requirements_dev.txt +++ /dev/null @@ -1,16 +0,0 @@ -pip>=19.2.3 -bump2version==0.5.11 -wheel==0.33.6 -watchdog==0.9.0 -flake8==3.7.8 -tox==3.14.0 -coverage==4.5.4 -Sphinx==6.1.3 -twine==1.14.0 -jupyterlab==3.6.1 -pytest==6.2.4 -matplotlib -nbconvert -coverage -sphinxcontrib.apidoc -nbsphinx \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 4628522..0000000 --- a/setup.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python - -"""The setup script.""" - -from setuptools import setup, find_packages - -with open('README.rst') as readme_file: - readme = readme_file.read() - -requirements = ["pandas > 1.2.0", "pandas< 2.0.0", - "scipy>=1.8.0", "scipy<2.0.0", "shapely"] -test_requirements = ['pytest>=3', ] - -setup( - author="Micah Johnson ", - author_email='info@adventuredata.com', - python_requires='>=3.6', - classifiers=[ - 'Development Status :: 2 - Pre-Alpha', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Natural Language :: English', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - ], - description="Package for doing analysis with Lyte Probe data", - install_requires=requirements, - license="GNU General Public License v3", - long_description=readme, - include_package_data=True, - keywords='study_lyte', - name='study_lyte', - packages=find_packages(include=['study_lyte', 'study_lyte.*'], exclude=['tests*']), - test_suite='tests', - tests_require=test_requirements, - url='https://github.com/AdventureData/study_lyte', - version='0.7.2', - zip_safe=False, -) diff --git a/study_lyte/adjustments.py b/study_lyte/adjustments.py index a8949c3..4ad1645 100644 --- a/study_lyte/adjustments.py +++ b/study_lyte/adjustments.py @@ -46,7 +46,8 @@ def get_neutral_bias_at_border(series: pd.Series, fractional_basis: float = 0.00 Returns: bias_adj: bias adjusted data to near zero """ - bias = get_directional_mean(series.values, fractional_basis=fractional_basis, direction=direction) + arr = series.values if hasattr(series,'values') else series + bias = get_directional_mean(arr, fractional_basis=fractional_basis, direction=direction) bias_adj = series - bias return bias_adj @@ -108,7 +109,7 @@ def merge_on_to_time(df_list, final_time): if i == 0: result = data else: - result = pd.merge_ordered(result, data, on='time', fill_method='cubic') + result = pd.merge_ordered(result, data, on='time') # interpolate the nan's result = result.interpolate(method='nearest', limit_direction='both') @@ -136,32 +137,47 @@ def merge_time_series(df_list): if i == 0: result = df.copy() else: - result = pd.merge_ordered(result, df, on='time', fill_method='cubic') + result = pd.merge_ordered(result, df, on='time') # interpolate the nan's result = result.interpolate(method='index') return result -def remove_ambient(active, ambient, min_ambient_range=50, direction='forward'): +def remove_ambient(active, ambient, min_ambient_range=100, direction='forward'): """ Attempts to remove the ambient signal from the active signal """ amb_max = ambient.max() amb_min = ambient.min() if abs(amb_max - amb_min) > min_ambient_range: + # Only adjust up to the dropdown + tol = 0.05 n = get_points_from_fraction(len(ambient), 0.01) amb = ambient.rolling(window=n, center=True, closed='both', min_periods=1).mean() + amb_back = get_directional_mean(amb, direction='backward', fractional_basis=0.1) + active_forward = get_directional_mean(active, direction='forward', fractional_basis=0.1) + + ind = amb < (amb_back * (1 + tol)) + decayed_idx = np.argwhere(ind.values) + if decayed_idx.any(): + decayed_idx = decayed_idx[0][0] + else: + decayed_idx = 0 + norm_ambient = get_normalized_at_border(amb, direction=direction) norm_active = get_normalized_at_border(active, direction=direction) - basis = get_directional_mean(active, direction=direction) - clean = (norm_active - norm_ambient) * basis - if clean.min() < 0 and 'backward' not in direction: - clean = clean - clean.min() + norm_ambient[decayed_idx:] = 0 + norm_diff = norm_active - norm_ambient + norm_diff[ norm_diff <= 0] = 0 #np.nan + norm_diff = norm_diff.interpolate(method='cubic') + clean = active_forward * norm_diff + clean[:int(decayed_idx*(0.5))] = 1 + # Zero cant work here + clean[clean < 1] = 1 else: clean = active - return clean @@ -175,7 +191,6 @@ def apply_calibration(series, coefficients, minimum=None, maximum=None): result[result > maximum] = maximum if minimum is not None: result[result < minimum] = minimum - return result @@ -312,4 +327,4 @@ def zfilter(series, fraction): zi = np.zeros(filter_coefficients.shape[0]-1) #lfilter_zi(filter_coefficients, 1) filtered, zf = lfilter(filter_coefficients, 1, series, zi=zi) filtered = lfilter(filter_coefficients, 1, filtered[::-1], zi=zf)[0][::-1] - return filtered \ No newline at end of file + return filtered diff --git a/study_lyte/depth.py b/study_lyte/depth.py index 7afe46d..55fc8de 100644 --- a/study_lyte/depth.py +++ b/study_lyte/depth.py @@ -1,5 +1,5 @@ import pandas as pd -from scipy.integrate import cumtrapz +from scipy.integrate import cumulative_trapezoid import numpy as np from types import SimpleNamespace @@ -33,9 +33,9 @@ def get_depth_from_acceleration(acceleration_df: pd.DataFrame) -> pd.DataFrame: position_vec = {} for i, axis in enumerate(acceleration_columns): # Integrate acceleration to velocity - v = cumtrapz(acc[axis].values, acc.index, initial=0) + v = cumulative_trapezoid(acc[axis].values, acc.index, initial=0) # Integrate velocity to position - position_vec[axis] = cumtrapz(v, acc.index, initial=0) + position_vec[axis] = cumulative_trapezoid(v, acc.index, initial=0) position_df = pd.DataFrame.from_dict(position_vec) position_df['time'] = acc.index diff --git a/study_lyte/detect.py b/study_lyte/detect.py index 63bf271..4db367a 100644 --- a/study_lyte/detect.py +++ b/study_lyte/detect.py @@ -178,7 +178,7 @@ def get_acceleration_stop(acceleration, threshold=-0.2, max_threshold=0.1): return acceleration_stop -def get_nir_surface(clean_active, threshold=-1, max_threshold=0.25): +def get_nir_surface(clean_active, threshold=30, max_threshold=None): """ Using the cleaned active, estimate the index at when the probe was in the snow. @@ -190,22 +190,22 @@ def get_nir_surface(clean_active, threshold=-1, max_threshold=0.25): Return: surface: Integer index of the estimated snow surface """ - n = get_points_from_fraction(len(clean_active), 0.01) + # n = get_points_from_fraction(len(clean_active), 0.01) # Normalize by data unaffected by ambient - clean_norm = clean_active / clean_active[n:].mean() - neutral = get_neutral_bias_at_border(clean_norm) + neutral = get_neutral_bias_at_border(clean_active) # Retrieve a likely candidate under challenging ambient conditions - - max_idx = np.argwhere((neutral == neutral.min()).values)[0][0] + window = get_points_from_fraction(len(neutral), 0.01) + diff = neutral.rolling(window=window).std().values # Detect likely candidate normal ambient conditions - surface = get_signal_event(neutral, search_direction='forward', threshold=threshold, - max_threshold=max_threshold, n_points=n) + surface = get_signal_event(diff, search_direction='backward', threshold=threshold, + max_threshold=max_threshold, n_points=1) # No surface found and all values met criteria - if surface == len(neutral)-1: + if surface == len(neutral)-1 or surface is None: surface = 0 - + # from .plotting import plot_nir_surface + # plot_nir_surface(neutral, diff, surface) return surface @@ -250,7 +250,7 @@ def get_ground_strike(signal, stop_idx): """ The probe hits ground somtimes before we detect stop. """ - buffer = get_points_from_fraction(len(signal), 0.05) + buffer = get_points_from_fraction(len(signal), 0.12) start = stop_idx - buffer start = start if start > 0 else 0 end = stop_idx + buffer @@ -258,20 +258,30 @@ def get_ground_strike(signal, stop_idx): rel_stop = stop_idx - start sig_arr = signal[start:end] - norm1 = get_neutral_bias_at_index(sig_arr, rel_stop + buffer) + window = get_points_from_fraction(len(sig_arr), 0.01) + diff = sig_arr.rolling(window=window).std().values + diff = get_neutral_bias_at_border(diff, direction='backward') - # norm1 = get_neutral_bias_at_border(signal[start:end], 0.1, 'backward') - diff = zfilter(norm1.diff(), 0.001) # Large change in signal - impact = get_signal_event(diff, threshold=-1000, max_threshold=-70, n_points=1, search_direction='forward') + # Large change in signal + impact = get_signal_event(diff, threshold=150, max_threshold=1000, n_points=1, search_direction='forward') # Large chunk of data that's the same near the stop + norm1 = get_neutral_bias_at_index(sig_arr, rel_stop+buffer).values n_points = get_points_from_fraction(len(norm1), 0.1) - long_press = get_signal_event(norm1, threshold=-150, max_threshold=150, n_points=n_points, search_direction='backward') - tol = get_points_from_fraction(len(norm1), 0.2) + long_press = get_signal_event(norm1, threshold=-10000, max_threshold=150, n_points=n_points, search_direction='backward') + tol = get_points_from_fraction(len(norm1), 0.1) ground = None - if impact is not None and long_press is not None: + if impact is not None: + impact += start + if long_press is not None: + long_press += start + + if long_press is not None and impact is not None: if (long_press-tol) <= impact <= (long_press+tol): - ground = impact + start + ground = impact + + # from .plotting import plot_ground_strike, plot_ts + # plot_ground_strike(signal, diff, norm1, start, stop_idx, impact, long_press,ground) return ground \ No newline at end of file diff --git a/study_lyte/logging.py b/study_lyte/logging.py index b6f9690..9e609ab 100644 --- a/study_lyte/logging.py +++ b/study_lyte/logging.py @@ -14,7 +14,7 @@ def setup_log(debug=False): format=default, level=level, handlers=handlers) # Set all ignored modules to be quiet. - ignore_modules = ['matplotlib'] + ignore_modules = ['matplotlib', 'pyngui'] for name in logging.Logger.manager.loggerDict.keys(): if any([m in name for m in ignore_modules]): logger = logging.getLogger(name) diff --git a/study_lyte/plotting.py b/study_lyte/plotting.py index 307462c..67fbc27 100644 --- a/study_lyte/plotting.py +++ b/study_lyte/plotting.py @@ -111,16 +111,55 @@ def plot_fused_depth(acc_depth, baro_depth, avg, scaled_baro=None, error=None): plt.show() - -def plot_ground_strike(signal, search_start, stop_idx, impact, long_press, ground): +def plot_ground_strike(signal, impact_series, long_press_series, search_start, stop_idx, impact, long_press, ground): events = [('stop', stop_idx)] + impact_events = [('stop', stop_idx - search_start)] + if long_press is not None: - events.append(('long_press', long_press + search_start)) + events.append(('long_press', long_press)) + impact_events.append(('long_press', long_press-search_start)) + if impact is not None: - events.append(('impact', impact + search_start)) + events.append(('impact', impact)) + impact_events.append(('impact', impact-search_start)) + if ground is not None: events.append(('ground', ground)) - ax = plot_ts(signal, events=events,show=False) - ax.legend() + + fig, (ax1, ax2, ax3) = plt.subplots(3,1) + ax1 = plot_ts(signal, events=events,show=False, ax=ax1) + ax1.set_title('Full series and events') + ax1.legend() + + plot_ts(impact_series, events=impact_events, ax=ax2, show=False) + ax2.set_title('impact') + ax2.legend() + + plot_ts(long_press_series, events=impact_events, ax=ax3, show=False) + ax3.set_title('Long Press') + ax3.legend() + plt.tight_layout() plt.show() +def plot_nir_cleaning(active, ambient, norm_active, norm_ambient, diff, clean): + + fig,axes = plt.subplots(2,1) + # Plot normalized + plot_ts(norm_ambient, ax=axes[0], data_label='norm amb.', show=False) + plot_ts(norm_active, ax=axes[0], data_label='norm act.', show=False) + plot_ts(diff, ax=axes[0], data_label='norm diff', show=False) + + plot_ts(ambient, ax=axes[1], data_label='ambient', show=False) + plot_ts(active, ax=axes[1], data_label='active', show=False) + + plot_ts(clean, ax=axes[1], data_label='clean.', show=False) + plt.show() + +def plot_nir_surface(clean_active, diff, surface): + events = [] + if surface is not None: + events.append(('surface', surface)) + + fig,axes = plt.subplots(2, 1) + plot_ts(clean_active,events=events, ax=axes[0], show=False) + plot_ts(diff, events=events, ax=axes[1]) \ No newline at end of file diff --git a/study_lyte/profile.py b/study_lyte/profile.py index 29a0204..41e95d2 100644 --- a/study_lyte/profile.py +++ b/study_lyte/profile.py @@ -27,547 +27,659 @@ class Sensor(Enum): UNAVAILABLE = -1 UNINTERPRETABLE = -2 -class LyteProfileV6: - def __init__(self, filename, surface_detection_offset=4.5, calibration=None, depth_method='fused', tip_diameter_mm=5): - """ - Args: - filename: path to valid lyte probe csv. - surface_detection_offset: Geometric offset between nir sensors and tip in cm. - calibration: Dictionary of keys and polynomial coefficients to calibration sensors - depth_method: Method/sensor to use for depth, Options are barometer, accelerometer, fused - tip_diameter_mm: diameter of the force tip in mm - - """ - self.filename = Path(filename) - self.surface_detection_offset = surface_detection_offset - self.calibration = calibration or {} - self.depth_method = depth_method - self.tip_diameter_mm = tip_diameter_mm - - # Properties - self._raw = None - self._meta = None - self._accelerometer = None - self._barometer = None - self._point = None - self.header_position = None - - self._depth = None # Final depth series used for analysis - self._acceleration = None # No gravity acceleration - self._cropped = None # Full dataframe cropped to surface and stop - self._force = None - self._nir = None - - # Useful stats/info - self._distance_traveled = None # distance travelled while moving - self._distance_through_snow = None # Distance travelled while in snow - self._motion_detect_name = None # column name containing accel data dir of travel - self._acceleration_names = None # All columns containing accel data - self._moving_time = None # time the probe was moving - self._avg_velocity = None # avg velocity of the probe while in the snow - self._resolution = None # Vertical resolution of the profile in the snow - self._datetime = None - self._has_upward_motion = None # Flag for datasets containing upward motion - self._angle = None - - # Events - self._start = None - self._stop = None - self._surface = None - self._error = None - self._ground = None - - @staticmethod - def process_df(df): - """ - Migrate all baro depths to filtereddepth and remove ambient - to add NIR column - """ - df = df.rename(columns={'depth': 'filtereddepth'}) - df['nir'] = remove_ambient(df['Sensor3'], df['Sensor2']) - return df - - @classmethod - def from_dataframe(cls, df): - profile = LyteProfileV6(None) - profile._raw = cls.process_df(df) - return profile - - def assign_event_depths(self): - """ - Enable depth assignment post depth realization - """ - self.events - for event in [self._start, self._stop, self._surface.nir, self._surface.force]: - event.depth = self.depth.iloc[event.index] - - @property - def raw(self): - """ - Pandas dataframe hold the data exactly as it read in. - """ - if self._raw is None: - metadata = self.metadata - self._raw, self._meta = read_data(str(self.filename), metadata, self.header_position) - self._raw = self.process_df(self.raw) - - return self._raw - - @property - def metadata(self): - """ - Returns a dictionary of all data held in the header portion of the csv - """ - if self._meta is None: - self.header_position, self._meta = find_metadata(str(self.filename)) - - # Manage misc naming of the acceleration range - if 'ACC. Range' not in self._meta.keys(): - if "ACCRANGE" in self._meta.keys(): - self._meta['ACC. Range'] = float(self._meta['ACCRANGE']) - else: - self._meta['ACC. Range'] = 2 - - else: - self._meta['ACC. Range'] = float(self._meta['ACC. Range']) - - if 'ZPFO' in self._meta.keys(): - self._meta['ZPFO'] = int(self._meta['ZPFO']) - - return self._meta - - - @property - def motion_detect_name(self): - """Return all the names of acceleration columns""" - if self._motion_detect_name is None: - self._motion_detect_name = self.get_motion_name(self.raw.columns) - return self._motion_detect_name - - @property - def acceleration_names(self): - """Return all the names of acceleration columns""" - if self._acceleration_names is None: - self._acceleration_names = self.get_acceleration_columns(self.raw.columns) - return self._acceleration_names - - @property - def acceleration(self): - """ - Retrieve acceleration with gravity removed - """ - # Assign the detection column if it is available - if self._acceleration is None: - if self.motion_detect_name != Sensor.UNAVAILABLE: - # Remove gravity - self._acceleration = get_neutral_bias_at_border(self.raw[self.motion_detect_name]) - else: - self._acceleration = Sensor.UNAVAILABLE - return self._acceleration - - @property - def nir(self): - """ - Retrieve the Active NIR sensor with ambient NIR removed - """ - if self._nir is None: - self._nir = self.raw[["Sensor2", "Sensor3", "nir"]] - self._nir['depth'] = self.depth.values - end = self.stop.index if self.ground.index is None else self.ground.index - if self.surface.nir.index < end: - self._nir = self._nir.iloc[self.surface.nir.index:end].reset_index() - self._nir = self._nir.drop(columns='index') - self._nir['depth'] = self._nir['depth'] - self._nir['depth'].iloc[0] - else: - self._nir = Sensor.UNINTERPRETABLE - return self._nir - - @property - def force(self): - """ - calibrated force and depth as a pandas dataframe cropped to the snow surface and the stop of motion - """ - if self._force is None: - if 'Sensor1' in self.calibration.keys(): - force = apply_calibration(self.raw['Sensor1'].values, self.calibration['Sensor1'], minimum=0) - # force = force - force[0] - else: - force = self.raw['Sensor1'].values - - self._force = pd.DataFrame({'force': force, 'depth': self.depth.values}) - # prefer a ground index if available - end = self.stop.index if self.ground.index is None else self.ground.index - self._force = self._force.iloc[self.surface.force.index:end].reset_index() - self._force = self._force.drop(columns='index') - if not self._force.empty: - self._force['depth'] = self._force['depth'] - self._force['depth'].iloc[0] - - return self._force - - @property - def pressure(self): - if 'pressure' not in self.force.columns: - # Add pressure in kpa - area = np.pi * (self.tip_diameter_mm / 1000)**2/4 - # Convert mN to kPa - self.force['pressure'] = ((self.force['force']/1000) / area) / 1000 - return self.force[['depth', 'pressure']] - - @property - def accelerometer(self): - """Returns a class holding timeseries of accelerometer based depth""" - if self._accelerometer is None: - if self.motion_detect_name == Sensor.UNAVAILABLE: - self._accelerometer = Sensor.UNAVAILABLE - else: - data = pd.DataFrame.from_dict({'time':self.raw['time'], self.acceleration.name:self.acceleration.values}) - data = data.set_index('time')[self.acceleration.name] - self._accelerometer = AccelerometerDepth(data, self.start.index, self.stop.index) - return self._accelerometer - - @property - def barometer(self): - """Returns a class holding timeseries of barometer based depth""" - if self._barometer is None: - baro = self.raw[['time', 'filtereddepth']].set_index('time')['filtereddepth'] - if 'ZPFO' in self.metadata.keys(): - if self.metadata['ZPFO'] < 50: - LOG.info('Filtering barometer data...') - # TODO: make this more intelligent - baro = zfilter(self.raw['filtereddepth'].values, 0.4) - baro = pd.DataFrame.from_dict({'baro':baro, 'time': self.raw['time']}) - baro = baro.set_index('time')['baro'] - - if self.accelerometer != Sensor.UNAVAILABLE: - idx = abs(self.accelerometer.depth - -1).argmin() - else: - idx = self.start.index - - angle = None if self.angle == Sensor.UNAVAILABLE else self.angle - self._barometer = BarometerDepth(baro, idx, self.stop.index, angle=angle) - - return self._barometer - - @property - def depth(self): - if self._depth is None: - if self.motion_detect_name != Sensor.UNAVAILABLE and self.depth_method != 'barometer': - # User requested fused - if self.depth_method == 'fused': - depth = self.fuse_depths(self.accelerometer.depth.values.copy(), - self.barometer.depth.values.copy(), - error=self.error.index) - if depth.min() < -230: - LOG.warning('Fused depth result produced a profile > 250 cm. Defaulting to accelerometer') - self._depth = self.accelerometer.depth - else: - self._depth = pd.Series(data=depth, index=self.raw['time']) - # User requested accelerometer - elif self.depth_method == 'accelerometer': - self._depth = self.accelerometer.depth - else: - self._depth = self.barometer.depth - self.assign_event_depths() - - return self._depth - - @property - def time(self): - """Return the sample time data""" - return self.raw['time'] - - @property - def start(self): - """ Return start event """ - if self._start is None: - if self.motion_detect_name != Sensor.UNAVAILABLE: - idx = get_acceleration_start(self.acceleration) - else: - idx=0 - - self._start = Event(name='start', index=idx, depth=None, time=self.raw['time'].iloc[idx]) - return self._start - - @property - def stop(self): - """ Return stop event """ - if self._stop is None: - if self.motion_detect_name != Sensor.UNAVAILABLE: - backward_accel = get_neutral_bias_at_border(self.raw[self.motion_detect_name], direction='backward') - idx = get_acceleration_stop(backward_accel) +class GenericProfileV6: + def __init__(self, filename, surface_detection_offset=4.5, calibration=None, + tip_diameter_mm=5): + """ + Args: + filename: path to valid lyte probe csv. + surface_detection_offset: Geometric offset between nir sensors and tip in cm. + calibration: Dictionary of keys and polynomial coefficients to calibration sensors + tip_diameter_mm: diameter of the force tip in mm + """ + self.filename = Path(filename) + self.surface_detection_offset = surface_detection_offset + self.calibration = calibration or {} + self.tip_diameter_mm = tip_diameter_mm + + # Properties + self._raw = None + self._meta = None + self._point = None + self.header_position = None + + self._depth = None # Final depth series used for analysis + self._acceleration = None # No gravity acceleration + self._cropped = None # Full dataframe cropped to surface and stop + self._force = None + self._nir = None + + # Useful stats/info + self._distance_traveled = None # distance travelled while moving + self._distance_through_snow = None # Distance travelled while in snow + self._avg_velocity = None # avg velocity of the probe while in the snow + self._resolution = None # Vertical resolution of the profile in the snow + self._datetime = None + self._has_upward_motion = None # Flag for datasets containing upward motion + + # Events + self._start = None + self._stop = None + self._surface = None + self._error = None + self._ground = None + + def assign_event_depths(self): + """" Enable depth assignment post depth realization """ + self.events + for event in [self._start, self._stop, self._surface.nir, self._surface.force]: + event.depth = self.depth.iloc[event.index] + + @classmethod + def from_metadata(cls, filename, **kwargs): + profile = cls(filename) + if 'APP VERSION' in profile.metadata.keys(): + return ProcessedProfileV6(filename) + else: + return LyteProfileV6(filename) + + + @property + def raw(self): + """ + Pandas dataframe hold the data exactly as it read in. + """ + if self._raw is None: + metadata = self.metadata + self._raw, self._meta = read_data(str(self.filename), metadata, self.header_position) + self._raw = self.process_df(self.raw) + + return self._raw + + @property + def metadata(self): + """ + Returns a dictionary of all data held in the header portion of the csv + """ + if self._meta is None: + self.header_position, self._meta = find_metadata(str(self.filename)) + + # Manage misc naming of the acceleration range + if 'ACC. Range' not in self._meta.keys(): + if "ACCRANGE" in self._meta.keys(): + self._meta['ACC. Range'] = float(self._meta['ACCRANGE']) else: - idx = get_nir_stop(self.raw['Sensor3']) - if idx is not None: - self._stop = Event(name='stop', index=idx, depth=None, time=self.raw['time'].iloc[idx]) - else: - self._stop = Event(name='stop', index=0, depth=None, time=self.raw['time'].iloc[0]) - - return self._stop - - @property - def surface(self): - """ - Return surface events for the nir and force which are physically separated by a distance - """ - if self._surface is None: - # Call to populate nir in raw - idx = get_nir_surface(self.raw['nir']) - if idx == 0: - LOG.warning("Unable to find snow surface, defaulting to first data point") - # Event according the NIR sensors - depth = self.depth.iloc[idx] - nir = Event(name='surface', index=idx, depth=depth, time=self.raw['time'].iloc[idx]) - - # Event according to the force sensor - force_surface_depth = depth + self.surface_detection_offset - f_idx = abs(self.depth - force_surface_depth).argmin() - # Retrieve force estimated start - f_start = get_sensor_start(self.raw['Sensor1'], max_threshold=0.02, threshold=-0.02) - f_start = f_start or f_idx - - # If the force start is before the NIR start then adjust - if f_start < self.start.index: - LOG.info(f'Choosing motion start ({self.start.index}) over force start ({f_start})...') - f_idx = self.start.index - force_surface_depth = self.depth.iloc[f_idx] - - elif f_start < f_idx: - LOG.info(f'Choosing force start ({f_start}) over nir derived ({f_idx})...') - f_idx = f_start - force_surface_depth = self.depth.iloc[f_idx] - - force = Event(name='surface', index=f_idx, depth=force_surface_depth, time=self.raw['time'].iloc[f_idx]) - self._surface = SimpleNamespace(name='surface', nir=nir, force=force) - - # Allow surface detection to modify the start if there is conflict. - if nir.time < self.start.time: - self._start = nir - self._start.name = 'start' - - return self._surface - @property - def ground(self): - """Event for ground detection""" - if self._ground is None: - ground = get_ground_strike(self.raw['Sensor1'], self.stop.index) - if ground is not None: - self._ground = Event(name='ground', index=ground, depth=self.depth.iloc[ground], time=self.raw['time'].iloc[ground]) - else: - self._ground = Event(name='ground', index=None, depth=None, time=None) - - return self._ground - - @property - def error(self): - """ Return error event """ - if self._error is None: - if self.motion_detect_name != Sensor.UNAVAILABLE: - idx = self.get_error(self.raw[self.motion_detect_name], self.metadata['ACC. Range']) - depth = None - if idx is None: - t = None - else: - t = self.raw['time'].iloc[idx] + self._meta['ACC. Range'] = 16 - else: - idx = None - depth = None - t = None - self._error = Event(name='error', index=idx, depth=depth, time=t) - - return self._error - - @property - def distance_traveled(self): - if self._distance_traveled is None: - # Call depth to ensure its populated - self.depth - self._distance_traveled = abs(self.start.depth - self.stop.depth) - return self._distance_traveled - - @property - def distance_through_snow(self): - if self._distance_through_snow is None: - self._distance_through_snow = abs(self.surface.nir.depth - self.stop.depth) - return self._distance_through_snow - - @property - def moving_time(self): - """Amount of time the probe was in motion""" - if self._moving_time is None: - self._moving_time = self.stop.time - self.start.time - return self._moving_time - - @property - def avg_velocity(self): - if self._avg_velocity is None: - self._avg_velocity = self.distance_traveled / self.moving_time - return self._avg_velocity - - @property - def datetime(self): - if self._datetime is None: - self._datetime = pd.to_datetime(self.metadata['RECORDED']) - return self._datetime - - @property - def resolution(self): - if self._resolution is None: - if type(self.nir) == pd.DataFrame: - n_points = len(self.nir) - self._resolution = n_points / self.distance_through_snow - else: - self._resolution = np.nan - return self._resolution - - @property - def events(self): - """ - Return all the common events recorded - """ - return [self.start, self.stop, self.surface.nir, self.surface.force, self.ground, self.error] - - @staticmethod - def get_motion_name(columns): - """ - Find a column containing acceleration data sometimes called - acceleration or Y-Axis to handle variations in formatting of file - """ - candidates = [c for c in columns if c.lower() in ['acceleration', 'y-axis']] - if candidates: - return candidates[0] else: - return Sensor.UNAVAILABLE - - @staticmethod - def get_acceleration_columns(columns): - """ - Find a columns containing acceleration data sometimes called - acceleration or X,Y,Z-Axis to handle variations in formatting of file - """ - candidates = [c for c in ['acceleration', 'X-Axis', 'Y-Axis', 'Z-Axis'] if c in columns] - if candidates: - return candidates + self._meta['ACC. Range'] = float(self._meta['ACC. Range']) + + if 'ZPFO' in self._meta.keys(): + self._meta['ZPFO'] = int(self._meta['ZPFO']) + + return self._meta + + @property + def nir(self): + """ + Retrieve the Active NIR sensor with ambient NIR removed + """ + if self._nir is None: + self._nir = self.raw[["Sensor2", "Sensor3", "nir"]] + self._nir['depth'] = self.depth.values + end = self.stop.index if self.ground.index is None else self.ground.index + if self.surface.nir.index < end: + self._nir = self._nir.iloc[self.surface.nir.index:end].reset_index() + self._nir = self._nir.drop(columns='index') + self._nir['depth'] = self._nir['depth'] - self._nir['depth'].iloc[0] else: - return Sensor.UNAVAILABLE - @property - def point(self): - """Return shapely geometry point of the measurement location in EPSG 4326""" - if self._point is None: - if all([k in self.metadata.keys() for k in ['Latitude', 'Longitude']]): - self._point = Point(float(self.metadata['Longitude']), float(self.metadata['Latitude'])) - else: - self._point = Sensor.UNAVAILABLE - - return self._point - - def report_card(self): - """ - Return a useful string to print about metrics - """ - msg = '| {:<15} {:<20} |\n' - n_chars = int((39 - len(self.filename.name)) / 2) - s = '-'* n_chars - header = f'\n{s} {self.filename.name} {s}\n' - profile_string = header - profile_string += msg.format('Recorded', f'{self.datetime.isoformat()}') - profile_string += msg.format('Points', f'{len(self.raw.index):,}') - profile_string += msg.format('Moving Time', f'{self.moving_time:0.1f} s') - profile_string += msg.format('Avg. Speed', f'{self.avg_velocity:0.0f} cm/s') - profile_string += msg.format('Resolution', f'{self.resolution:0.1f} pts/cm') - profile_string += msg.format('Total Travel', f'{self.distance_traveled:0.1f} cm') - profile_string += msg.format('Snow Depth', f'{self.distance_through_snow:0.1f} cm') - profile_string += msg.format('Ground Strike:', 'True' if self.ground.time is not None else 'False') - profile_string += msg.format('Upward Motion:', "True" if self.has_upward_motion else "False") - profile_string += msg.format('Errors:', f'@ {self.error.time:0.1f} s' if self.error.time is not None else 'None') - - profile_string += '-' * (len(header)-2) + '\n' - return profile_string - - @classmethod - def fuse_depths(cls, acc_depth, baro_depth, error=None): - """ - Function to intelligently fuse together depth timeseries - - """ - # Accelerometer is always solid in the beginning, unknown as we move on in time - weights_acc = np.ones_like(acc_depth) * 100 - weights_baro = np.ones_like(baro_depth) - avg = np.average(np.array([acc_depth, baro_depth]).T, axis=1, - weights=np.array([weights_acc, weights_baro]).T) - scaled_baro = baro_depth.copy() - - if error is not None: - LOG.info("Blending depth timeseries...") - minimum = 0.01 - # Ensure the same starting place - scaled_baro[error:] = scaled_baro[error:] - (scaled_baro[error] - avg[error]) - - # Full reliance on the constrained baro - weights_acc[error:] = minimum - - avg = np.average(np.array([acc_depth, scaled_baro]).T, axis=1, - weights=np.array([weights_acc, weights_baro]).T) - - # The deeper we go the more the baro constrains - baro_bottom = baro_depth.min() - acc_bottom = acc_depth.min() - avg_bottom = avg.min() - - # Scale total - sensor_diff = abs(acc_bottom) - abs(baro_bottom) - delta = 0.572 * abs(acc_bottom) + 0.308 * abs(baro_bottom) + 0.264 * sensor_diff + 8.916 - # delta = (acc_bottom * (5 - scale) + baro_bottom * scale) / 5 - avg = (avg / avg_bottom) * -1*delta - return avg - - - @property - def has_upward_motion(self): - """ - Bool indicating if upward motion was detected - - """ - if self._has_upward_motion is None: - self._has_upward_motion = False - # crop the depth data and downsample for speedy check - n = get_points_from_fraction(len(self.depth), 0.005) - coarse = self.depth.iloc[self.start.index:self.stop.index:n] - # loop and find any values greater than the current value - for i,v in coarse.items(): - upward = np.any(coarse.loc[i:] > v + 5) - if upward: - self._has_upward_motion = True - break - - return self._has_upward_motion - - @property - def angle(self): - """ - float indicating the angle at the start of a measuruement - """ - if self._angle is None and self.acceleration_names != Sensor.UNAVAILABLE: - if 'Y-Axis' in self.acceleration_names: - data = self.raw[self.acceleration_names].iloc[0:self.start.index].mean(axis=0) - magn = data.pow(2).sum()**0.5 - self._angle = np.arccos(abs(data['Y-Axis']) / magn) * 180 / np.pi - else: - self._angle = Sensor.UNAVAILABLE + self._nir = Sensor.UNINTERPRETABLE + return self._nir + + @property + def force(self): + """ + calibrated force and depth as a pandas dataframe cropped to the snow surface and the stop of motion + """ + if self._force is None: + if 'Sensor1' in self.calibration.keys(): + force = apply_calibration(self.raw['Sensor1'].values, self.calibration['Sensor1'], minimum=0, maximum=15000) + # force = force - force[0] + else: + force = self.raw['Sensor1'].values + + self._force = pd.DataFrame({'force': force, 'depth': self.depth.values}) + # prefer a ground index if available + end = self.stop.index if self.ground.index is None else self.ground.index + self._force = self._force.iloc[self.surface.force.index:end].reset_index() + self._force = self._force.drop(columns='index') + if not self._force.empty: + self._force['depth'] = self._force['depth'] - self._force['depth'].iloc[0] + + return self._force + + @property + def pressure(self): + """ Force converted into pressure in kpa""" + if 'pressure' not in self.force.columns: + # Add pressure in kpa + area = np.pi * (self.tip_diameter_mm / 1000)**2/4 + # Convert mN to kPa + self.force['pressure'] = ((self.force['force']/1000) / area) / 1000 + return self.force[['depth', 'pressure']] + + @property + def depth(self): + raise NotImplemented("Must implement depth") + + @property + def start(self): + """ Return start event """ + raise NotImplemented("Must implement start event") + + @property + def stop(self): + """ Return stop event """ + raise NotImplemented("Must implement start event") + + @property + def surface(self): + """ + Return surface events for the nir and force which are physically separated by a distance + """ + raise NotImplemented("Must implement surface event") + + @property + def ground(self): + """Event for ground detection""" + if self._ground is None: + ground = get_ground_strike(self.raw['Sensor1'], self.stop.index) + if ground is not None: + self._ground = Event(name='ground', index=ground, depth=self.depth.iloc[ground], time=None) + if 'time' in self.raw.columns: + self._ground.time = self.raw['time'].iloc[ground] + else: + self._ground = Event(name='ground', index=None, depth=None, time=None) + + return self._ground + + @property + def error(self): + """ Return error event """ + raise NotImplemented("Error event not implemented") + + @property + def distance_traveled(self): + if self._distance_traveled is None: + # Call depth to ensure its populated + self.depth + self._distance_traveled = abs(self.start.depth - self.stop.depth) + return self._distance_traveled + + @property + def distance_through_snow(self): + if self._distance_through_snow is None: + self._distance_through_snow = abs(self.surface.nir.depth - self.stop.depth) + return self._distance_through_snow + + @property + def datetime(self): + """Retrieves the datetime object the measurement was taken""" + if self._datetime is None: + self._datetime = pd.to_datetime(self.metadata['RECORDED']) + return self._datetime + + @property + def resolution(self): + """ Estimates the resolution of the profile""" + if self._resolution is None: + if type(self.nir) == pd.DataFrame: + n_points = len(self.nir) + self._resolution = n_points / self.distance_through_snow + else: + self._resolution = np.nan + return self._resolution + + @property + def events(self): + """ + Return all the common events recorded + """ + return [self.start, self.stop, self.surface.nir, self.surface.force, + self.ground, self.error] + + @property + def point(self): + """Return shapely geometry point of the measurement location in EPSG 4326""" + if self._point is None: + if all([k in self.metadata.keys() for k in ['Latitude', 'Longitude']]): + self._point = Point(float(self.metadata['Longitude']), float(self.metadata['Latitude'])) + else: + self._point = Sensor.UNAVAILABLE + + return self._point + + @property + def has_upward_motion(self): + """ + Bool indicating if upward motion was detected + """ + if self._has_upward_motion is None: + self._has_upward_motion = False + # crop the depth data and down sample for speedy check + n = get_points_from_fraction(len(self.depth), 0.005) + coarse = self.depth.iloc[self.start.index:self.stop.index:n] + # loop and find any values greater than the current value + for i,v in coarse.items(): + upward = np.any(coarse.loc[i:] > v + 5) + if upward: + self._has_upward_motion = True + break + + return self._has_upward_motion + + +class LyteProfileV6(GenericProfileV6): + """ + Class for managing raw profiles pulled from the probe over USB. This class computes the + depth profile from the raw data + """ + def __init__(self, filename, depth_method='fused', **kwargs): + super().__init__(filename, **kwargs) + self.depth_method = depth_method + # properties + self._accelerometer = None + self._barometer = None + self._motion_detect_name = None # column name containing accel data dir of travel + self._acceleration_names = None # All columns containing accel data + self._moving_time = None # time the probe was moving + self._angle = None + + @staticmethod + def process_df(df): + """ + Migrate all baro depths to filtereddepth and remove ambient + to add NIR column + """ + df = df.rename(columns={'depth': 'filtereddepth'}) + df['nir'] = remove_ambient(df['Sensor3'], df['Sensor2']) + return df + + @classmethod + def from_dataframe(cls, df): + profile = LyteProfileV6(None) + profile._raw = cls.process_df(df) + return profile + + @property + def motion_detect_name(self): + """Return all the names of acceleration columns""" + if self._motion_detect_name is None: + self._motion_detect_name = self.get_motion_name(self.raw.columns) + return self._motion_detect_name + + @property + def acceleration_names(self): + """Return all the names of acceleration columns""" + if self._acceleration_names is None: + self._acceleration_names = self.get_acceleration_columns(self.raw.columns) + return self._acceleration_names + + @property + def acceleration(self): + """ + Retrieve acceleration with gravity removed + """ + # Assign the detection column if it is available + if self._acceleration is None: + if self.motion_detect_name != Sensor.UNAVAILABLE: + # Remove gravity + self._acceleration = get_neutral_bias_at_border(self.raw[self.motion_detect_name]) + else: + self._acceleration = Sensor.UNAVAILABLE + return self._acceleration + + @property + def accelerometer(self): + """Returns a class holding timeseries of accelerometer based depth""" + if self._accelerometer is None: + if self.motion_detect_name == Sensor.UNAVAILABLE: + self._accelerometer = Sensor.UNAVAILABLE + else: + data = pd.DataFrame.from_dict({'time':self.raw['time'], self.acceleration.name:self.acceleration.values}) + data = data.set_index('time')[self.acceleration.name] + self._accelerometer = AccelerometerDepth(data, self.start.index, self.stop.index) + return self._accelerometer + + @property + def barometer(self): + """Returns a class holding timeseries of barometer based depth""" + if self._barometer is None: + baro = self.raw[['time', 'filtereddepth']].set_index('time')['filtereddepth'] + if 'ZPFO' in self.metadata.keys(): + if self.metadata['ZPFO'] < 50: + LOG.info('Filtering barometer data...') + # TODO: make this more intelligent + baro = zfilter(self.raw['filtereddepth'].values, 0.4) + baro = pd.DataFrame.from_dict({'baro':baro, 'time': self.raw['time']}) + baro = baro.set_index('time')['baro'] + + if self.accelerometer != Sensor.UNAVAILABLE: + idx = abs(self.accelerometer.depth - -1).argmin() + else: + idx = self.start.index - return self._angle + angle = None if self.angle == Sensor.UNAVAILABLE else self.angle + self._barometer = BarometerDepth(baro, idx, self.stop.index, angle=angle) - @classmethod - def get_error(cls, acc, acc_range, threshold=0.95): - """Find a likely ACC error""" - idx = acc.abs() >= (threshold * acc_range) - error = None - if np.any(idx): - error = np.argwhere(idx.values)[0][0] - return error + return self._barometer - def __repr__(self): - profile_str = f"LyteProfile (Recorded {len(self.raw):,} points, {self.datetime.isoformat()})" - return profile_str + @property + def depth(self): + if self._depth is None: + if self.motion_detect_name != Sensor.UNAVAILABLE and self.depth_method != 'barometer': + # User requested fused + if self.depth_method == 'fused': + depth = self.fuse_depths(self.accelerometer.depth.values.copy(), + self.barometer.depth.values.copy(), + error=self.error.index) + if depth.min() < -230 and self.accelerometer.depth.min() > -230: + LOG.warning('Fused depth result produced a profile > 250 cm. Defaulting to accelerometer') + self._depth = self.accelerometer.depth + + elif depth.min() < -230 and self.barometer.depth.min() > -230: + LOG.warning('Fused and accelerometer depth resulted in a profile > 230 cm. Defaulting to barometer') + self._depth = self.barometer.depth + else: + self._depth = pd.Series(data=depth, index=self.raw['time']) + # User requested accelerometer + elif self.depth_method == 'accelerometer': + self._depth = self.accelerometer.depth + else: + self._depth = self.barometer.depth + self.assign_event_depths() + + return self._depth + + @property + def time(self): + """Return the sample time data""" + return self.raw['time'] + + @property + def start(self): + """ Return start event """ + if self._start is None: + if self.motion_detect_name != Sensor.UNAVAILABLE: + idx = get_acceleration_start(self.acceleration) + else: + idx = 0 + + self._start = Event(name='start', index=idx, depth=None, time=self.raw['time'].iloc[idx]) + return self._start + + @property + def stop(self): + """ Return stop event """ + if self._stop is None: + if self.motion_detect_name != Sensor.UNAVAILABLE: + backward_accel = get_neutral_bias_at_border(self.raw[self.motion_detect_name], direction='backward') + idx = get_acceleration_stop(backward_accel) + else: + idx = get_nir_stop(self.raw['Sensor3']) + if idx is not None: + self._stop = Event(name='stop', index=idx, depth=None, time=self.raw['time'].iloc[idx]) + else: + self._stop = Event(name='stop', index=0, depth=None, time=self.raw['time'].iloc[0]) + + return self._stop + + @property + def surface(self): + """ + Return surface events for the nir and force which are physically separated by a distance + """ + if self._surface is None: + # Call to populate nir in raw + idx = get_nir_surface(self.raw['Sensor3']) + if idx == 0: + LOG.warning("Unable to find snow surface, defaulting to first data point") + # Event according the NIR sensors + depth = self.depth.iloc[idx] + nir = Event(name='surface', index=idx, depth=depth, time=self.raw['time'].iloc[idx]) + + # Event according to the force sensor + force_surface_depth = depth + self.surface_detection_offset + f_idx = abs(self.depth - force_surface_depth).argmin() + # Retrieve force estimated start + f_start = get_sensor_start(self.raw['Sensor1'], max_threshold=0.02, threshold=-0.02) + f_start = f_start or f_idx + + # If the force start is before the NIR start then adjust + if f_start < self.start.index: + LOG.info(f'Choosing motion start ({self.start.index}) over force start ({f_start})...') + f_idx = self.start.index + force_surface_depth = self.depth.iloc[f_idx] + + elif f_start < f_idx: + LOG.info(f'Choosing force start ({f_start}) over nir derived ({f_idx})...') + f_idx = f_start + force_surface_depth = self.depth.iloc[f_idx] + + force = Event(name='surface', index=f_idx, depth=force_surface_depth, time=self.raw['time'].iloc[f_idx]) + self._surface = SimpleNamespace(name='surface', nir=nir, force=force) + + # Allow surface detection to modify the start if there is conflict. + if nir.time < self.start.time: + self._start = nir + self._start.name = 'start' + + return self._surface + + @property + def ground(self): + """Event for ground detection""" + if self._ground is None: + ground = get_ground_strike(self.raw['Sensor1'], self.stop.index) + if ground is not None: + self._ground = Event(name='ground', index=ground, depth=self.depth.iloc[ground], time=self.raw['time'].iloc[ground]) + else: + self._ground = Event(name='ground', index=None, depth=None, time=None) + + return self._ground + + @property + def error(self): + """ Return error event """ + if self._error is None: + if self.motion_detect_name != Sensor.UNAVAILABLE: + idx = self.get_error(self.raw[self.motion_detect_name], self.metadata['ACC. Range']) + depth = None + if idx is None: + t = None + else: + t = self.raw['time'].iloc[idx] + + else: + idx = None + depth = None + t = None + self._error = Event(name='error', index=idx, depth=depth, time=t) + + return self._error + + @property + def moving_time(self): + """Amount of time the probe was in motion""" + if self._moving_time is None: + self._moving_time = self.stop.time - self.start.time + return self._moving_time + + @property + def avg_velocity(self): + if self._avg_velocity is None: + self._avg_velocity = self.distance_traveled / self.moving_time + return self._avg_velocity + + @staticmethod + def get_motion_name(columns): + """ + Find a column containing acceleration data sometimes called + acceleration or Y-Axis to handle variations in formatting of file + """ + candidates = [c for c in columns if c.lower() in ['acceleration', 'y-axis']] + if candidates: + return candidates[0] + else: + return Sensor.UNAVAILABLE + + @staticmethod + def get_acceleration_columns(columns): + """ + Find a columns containing acceleration data sometimes called + acceleration or X,Y,Z-Axis to handle variations in formatting of file + """ + candidates = [c for c in ['acceleration', 'X-Axis', 'Y-Axis', 'Z-Axis'] if c in columns] + if candidates: + return candidates + else: + return Sensor.UNAVAILABLE + + def report_card(self): + """ + Return a useful string to print about metrics + """ + msg = '| {:<15} {:<20} |\n' + n_chars = int((39 - len(self.filename.name)) / 2) + s = '-'* n_chars + header = f'\n{s} {self.filename.name} {s}\n' + profile_string = header + profile_string += msg.format('Recorded', f'{self.datetime.isoformat()}') + profile_string += msg.format('Points', f'{len(self.raw.index):,}') + profile_string += msg.format('Moving Time', f'{self.moving_time:0.1f} s') + profile_string += msg.format('Avg. Speed', f'{self.avg_velocity:0.0f} cm/s') + profile_string += msg.format('Resolution', f'{self.resolution:0.1f} pts/cm') + profile_string += msg.format('Total Travel', f'{self.distance_traveled:0.1f} cm') + profile_string += msg.format('Snow Depth', f'{self.distance_through_snow:0.1f} cm') + profile_string += msg.format('Ground Strike:', 'True' if self.ground.time is not None else 'False') + profile_string += msg.format('Upward Motion:', "True" if self.has_upward_motion else "False") + profile_string += msg.format('Errors:', f'@ {self.error.time:0.1f} s' if self.error.time is not None else 'None') + + profile_string += '-' * (len(header)-2) + '\n' + return profile_string + + @classmethod + def fuse_depths(cls, acc_depth, baro_depth, error=None): + """ + Function to intelligently fuse together depth timeseries + + """ + # Accelerometer is always solid in the beginning, unknown as we move on in time + weights_acc = np.ones_like(acc_depth) * 100 + weights_baro = np.ones_like(baro_depth) + avg = np.average(np.array([acc_depth, baro_depth]).T, axis=1, + weights=np.array([weights_acc, weights_baro]).T) + scaled_baro = baro_depth.copy() + + if error is not None: + LOG.info("Blending depth timeseries...") + minimum = 0.01 + # Ensure the same starting place + scaled_baro[error:] = scaled_baro[error:] - (scaled_baro[error] - avg[error]) + + # Full reliance on the constrained baro + weights_acc[error:] = minimum + + avg = np.average(np.array([acc_depth, scaled_baro]).T, axis=1, + weights=np.array([weights_acc, weights_baro]).T) + + # The deeper we go the more the baro constrains + baro_bottom = baro_depth.min() + acc_bottom = acc_depth.min() + avg_bottom = avg.min() + + # Scale total + sensor_diff = abs(acc_bottom) - abs(baro_bottom) + delta = 0.572 * abs(acc_bottom) + 0.308 * abs(baro_bottom) + 0.264 * sensor_diff + 8.916 + # delta = (acc_bottom * (5 - scale) + baro_bottom * scale) / 5 + avg = (avg / avg_bottom) * -1*delta + return avg + + @property + def angle(self): + """ + float indicating the angle at the start of a measurement + """ + if self._angle is None and self.acceleration_names != Sensor.UNAVAILABLE: + if 'Y-Axis' in self.acceleration_names: + data = self.raw[self.acceleration_names].iloc[0:self.start.index].mean(axis=0) + magn = data.pow(2).sum()**0.5 + self._angle = np.arccos(abs(data['Y-Axis']) / magn) * 180 / np.pi + else: + self._angle = Sensor.UNAVAILABLE + + return self._angle + + @classmethod + def get_error(cls, acc, acc_range, threshold=0.95): + """Find a likely ACC error""" + idx = acc.abs() >= (threshold * acc_range) + error = None + if np.any(idx): + error = np.argwhere(idx.values)[0][0] + return error + + def __repr__(self): + profile_str = f"LyteProfile (Recorded {len(self.raw):,} points, {self.datetime.isoformat()})" + return profile_str + + +class ProcessedProfileV6(GenericProfileV6): + """ Class for managing profiles that have been depth processed already """ + def __init__(self, filename, **kwargs): + super().__init__(filename, **kwargs) + + @staticmethod + def process_df(df): + """ + Migrate all baro depths to filtereddepth and remove ambient + to add NIR column + """ + df['nir'] = remove_ambient(df['Sensor3'], df['Sensor2']) + return df + + @property + def depth(self): + return self.raw['depth']* -1 + + @property + def start(self): + """ Return start event """ + if self._start is None: + # TODO: PLACEHOLDER + idx = 0 + self._start = Event(name='start', index=idx, depth=self.raw['depth'].iloc[idx], time=None) + return self._start + + @property + def stop(self): + """ Return stop event """ + if self._stop is None: + idx = self.raw.index[-1] + self._stop = Event(name='stop', index=idx, depth=self.raw['depth'].iloc[idx], time=None) + return self._stop + + @property + def surface(self): + """ + Return surface events for the nir and force which are physically separated by a distance + """ + if self._surface is None: + idx = 0 + force = Event(name='force', index=idx, depth=self.raw['depth'].iloc[0], time=None) + nir = Event(name='nir', index=idx, depth=self.raw['depth'].iloc[0], time=None) + self._surface = SimpleNamespace(name='surface', nir=nir, force=force) + + return self._surface \ No newline at end of file diff --git a/tests/data/kaslo.csv b/tests/data/kaslo.csv index a7dc7c1..bada27c 100644 --- a/tests/data/kaslo.csv +++ b/tests/data/kaslo.csv @@ -5,6 +5,7 @@ HARDWARE REVISION = 1 MODEL NUMBER = 3 SAMPLE RATE = 16000 ZPFO = 70 +ACCRANGE = 2 time,Sensor1,Sensor2,Sensor3,depth,acceleration 0.0,4025.0,1745.0,2781.0,-12.9638671875,-0.94446 6.25017269486889e-05,4086.0,1772.0,2793.0,-12.95329906322338,-0.9444130434782608 diff --git a/tests/data/ls_app.csv b/tests/data/ls_app.csv new file mode 100644 index 0000000..9014c48 --- /dev/null +++ b/tests/data/ls_app.csv @@ -0,0 +1,995 @@ +RECORDED = 2024-10-29--20:55:53 +APP VERSION = v0.1.0 +FIRMWARE REVISION = v1.46.1.0 +HARDWARE REVISION = B +MODEL NUMBER = PB3 +SERIAL NO. = 252813070a020004 +packet_id,depth,Sensor1,Sensor2,Sensor3,Sensor4 +0,0.0,3570,7,2045,3623 +1,0.01,3570,7,2045,3623 +2,0.02,3570,7,2051,3637 +3,0.03,3570,7,2051,3637 +4,0.04,3569,7,2060,3592 +5,0.05,3569,7,2060,3592 +6,0.06,3571,7,2074,3567 +7,0.07,3571,7,2074,3567 +8,0.08,3572,6,2086,3598 +9,0.09,3572,6,2086,3598 +10,0.1,3564,6,2098,3633 +11,0.11,3564,6,2098,3633 +12,0.12,3562,6,2111,3635 +13,0.13,3562,6,2111,3635 +14,0.14,3563,6,2128,3617 +15,0.15,3563,6,2128,3617 +16,0.16,3567,6,2146,3574 +17,0.17,3567,6,2146,3574 +18,0.18,3576,7,2169,3520 +19,0.19,3576,7,2169,3520 +20,0.2,3569,7,2190,3492 +21,0.21,3569,7,2190,3492 +22,0.22,3569,7,2212,3462 +23,0.23,3569,7,2212,3462 +24,0.24,3573,6,2235,3459 +25,0.25,3573,6,2235,3459 +26,0.26,3567,7,2255,3504 +27,0.27,3567,7,2255,3504 +28,0.28,3570,6,2284,3501 +29,0.29,3570,6,2284,3501 +30,0.3,3566,6,2307,3516 +31,0.31,3566,6,2307,3516 +32,0.32,3574,6,2331,3545 +33,0.33,3574,6,2331,3545 +34,0.34,3570,6,2351,3543 +35,0.35,3570,6,2351,3543 +36,0.36,3574,7,2384,3540 +37,0.37,3574,7,2384,3540 +38,0.38,3577,7,2413,3531 +39,0.39,3577,7,2413,3531 +40,0.4,3572,7,2440,3530 +41,0.41,3572,7,2440,3530 +42,0.42,3566,6,2470,3577 +43,0.43,3566,6,2470,3577 +44,0.44,3560,6,2500,3569 +45,0.45,3560,6,2500,3569 +46,0.46,3565,6,2531,3544 +47,0.47,3565,6,2531,3544 +48,0.48,3573,6,2560,3521 +49,0.49,3573,6,2560,3521 +50,0.5,3573,7,2588,3510 +51,0.51,3573,7,2588,3510 +52,0.52,3569,7,2619,3504 +53,0.53,3569,7,2619,3504 +54,0.54,3574,7,2651,3470 +55,0.55,3574,7,2651,3470 +56,0.56,3571,7,2683,3435 +57,0.57,3571,7,2683,3435 +58,0.58,3571,6,2719,3489 +59,0.59,3571,6,2719,3489 +60,0.6,3566,6,2760,3510 +61,0.61,3566,6,2760,3510 +62,0.62,3570,6,2799,3491 +63,0.63,3570,6,2799,3491 +64,0.64,3570,6,2836,3472 +65,0.65,3570,6,2836,3472 +66,0.66,3567,7,2870,3532 +67,0.67,3567,7,2870,3532 +68,0.68,3574,7,2911,3504 +69,0.69,3574,7,2911,3504 +70,0.7,3571,7,2947,3443 +71,0.71,3571,7,2947,3443 +72,0.72,3573,7,2981,3380 +73,0.73,3573,7,2981,3380 +74,0.74,3563,7,3008,3380 +75,0.75,3563,7,3008,3380 +76,0.76,3564,7,3040,3369 +77,0.77,3564,7,3040,3369 +78,0.78,3569,6,3067,3313 +79,0.79,3569,6,3067,3313 +80,0.8,3573,6,3087,3311 +81,0.81,3573,6,3087,3311 +82,0.82,3570,8,3102,3322 +83,0.83,3570,8,3102,3322 +84,0.84,3566,7,3119,3328 +85,0.85,3566,7,3119,3328 +86,0.86,3573,8,3141,3329 +87,0.87,3573,8,3141,3329 +88,0.88,3569,8,3153,3316 +89,0.89,3569,8,3153,3316 +90,0.9,3575,8,3163,3302 +91,0.91,3575,8,3163,3302 +92,0.92,3567,8,3174,3328 +93,0.93,3567,8,3174,3328 +94,0.94,3567,7,3176,3292 +95,0.95,3567,7,3176,3292 +96,0.96,3564,7,3178,3219 +97,0.97,3564,7,3178,3219 +98,0.98,3568,8,3177,3221 +99,0.99,3568,8,3177,3221 +100,1.0,3572,7,3178,3278 +101,1.01,3572,7,3178,3278 +102,1.02,3569,8,3178,3281 +103,1.03,3569,8,3178,3281 +104,1.04,3565,9,3173,3293 +105,1.05,3565,9,3173,3293 +106,1.06,3562,9,3183,3302 +107,1.07,3562,9,3183,3302 +108,1.08,3567,8,3196,3358 +109,1.09,3567,8,3196,3358 +110,1.1,3571,8,3197,3395 +111,1.11,3571,8,3197,3395 +112,1.12,3568,9,3205,3340 +113,1.13,3568,9,3205,3340 +114,1.14,3567,9,3194,3271 +115,1.15,3567,9,3194,3271 +116,1.16,3571,9,3210,3316 +117,1.17,3571,9,3210,3316 +118,1.18,3570,10,3216,3289 +119,1.19,3570,10,3216,3289 +120,1.2,3573,11,3226,3261 +121,1.21,3573,11,3226,3261 +122,1.22,3572,11,3234,3254 +123,1.23,3572,11,3234,3254 +124,1.24,3567,11,3242,3243 +125,1.25,3567,11,3242,3243 +126,1.26,3570,11,3239,3279 +127,1.27,3570,11,3239,3279 +128,1.28,3566,10,3248,3287 +129,1.29,3566,10,3248,3287 +130,1.3,3574,11,3261,3256 +131,1.31,3574,11,3261,3256 +132,1.32,3572,11,3252,3245 +133,1.33,3572,11,3252,3245 +134,1.34,3565,11,3255,3263 +135,1.35,3565,11,3255,3263 +136,1.36,3566,11,3251,3207 +137,1.37,3566,11,3251,3207 +138,1.38,3568,11,3233,3189 +139,1.39,3568,11,3233,3189 +140,1.4,3573,10,3227,3186 +141,1.41,3573,10,3227,3186 +142,1.42,3571,10,3226,3229 +143,1.43,3571,10,3226,3229 +144,1.44,3568,10,3216,3229 +145,1.45,3568,10,3216,3229 +146,1.46,3568,10,3213,3268 +147,1.47,3568,10,3213,3268 +148,1.48,3567,10,3215,3275 +149,1.49,3567,10,3215,3275 +150,1.5,3566,10,3209,3298 +151,1.51,3566,10,3209,3298 +152,1.52,3573,11,3207,3342 +153,1.53,3573,11,3207,3342 +154,1.54,3568,11,3197,3307 +155,1.55,3568,11,3197,3307 +156,1.56,3570,11,3221,3278 +157,1.57,3570,11,3221,3278 +158,1.58,3561,10,3264,3260 +159,1.59,3561,10,3264,3260 +160,1.6,3573,10,3296,3271 +161,1.61,3573,10,3296,3271 +162,1.62,3568,9,3287,3250 +163,1.63,3568,9,3287,3250 +164,1.64,3557,9,3278,3245 +165,1.65,3557,9,3278,3245 +166,1.66,3557,10,3262,3267 +167,1.67,3557,10,3262,3267 +168,1.68,3573,10,3264,3280 +169,1.69,3573,10,3264,3280 +170,1.7,3571,10,3273,3295 +171,1.71,3571,10,3273,3295 +172,1.72,3571,9,3284,3279 +173,1.73,3571,9,3284,3279 +174,1.74,3570,9,3283,3325 +175,1.75,3570,9,3283,3325 +176,1.76,3570,8,3278,3347 +177,1.77,3570,8,3278,3347 +178,1.78,3568,7,3274,3314 +179,1.79,3568,7,3274,3314 +180,1.8,3565,8,3288,3298 +181,1.81,3565,8,3288,3298 +182,1.82,3566,8,3305,3282 +183,1.83,3566,8,3305,3282 +184,1.84,3571,8,3315,3318 +185,1.85,3571,8,3315,3318 +186,1.86,3567,8,3318,3303 +187,1.87,3567,8,3318,3303 +188,1.88,3571,9,3322,3250 +189,1.89,3571,9,3322,3250 +190,1.9,3567,8,3311,3210 +191,1.91,3567,8,3311,3210 +192,1.92,3575,8,3273,3206 +193,1.93,3575,8,3273,3206 +194,1.94,3559,8,2844,3214 +195,1.95,3559,8,2844,3214 +196,1.96,3559,8,2172,3321 +197,1.97,3559,8,2172,3321 +198,1.98,3568,8,2381,3431 +199,1.99,3568,8,2381,3431 +200,2.0,3577,8,2791,3446 +201,2.01,3577,8,2791,3446 +202,2.02,3570,8,3097,3386 +203,2.03,3570,8,3097,3386 +204,2.04,3566,8,3299,3304 +205,2.05,3566,8,3299,3304 +206,2.06,3571,9,3354,3260 +207,2.07,3571,9,3354,3260 +208,2.08,3570,8,3379,3263 +209,2.09,3570,8,3379,3263 +210,2.1,3568,8,3392,3278 +211,2.11,3568,8,3392,3278 +212,2.12,3570,8,3399,3236 +213,2.13,3570,8,3399,3236 +214,2.14,3570,8,3402,3202 +215,2.15,3570,8,3402,3202 +216,2.16,3569,8,3411,3181 +217,2.17,3569,8,3411,3181 +218,2.18,3573,8,3424,3199 +219,2.19,3573,8,3424,3199 +220,2.2,3572,8,3415,3183 +221,2.21,3572,8,3415,3183 +222,2.22,3575,8,3409,3208 +223,2.23,3575,8,3409,3208 +224,2.24,3575,8,3418,3261 +225,2.25,3575,8,3418,3261 +226,2.26,3558,8,3416,3302 +227,2.27,3558,8,3416,3302 +228,2.28,3571,8,3410,3291 +229,2.29,3571,8,3410,3291 +230,2.3,3573,8,3441,3307 +231,2.31,3573,8,3441,3307 +232,2.32,3580,8,3448,3299 +233,2.33,3580,8,3448,3299 +234,2.34,3568,8,3474,3291 +235,2.35,3568,8,3474,3291 +236,2.36,3573,8,3493,3234 +237,2.37,3573,8,3493,3234 +238,2.38,3574,8,3511,3203 +239,2.39,3574,8,3511,3203 +240,2.4,3571,8,3529,3178 +241,2.41,3571,8,3529,3178 +242,2.42,3572,8,3532,3171 +243,2.43,3572,8,3532,3171 +244,2.44,3568,7,3526,3157 +245,2.45,3568,7,3526,3157 +246,2.46,3570,8,3538,3148 +247,2.47,3570,8,3538,3148 +248,2.48,3570,8,3539,3148 +249,2.49,3570,8,3539,3148 +250,2.5,3574,8,3541,3125 +251,2.51,3574,8,3541,3125 +252,2.52,3572,8,3508,3098 +253,2.53,3572,8,3508,3098 +254,2.54,3569,8,3371,3092 +255,2.55,3569,8,3371,3092 +256,2.56,3560,8,2338,3158 +257,2.57,3560,8,2338,3158 +258,2.58,3570,8,2400,3314 +259,2.59,3570,8,2400,3314 +260,2.6,3572,7,3027,3324 +261,2.61,3572,7,3027,3324 +262,2.62,3567,8,3420,3225 +263,2.63,3567,8,3420,3225 +264,2.64,3567,7,3508,3161 +265,2.65,3567,7,3508,3161 +266,2.66,3567,8,3510,3190 +267,2.67,3567,8,3510,3190 +268,2.68,3572,8,3529,3183 +269,2.69,3572,8,3529,3183 +270,2.7,3571,8,3541,3122 +271,2.71,3571,8,3541,3122 +272,2.72,3573,8,3559,3077 +273,2.73,3573,8,3559,3077 +274,2.74,3569,9,3553,3046 +275,2.75,3569,9,3553,3046 +276,2.76,3565,8,3542,3038 +277,2.77,3565,8,3542,3038 +278,2.78,3566,8,3562,3003 +279,2.79,3566,8,3562,3003 +280,2.8,3572,8,3555,3028 +281,2.81,3572,8,3555,3028 +282,2.82,3574,8,3552,3053 +283,2.83,3574,8,3552,3053 +284,2.84,3564,8,3548,3052 +285,2.85,3564,8,3548,3052 +286,2.86,3568,8,3534,3027 +287,2.87,3568,8,3534,3027 +288,2.88,3566,8,3518,3084 +289,2.89,3566,8,3518,3084 +290,2.9,3575,8,3470,3106 +291,2.91,3575,8,3470,3106 +292,2.92,3570,8,3473,3114 +293,2.93,3570,8,3473,3114 +294,2.94,3570,8,3496,3145 +295,2.95,3570,8,3496,3145 +296,2.96,3565,8,3537,3134 +297,2.97,3565,8,3537,3134 +298,2.98,3569,8,3564,3148 +299,2.99,3569,8,3564,3148 +300,3.0,3569,8,3553,3180 +301,3.01,3569,8,3553,3180 +302,3.02,3572,8,3128,3166 +303,3.03,3572,8,3128,3166 +304,3.04,3573,8,2098,3331 +305,3.05,3573,8,2098,3331 +306,3.06,3567,8,2880,3400 +307,3.07,3567,8,2880,3400 +308,3.08,3565,8,3454,3355 +309,3.09,3565,8,3454,3355 +310,3.1,3566,8,3569,3228 +311,3.11,3566,8,3569,3228 +312,3.12,3574,8,3573,3157 +313,3.13,3574,8,3573,3157 +314,3.14,3568,7,3594,3106 +315,3.15,3568,7,3594,3106 +316,3.16,3568,8,3600,3079 +317,3.17,3568,8,3600,3079 +318,3.18,3562,8,3601,3067 +319,3.19,3562,8,3601,3067 +320,3.2,3574,8,3612,3048 +321,3.21,3574,8,3612,3048 +322,3.22,3579,8,3644,3032 +323,3.23,3579,8,3644,3032 +324,3.24,3572,8,3657,3061 +325,3.25,3572,8,3657,3061 +326,3.26,3570,8,3647,3060 +327,3.27,3570,8,3647,3060 +328,3.28,3567,7,3604,3039 +329,3.29,3567,7,3604,3039 +330,3.3,3569,8,3575,3033 +331,3.31,3569,8,3575,3033 +332,3.32,3567,8,3559,3068 +333,3.33,3567,8,3559,3068 +334,3.34,3572,8,3521,3085 +335,3.35,3572,8,3521,3085 +336,3.36,3572,8,3491,3105 +337,3.37,3572,8,3491,3105 +338,3.38,3571,8,3480,3139 +339,3.39,3571,8,3480,3139 +340,3.4,3567,8,3513,3153 +341,3.41,3567,8,3513,3153 +342,3.42,3571,8,3517,3202 +343,3.43,3571,8,3517,3202 +344,3.44,3573,8,3067,3240 +345,3.45,3573,8,3067,3240 +346,3.46,3565,7,2046,3379 +347,3.47,3565,7,2046,3379 +348,3.48,3560,8,2845,3406 +349,3.49,3560,8,2845,3406 +350,3.5,3567,8,3364,3357 +351,3.51,3567,8,3364,3357 +352,3.52,3580,8,3468,3322 +353,3.53,3580,8,3468,3322 +354,3.54,3570,8,3563,3221 +355,3.55,3570,8,3563,3221 +356,3.56,3571,8,3607,3173 +357,3.57,3571,8,3607,3173 +358,3.58,3572,8,3621,3167 +359,3.59,3572,8,3621,3167 +360,3.6,3569,8,3646,3172 +361,3.61,3569,8,3646,3172 +362,3.62,3568,8,3659,3117 +363,3.63,3568,8,3659,3117 +364,3.64,3571,8,3679,3098 +365,3.65,3571,8,3679,3098 +366,3.66,3573,8,3704,3094 +367,3.67,3573,8,3704,3094 +368,3.68,3569,8,3700,3097 +369,3.69,3569,8,3700,3097 +370,3.7,3573,8,3689,3105 +371,3.71,3573,8,3689,3105 +372,3.72,3575,8,3646,3098 +373,3.73,3575,8,3646,3098 +374,3.74,3576,8,3568,3099 +375,3.75,3576,8,3568,3099 +376,3.76,3561,8,3558,3102 +377,3.77,3561,8,3558,3102 +378,3.78,3570,8,3650,3077 +379,3.79,3570,8,3650,3077 +380,3.8,3572,7,3677,3085 +381,3.81,3572,7,3677,3085 +382,3.82,3576,8,3626,3088 +383,3.83,3576,8,3626,3088 +384,3.84,3570,8,3569,3099 +385,3.85,3570,8,3569,3099 +386,3.86,3571,8,2898,3168 +387,3.87,3571,8,2898,3168 +388,3.88,3574,8,2441,3310 +389,3.89,3574,8,2441,3310 +390,3.9,3571,8,3193,3335 +391,3.91,3571,8,3193,3335 +392,3.92,3572,8,3560,3277 +393,3.93,3572,8,3560,3277 +394,3.94,3569,7,3590,3219 +395,3.95,3569,7,3590,3219 +396,3.96,3570,7,3515,3187 +397,3.97,3570,7,3515,3187 +398,3.98,3570,7,3427,3178 +399,3.99,3570,7,3427,3178 +400,4.0,3571,7,3533,3175 +401,4.01,3571,7,3533,3175 +402,4.02,3576,8,3602,3165 +403,4.03,3576,8,3602,3165 +404,4.04,3575,8,3631,3137 +405,4.05,3575,8,3631,3137 +406,4.06,3569,8,3686,3150 +407,4.07,3569,8,3686,3150 +408,4.08,3559,8,3674,3164 +409,4.09,3559,8,3674,3164 +410,4.1,3567,8,3660,3177 +411,4.11,3567,8,3660,3177 +412,4.12,3571,8,3655,3172 +413,4.13,3571,8,3655,3172 +414,4.14,3574,8,3661,3137 +415,4.15,3574,8,3661,3137 +416,4.16,3570,9,3649,3180 +417,4.17,3570,9,3649,3180 +418,4.18,3570,8,3647,3191 +419,4.19,3570,8,3647,3191 +420,4.2,3574,9,3639,3173 +421,4.21,3574,9,3639,3173 +422,4.22,3573,8,3639,3160 +423,4.23,3573,8,3639,3160 +424,4.24,3569,9,3624,3159 +425,4.25,3569,9,3624,3159 +426,4.26,3567,8,3600,3201 +427,4.27,3567,8,3600,3201 +428,4.28,3571,8,3480,3158 +429,4.29,3571,8,3480,3158 +430,4.3,3565,8,2685,3219 +431,4.31,3565,8,2685,3219 +432,4.32,3575,7,1966,3388 +433,4.33,3575,7,1966,3388 +434,4.34,3575,8,2822,3495 +435,4.35,3575,8,2822,3495 +436,4.36,3569,8,3416,3400 +437,4.37,3569,8,3416,3400 +438,4.38,3564,8,3570,3253 +439,4.39,3564,8,3570,3253 +440,4.4,3561,8,3492,3185 +441,4.41,3561,8,3492,3185 +442,4.42,3574,7,3456,3157 +443,4.43,3574,7,3456,3157 +444,4.44,3568,8,3553,3144 +445,4.45,3568,8,3553,3144 +446,4.46,3570,8,3610,3139 +447,4.47,3570,8,3610,3139 +448,4.48,3567,8,3648,3133 +449,4.49,3567,8,3648,3133 +450,4.5,3570,8,3675,3166 +451,4.51,3570,8,3675,3166 +452,4.52,3570,9,3659,3112 +453,4.53,3570,9,3659,3112 +454,4.54,3571,8,3659,3073 +455,4.55,3571,8,3659,3073 +456,4.56,3567,9,3636,3088 +457,4.57,3567,9,3636,3088 +458,4.58,3567,8,3661,3109 +459,4.59,3567,8,3661,3109 +460,4.6,3566,8,3663,3145 +461,4.61,3566,8,3663,3145 +462,4.62,3567,8,3664,3114 +463,4.63,3567,8,3664,3114 +464,4.64,3577,8,3665,3116 +465,4.65,3577,8,3665,3116 +466,4.66,3570,8,3638,3111 +467,4.67,3570,8,3638,3111 +468,4.68,3564,9,3687,3125 +469,4.69,3564,9,3687,3125 +470,4.7,3569,8,3705,3130 +471,4.71,3569,8,3705,3130 +472,4.72,3576,8,3724,3084 +473,4.73,3576,8,3724,3084 +474,4.74,3575,8,3705,3080 +475,4.75,3575,8,3705,3080 +476,4.76,3569,8,3718,3079 +477,4.77,3569,8,3718,3079 +478,4.78,3566,7,3726,3078 +479,4.79,3566,7,3726,3078 +480,4.8,3567,8,3708,3078 +481,4.81,3567,8,3708,3078 +482,4.82,3568,8,3607,3121 +483,4.83,3568,8,3607,3121 +484,4.84,3568,8,2853,3178 +485,4.85,3568,8,2853,3178 +486,4.86,3573,7,2181,3326 +487,4.87,3573,7,2181,3326 +488,4.88,3570,8,2714,3355 +489,4.89,3570,8,2714,3355 +490,4.9,3567,8,3268,3314 +491,4.91,3567,8,3268,3314 +492,4.92,3571,8,3619,3222 +493,4.93,3571,8,3619,3222 +494,4.94,3569,8,3708,3156 +495,4.95,3569,8,3708,3156 +496,4.96,3557,8,3732,3120 +497,4.97,3557,8,3732,3120 +498,4.98,3558,8,3769,3076 +499,4.99,3558,8,3769,3076 +500,5.0,3573,8,3787,3048 +501,5.01,3573,8,3787,3048 +502,5.02,3572,8,3755,3015 +503,5.03,3572,8,3755,3015 +504,5.04,3569,8,3712,3011 +505,5.05,3569,8,3712,3011 +506,5.06,3570,8,3718,3013 +507,5.07,3570,8,3718,3013 +508,5.08,3574,8,3729,3007 +509,5.09,3574,8,3729,3007 +510,5.1,3566,8,3739,3010 +511,5.11,3566,8,3739,3010 +512,5.12,3571,8,3755,2987 +513,5.13,3571,8,3755,2987 +514,5.14,3570,8,3773,2966 +515,5.15,3570,8,3773,2966 +516,5.16,3572,8,3753,2947 +517,5.17,3572,8,3753,2947 +518,5.18,3575,8,3765,2975 +519,5.19,3575,8,3765,2975 +520,5.2,3572,8,3745,2963 +521,5.21,3572,8,3745,2963 +522,5.22,3567,8,3724,2981 +523,5.23,3567,8,3724,2981 +524,5.24,3559,8,3719,3049 +525,5.25,3559,8,3719,3049 +526,5.26,3570,7,3720,3050 +527,5.27,3570,7,3720,3050 +528,5.28,3570,8,3730,3048 +529,5.29,3570,8,3730,3048 +530,5.3,3570,8,3728,3037 +531,5.31,3570,8,3728,3037 +532,5.32,3565,8,3717,3041 +533,5.33,3565,8,3717,3041 +534,5.34,3569,8,3707,3052 +535,5.35,3569,8,3707,3052 +536,5.36,3572,8,3682,3025 +537,5.37,3572,8,3682,3025 +538,5.38,3572,9,3671,3007 +539,5.39,3572,9,3671,3007 +540,5.4,3570,8,3676,3043 +541,5.41,3570,8,3676,3043 +542,5.42,3566,8,3668,3158 +543,5.43,3566,8,3668,3158 +544,5.44,3566,8,3682,3173 +545,5.45,3566,8,3682,3173 +546,5.46,3568,8,3697,3139 +547,5.47,3568,8,3697,3139 +548,5.48,3576,8,3716,3102 +549,5.49,3576,8,3716,3102 +550,5.5,3566,8,3731,3157 +551,5.51,3566,8,3731,3157 +552,5.52,3563,8,3731,3111 +553,5.53,3563,8,3731,3111 +554,5.54,3566,9,3727,3081 +555,5.55,3566,9,3727,3081 +556,5.56,3572,9,3709,3062 +557,5.57,3572,9,3709,3062 +558,5.58,3571,7,3587,3046 +559,5.59,3571,7,3587,3046 +560,5.6,3565,8,3108,3094 +561,5.61,3565,8,3108,3094 +562,5.62,3570,7,2457,3188 +563,5.63,3570,7,2457,3188 +564,5.64,3564,8,2151,3324 +565,5.65,3564,8,2151,3324 +566,5.66,3568,8,2445,3448 +567,5.67,3568,8,2445,3448 +568,5.68,3569,8,2860,3438 +569,5.69,3569,8,2860,3438 +570,5.7,3570,9,3188,3353 +571,5.71,3570,9,3188,3353 +572,5.72,3566,8,3449,3281 +573,5.73,3566,8,3449,3281 +574,5.74,3565,8,3652,3280 +575,5.75,3565,8,3652,3280 +576,5.76,3571,8,3771,3223 +577,5.77,3571,8,3771,3223 +578,5.78,3567,8,3804,3158 +579,5.79,3567,8,3804,3158 +580,5.8,3561,8,3803,3086 +581,5.81,3561,8,3803,3086 +582,5.82,3558,9,3798,3099 +583,5.83,3558,9,3798,3099 +584,5.84,3565,8,3809,3125 +585,5.85,3565,8,3809,3125 +586,5.86,3568,8,3801,3085 +587,5.87,3568,8,3801,3085 +588,5.88,3575,8,3802,3092 +589,5.89,3575,8,3802,3092 +590,5.9,3565,8,3795,3101 +591,5.91,3565,8,3795,3101 +592,5.92,3570,8,3783,3110 +593,5.93,3570,8,3783,3110 +594,5.94,3570,8,3784,3073 +595,5.95,3570,8,3784,3073 +596,5.96,3570,7,3767,3043 +597,5.97,3570,7,3767,3043 +598,5.98,3563,8,3762,3078 +599,5.99,3563,8,3762,3078 +600,6.0,3566,7,3756,3131 +601,6.01,3566,7,3756,3131 +602,6.02,3571,8,3741,3171 +603,6.03,3571,8,3741,3171 +604,6.04,3569,8,3737,3154 +605,6.05,3569,8,3737,3154 +606,6.06,3571,8,3737,3143 +607,6.07,3571,8,3737,3143 +608,6.08,3565,8,3740,3120 +609,6.09,3565,8,3740,3120 +610,6.1,3564,8,3731,3109 +611,6.11,3564,8,3731,3109 +612,6.12,3565,8,3725,3086 +613,6.13,3565,8,3725,3086 +614,6.14,3574,8,3723,3093 +615,6.15,3574,8,3723,3093 +616,6.16,3575,8,3712,3102 +617,6.17,3575,8,3712,3102 +618,6.18,3573,8,3708,3081 +619,6.19,3573,8,3708,3081 +620,6.2,3568,8,3705,3073 +621,6.21,3568,8,3705,3073 +622,6.22,3560,8,3703,3067 +623,6.23,3560,8,3703,3067 +624,6.24,3567,8,3698,3087 +625,6.25,3567,8,3698,3087 +626,6.26,3569,8,3673,3125 +627,6.27,3569,8,3673,3125 +628,6.28,3568,8,3665,3125 +629,6.29,3568,8,3665,3125 +630,6.3,3563,8,3651,3117 +631,6.31,3563,8,3651,3117 +632,6.32,3568,8,3640,3127 +633,6.33,3568,8,3640,3127 +634,6.34,3568,8,3609,3149 +635,6.35,3568,8,3609,3149 +636,6.36,3572,8,3595,3161 +637,6.37,3572,8,3595,3161 +638,6.38,3572,8,3589,3200 +639,6.39,3572,8,3589,3200 +640,6.4,3570,8,3588,3207 +641,6.41,3570,8,3588,3207 +642,6.42,3566,8,3567,3237 +643,6.43,3566,8,3567,3237 +644,6.44,3567,8,3562,3214 +645,6.45,3567,8,3562,3214 +646,6.46,3574,7,3572,3173 +647,6.47,3574,7,3572,3173 +648,6.48,3572,8,3590,3189 +649,6.49,3572,8,3590,3189 +650,6.5,3566,7,3607,3177 +651,6.51,3566,7,3607,3177 +652,6.52,3563,8,3621,3166 +653,6.53,3563,8,3621,3166 +654,6.54,3571,8,3640,3138 +655,6.55,3571,8,3640,3138 +656,6.56,3572,8,3653,3156 +657,6.57,3572,8,3653,3156 +658,6.58,3571,8,3650,3131 +659,6.59,3571,8,3650,3131 +660,6.6,3566,8,3652,3080 +661,6.61,3566,8,3652,3080 +662,6.62,3568,7,3660,3014 +663,6.63,3568,7,3660,3014 +664,6.64,3568,7,3676,3041 +665,6.65,3568,7,3676,3041 +666,6.66,3565,8,3689,3044 +667,6.67,3565,8,3689,3044 +668,6.68,3572,8,3699,3061 +669,6.69,3572,8,3699,3061 +670,6.7,3570,8,3706,3028 +671,6.71,3570,8,3706,3028 +672,6.72,3569,8,3709,3026 +673,6.73,3569,8,3709,3026 +674,6.74,3564,8,3710,3069 +675,6.75,3564,8,3710,3069 +676,6.76,3571,7,3717,3132 +677,6.77,3571,7,3717,3132 +678,6.78,3569,7,3722,3123 +679,6.79,3569,7,3722,3123 +680,6.8,3559,8,3724,3097 +681,6.81,3559,8,3724,3097 +682,6.82,3557,8,3728,3085 +683,6.83,3557,8,3728,3085 +684,6.84,3570,8,3722,3101 +685,6.85,3570,8,3722,3101 +686,6.86,3573,8,3721,3112 +687,6.87,3573,8,3721,3112 +688,6.88,3570,8,3718,3116 +689,6.89,3570,8,3718,3116 +690,6.9,3570,8,3721,3087 +691,6.91,3570,8,3721,3087 +692,6.92,3565,8,3723,3064 +693,6.93,3565,8,3723,3064 +694,6.94,3568,8,3724,3029 +695,6.95,3568,8,3724,3029 +696,6.96,3565,8,3721,2960 +697,6.97,3565,8,3721,2960 +698,6.98,3562,8,3716,2987 +699,6.99,3562,8,3716,2987 +700,7.0,3570,8,3697,3004 +701,7.01,3570,8,3697,3004 +702,7.02,3567,8,3673,2984 +703,7.03,3567,8,3673,2984 +704,7.04,3571,8,3673,2988 +705,7.05,3571,8,3673,2988 +706,7.06,3570,9,3667,2974 +707,7.07,3570,9,3667,2974 +708,7.08,3575,8,3665,3010 +709,7.09,3575,8,3665,3010 +710,7.1,3560,8,3657,3034 +711,7.11,3560,8,3657,3034 +712,7.12,3560,8,3660,2987 +713,7.13,3560,8,3660,2987 +714,7.14,3559,8,3666,2981 +715,7.15,3559,8,3666,2981 +716,7.16,3569,8,3678,3026 +717,7.17,3569,8,3678,3026 +718,7.18,3572,9,3696,3056 +719,7.19,3572,9,3696,3056 +720,7.2,3568,8,3698,3020 +721,7.21,3568,8,3698,3020 +722,7.22,3570,8,3690,3021 +723,7.23,3570,8,3690,3021 +724,7.24,3569,8,3660,3043 +725,7.25,3569,8,3660,3043 +726,7.26,3568,8,3578,3026 +727,7.27,3568,8,3578,3026 +728,7.28,3563,8,3368,3031 +729,7.29,3563,8,3368,3031 +730,7.3,3568,7,3065,3084 +731,7.31,3568,7,3065,3084 +732,7.32,3567,7,2751,3197 +733,7.33,3567,7,2751,3197 +734,7.34,3570,8,2485,3278 +735,7.35,3570,8,2485,3278 +736,7.36,3570,8,2344,3391 +737,7.37,3570,8,2344,3391 +738,7.38,3578,8,2352,3448 +739,7.39,3578,8,2352,3448 +740,7.4,3575,8,2483,3515 +741,7.41,3575,8,2483,3515 +742,7.42,3555,8,2647,3501 +743,7.43,3555,8,2647,3501 +744,7.44,3564,7,2828,3481 +745,7.45,3564,7,2828,3481 +746,7.46,3563,8,2988,3391 +747,7.47,3563,8,2988,3391 +748,7.48,3573,7,3127,3308 +749,7.49,3573,7,3127,3308 +750,7.5,3570,8,3251,3264 +751,7.51,3570,8,3251,3264 +752,7.52,3567,8,3369,3227 +753,7.53,3567,8,3369,3227 +754,7.54,3570,8,3475,3201 +755,7.55,3570,8,3475,3201 +756,7.56,3570,8,3565,3185 +757,7.57,3570,8,3565,3185 +758,7.58,3569,8,3652,3132 +759,7.59,3569,8,3652,3132 +760,7.6,3567,7,3697,3093 +761,7.61,3567,7,3697,3093 +762,7.62,3568,8,3679,3069 +763,7.63,3568,8,3679,3069 +764,7.64,3565,8,3669,3015 +765,7.65,3565,8,3669,3015 +766,7.66,3573,9,3678,3085 +767,7.67,3573,9,3678,3085 +768,7.68,3574,9,3682,3074 +769,7.69,3574,9,3682,3074 +770,7.7,3576,8,3679,3022 +771,7.71,3576,8,3679,3022 +772,7.72,3569,9,3673,3019 +773,7.73,3569,9,3673,3019 +774,7.74,3566,8,3664,3022 +775,7.75,3566,8,3664,3022 +776,7.76,3558,8,3658,3058 +777,7.77,3558,8,3658,3058 +778,7.78,3570,8,3653,3013 +779,7.79,3570,8,3653,3013 +780,7.8,3572,8,3646,2987 +781,7.81,3572,8,3646,2987 +782,7.82,3572,8,3633,3047 +783,7.83,3572,8,3633,3047 +784,7.84,3569,8,3628,3105 +785,7.85,3569,8,3628,3105 +786,7.86,3569,9,3619,3088 +787,7.87,3569,9,3619,3088 +788,7.88,3572,9,3617,3104 +789,7.89,3572,9,3617,3104 +790,7.9,3568,9,3609,3133 +791,7.91,3568,9,3609,3133 +792,7.92,3565,9,3610,3106 +793,7.93,3565,9,3610,3106 +794,7.94,3566,8,3610,3096 +795,7.95,3566,8,3610,3096 +796,7.96,3569,8,3609,3063 +797,7.97,3569,8,3609,3063 +798,7.98,3571,8,3614,3064 +799,7.99,3571,8,3614,3064 +800,8.0,3567,8,3626,3044 +801,8.01,3567,8,3626,3044 +802,8.02,3575,8,3641,3065 +803,8.03,3575,8,3641,3065 +804,8.04,3573,8,3649,3028 +805,8.05,3573,8,3649,3028 +806,8.06,3571,9,3657,3014 +807,8.07,3571,9,3657,3014 +808,8.08,3558,9,3661,3035 +809,8.09,3558,9,3661,3035 +810,8.1,3565,7,3667,3056 +811,8.11,3565,7,3667,3056 +812,8.12,3569,8,3671,3028 +813,8.13,3569,8,3671,3028 +814,8.14,3573,8,3671,3024 +815,8.15,3573,8,3671,3024 +816,8.16,3569,8,3676,3044 +817,8.17,3569,8,3676,3044 +818,8.18,3565,9,3679,2985 +819,8.19,3565,9,3679,2985 +820,8.2,3568,8,3683,2941 +821,8.21,3568,8,3683,2941 +822,8.22,3570,8,3683,2958 +823,8.23,3570,8,3683,2958 +824,8.24,3574,8,3686,2972 +825,8.25,3574,8,3686,2972 +826,8.26,3566,7,3688,3000 +827,8.27,3566,7,3688,3000 +828,8.28,3571,8,3688,2986 +829,8.29,3571,8,3688,2986 +830,8.3,3567,8,3685,2979 +831,8.31,3567,8,3685,2979 +832,8.32,3569,8,3683,2969 +833,8.33,3569,8,3683,2969 +834,8.34,3573,9,3689,2981 +835,8.35,3573,9,3689,2981 +836,8.36,3572,8,3690,2977 +837,8.37,3572,8,3690,2977 +838,8.38,3572,8,3692,3035 +839,8.39,3572,8,3692,3035 +840,8.4,3566,7,3691,3098 +841,8.41,3566,7,3691,3098 +842,8.42,3560,8,3688,3139 +843,8.43,3560,8,3688,3139 +844,8.44,3566,8,3685,3133 +845,8.45,3566,8,3685,3133 +846,8.46,3572,7,3687,3091 +847,8.47,3572,7,3687,3091 +848,8.48,3567,8,3682,3013 +849,8.49,3567,8,3682,3013 +850,8.5,3567,8,3686,3014 +851,8.51,3567,8,3686,3014 +852,8.52,3570,9,3683,3016 +853,8.53,3570,9,3683,3016 +854,8.54,3569,8,3680,2985 +855,8.55,3569,8,3680,2985 +856,8.56,3569,8,3679,2963 +857,8.57,3569,8,3679,2963 +858,8.58,3566,8,3682,2988 +859,8.59,3566,8,3682,2988 +860,8.6,3564,8,3675,2975 +861,8.61,3564,8,3675,2975 +862,8.62,3566,8,3673,2941 +863,8.63,3566,8,3673,2941 +864,8.64,3571,8,3676,2939 +865,8.65,3571,8,3676,2939 +866,8.66,3573,9,3678,2970 +867,8.67,3573,9,3678,2970 +868,8.68,3568,8,3679,3017 +869,8.69,3568,8,3679,3017 +870,8.7,3563,8,3675,3004 +871,8.71,3563,8,3675,3004 +872,8.72,3562,8,3676,3009 +873,8.73,3562,8,3676,3009 +874,8.74,3567,8,3678,3045 +875,8.75,3567,8,3678,3045 +876,8.76,3564,8,3674,3045 +877,8.77,3564,8,3674,3045 +878,8.78,3569,7,3673,3039 +879,8.79,3569,7,3673,3039 +880,8.8,3564,8,3671,3000 +881,8.81,3564,8,3671,3000 +882,8.82,3568,8,3672,3000 +883,8.83,3568,8,3672,3000 +884,8.84,3567,9,3672,2972 +885,8.85,3567,9,3672,2972 +886,8.86,3571,8,3670,2922 +887,8.87,3571,8,3670,2922 +888,8.88,3567,8,3669,2906 +889,8.89,3567,8,3669,2906 +890,8.9,3568,8,3670,2974 +891,8.91,3568,8,3670,2974 +892,8.92,3565,8,3667,3003 +893,8.93,3565,8,3667,3003 +894,8.94,3566,8,3665,2964 +895,8.95,3566,8,3665,2964 +896,8.96,3566,7,3664,2966 +897,8.97,3566,7,3664,2966 +898,8.98,3572,8,3665,2976 +899,8.99,3572,8,3665,2976 +900,9.0,3573,8,3668,2997 +901,9.01,3573,8,3668,2997 +902,9.02,3563,8,3662,3038 +903,9.03,3563,8,3662,3038 +904,9.04,3562,8,3662,3034 +905,9.05,3562,8,3662,3034 +906,9.06,3562,8,3661,3074 +907,9.07,3562,8,3661,3074 +908,9.08,3578,8,3663,3094 +909,9.09,3578,8,3663,3094 +910,9.1,3566,8,3657,3074 +911,9.11,3566,8,3657,3074 +912,9.12,3565,8,3657,3035 +913,9.13,3565,8,3657,3035 +914,9.14,3563,8,3658,3018 +915,9.15,3563,8,3658,3018 +916,9.16,3570,8,3659,2979 +917,9.17,3570,8,3659,2979 +918,9.18,3567,8,3659,3014 +919,9.19,3567,8,3659,3014 +920,9.2,3566,8,3659,3043 +921,9.21,3566,8,3659,3043 +922,9.22,3569,8,3661,3054 +923,9.23,3569,8,3661,3054 +924,9.24,3564,8,3658,3101 +925,9.25,3564,8,3658,3101 +926,9.26,3569,8,3657,3122 +927,9.27,3569,8,3657,3122 +928,9.28,3566,8,3657,3115 +929,9.29,3566,8,3657,3115 +930,9.3,3571,8,3658,3084 +931,9.31,3571,8,3658,3084 +932,9.32,3567,8,3658,3115 +933,9.33,3567,8,3658,3115 +934,9.34,3560,9,3656,3131 +935,9.35,3560,9,3656,3131 +936,9.36,3566,8,3657,3101 +937,9.37,3566,8,3657,3101 +938,9.38,3571,8,3657,3074 +939,9.39,3571,8,3657,3074 +940,9.4,3576,8,3660,3051 +941,9.41,3576,8,3660,3051 +942,9.42,3567,8,3656,3030 +943,9.43,3567,8,3656,3030 +944,9.44,3569,8,3657,3017 +945,9.45,3569,8,3657,3017 +946,9.46,3561,8,3656,3021 +947,9.47,3561,8,3656,3021 +948,9.48,3568,7,3657,3049 +949,9.49,3568,7,3657,3049 +950,9.5,3566,8,3657,3051 +951,9.51,3566,8,3657,3051 +952,9.52,3571,8,3658,3043 +953,9.53,3571,8,3658,3043 +954,9.54,3572,8,3658,3068 +955,9.55,3572,8,3658,3068 +956,9.56,3561,8,3652,3093 +957,9.57,3561,8,3652,3093 +958,9.58,3561,8,3652,3122 +959,9.59,3561,8,3652,3122 +960,9.6,3570,8,3654,3088 +961,9.61,3570,8,3654,3088 +962,9.62,3569,8,3655,3080 +963,9.63,3569,8,3655,3080 +964,9.64,3560,8,3653,3061 +965,9.65,3560,8,3653,3061 +966,9.66,3555,8,3652,3084 +967,9.67,3555,8,3652,3084 +968,9.68,3568,8,3652,3098 +969,9.69,3568,8,3652,3098 +970,9.7,3574,8,3653,3047 +971,9.71,3574,8,3653,3047 +972,9.72,3569,8,3650,3097 +973,9.73,3569,8,3650,3097 +974,9.74,3570,8,3648,3134 +975,9.75,3570,8,3648,3134 +976,9.76,3567,8,3643,3135 +977,9.77,3567,8,3643,3135 +978,9.78,3567,8,3643,3104 +979,9.79,3567,8,3643,3104 +980,9.8,3567,8,3639,3094 +981,9.81,3567,8,3639,3094 +982,9.82,3562,8,3637,3130 +983,9.83,3562,8,3637,3130 +984,9.84,3573,9,3637,3161 +985,9.85,3573,9,3637,3161 +986,9.86,3565,9,3630,3107 +987,9.87,3565,9,3630,3107 diff --git a/tests/test_adjustments.py b/tests/test_adjustments.py index 516d675..98df124 100644 --- a/tests/test_adjustments.py +++ b/tests/test_adjustments.py @@ -123,10 +123,9 @@ def test_merge_time_series(data_list, expected): @pytest.mark.parametrize('active, ambient, min_ambient_range, expected', [ # Test normal situation with ambient present - ([200, 200, 400, 1000], [200, 200, 0, 0], 100, [0, 0, 300, 1000]), + ([200, 200, 400, 1000], [200, 200, 50, 50], 100, [1.0, 1.0, 275, 1000]), # Test no cleaning required - ([200, 200, 400, 400], [210, 210, 200, 200], 90, [200, 200, 400, 400]) - + # ([200, 200, 400, 400], [210, 210, 200, 200], 90, [200, 200, 400, 400]) ]) def test_remove_ambient(active, ambient, min_ambient_range, expected): """ @@ -138,6 +137,10 @@ def test_remove_ambient(active, ambient, min_ambient_range, expected): result = remove_ambient(active, ambient, min_ambient_range=100) np.testing.assert_equal(result.values, expected) +# @pytest.mark.parametrize('fname', ['2024-01-31--104419.csv']) +# def test_remove_ambient_real(raw_df, fname): +# remove_ambient(raw_df['Sensor3'], raw_df['Sensor2']) + @pytest.mark.parametrize('data, coefficients, expected', [ ([1, 2, 3, 4], [2, 0], [2, 4, 6, 8]) ]) @@ -172,7 +175,7 @@ def test_aggregate_by_depth(data, depth, new_depth, resolution, agg_method, expe expected = pd.DataFrame.from_dict(exp) - pd.testing.assert_frame_equal(result, expected, check_dtype=False) + pd.testing.assert_frame_equal(result, expected, check_dtype=False, check_like=True) @pytest.mark.skip('Function not ready') @pytest.mark.parametrize('data, method, expected', [ diff --git a/tests/test_cropping.py b/tests/test_cropping.py index e6b461b..76c88e7 100644 --- a/tests/test_cropping.py +++ b/tests/test_cropping.py @@ -15,21 +15,4 @@ def test_crop_to_motion(raw_df, fname, start_kwargs, stop_kwargs, expected_time_ df = crop_to_motion(raw_df, start_kwargs=start_kwargs, stop_kwargs=stop_kwargs) delta_t = df.index.max() - df.index.min() - assert pytest.approx(delta_t, abs=0.02) == expected_time_delta - - -@pytest.mark.parametrize('active, ambient, cropped_values', [ - ([100, 100, 120, 300, 300], [200, 200, 200, 100, 100], [120, 300, 300]), -]) -def test_crop_to_snow(active, ambient, cropped_values): - """ - Test that the dataframe is cropped correctly according to motion - then compare with the timing - """ - data = {'time': np.linspace(0, 1, len(active)), - 'active': np.array(active), - 'ambient': np.array(ambient)} - df = pd.DataFrame(data) - expected = np.array(cropped_values) - cropped = crop_to_snow(df, active_col='active', ambient_col='ambient') - np.testing.assert_array_equal(cropped['active'].values, expected) + assert pytest.approx(delta_t, abs=0.02) == expected_time_delta \ No newline at end of file diff --git a/tests/test_detect.py b/tests/test_detect.py index 752338d..5aa584a 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -139,25 +139,10 @@ def test_get_acceleration_stop_time_index(raw_df): assert idx1 == idx2 -@pytest.mark.parametrize("active, threshold, max_threshold, expected", [ - # Typical bright->dark ambient - ([0, 200, 3000, 4000], 0.01, 0.1, 1), - # no ambient change ( dark or super cloudy) - ([1000, 1100, 2000, 3000], .01, 0.2, 1), - # No surface detectable but all the values meet criteria - ([1000,1010,9990,1010],-0.01,0.2,0) -]) -def test_get_nir_surface(active, threshold, max_threshold, expected): - idx = get_nir_surface(pd.Series(active), - threshold=threshold, - max_threshold=max_threshold) - assert idx == expected - - @pytest.mark.parametrize('fname, surface_idx', [ ('bogus.csv', 20385), ('pilots.csv', 9496), - ('hard_surface_hard_stop.csv', 10167), + ('hard_surface_hard_stop.csv', 8515), # No Ambient with tester stick ('tester_stick.csv', 9887), # Noise Ambient @@ -166,15 +151,14 @@ def test_get_nir_surface(active, threshold, max_threshold, expected): ('toolik.csv', 13684), ('banner_legacy.csv', 8177), # Get surface with challenging ambient conditions - ('egrip_tough_surface.csv', 31551), - + ('egrip_tough_surface.csv', 29964), ]) def test_get_nir_surface_real(raw_df, fname, surface_idx): """ Test surface with real data """ - clean = remove_ambient(raw_df['Sensor3'], raw_df['Sensor2']) - result = get_nir_surface(clean) + # clean = remove_ambient(raw_df['Sensor3'], raw_df['Sensor2']) + result = get_nir_surface(raw_df['Sensor3']) # Ensure within 3% of original answer all the time. assert pytest.approx(surface_idx, abs=int(0.02 * len(raw_df.index))) == result diff --git a/tests/test_profile.py b/tests/test_profile.py index c5e9f82..4a3ed2d 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -1,6 +1,6 @@ import pytest from os.path import join -from study_lyte.profile import LyteProfileV6, Sensor +from study_lyte.profile import ProcessedProfileV6, LyteProfileV6, Sensor from operator import attrgetter from shapely.geometry import Point @@ -27,7 +27,7 @@ def test_stop_property(self, profile, filename, depth_method, expected): @pytest.mark.parametrize('filename, depth_method, expected', [ - ('kaslo.csv', 'fused', 12479) + ('kaslo.csv', 'fused', 11641) ]) def test_nir_surface_property(self, profile, filename, depth_method, expected): nir_surface = profile.surface.nir.index @@ -81,11 +81,14 @@ def test_pressure_profile(self, profile, filename, depth_method, expected_points @pytest.mark.parametrize('filename, depth_method, expected_points, mean_nir', [ - ('kaslo.csv', 'fused', 14799, 2863) + ('kaslo.csv', 'fused', 14799, 2638) ]) def test_nir_profile(self, profile, filename, depth_method, expected_points, mean_nir): - assert pytest.approx(len(profile.nir), len(profile.raw)*0.05) == expected_points - assert pytest.approx(profile.nir['nir'].mean(), abs=50) == mean_nir + # Assert the len of the nir profile within 5% + assert pytest.approx(len(profile.nir), len(profile.raw) * 0.05) == expected_points + # Use the median as an approximation for the processing + assert pytest.approx(profile.nir['nir'].median(), abs=50) == mean_nir + @pytest.mark.parametrize('filename, depth_method, expected', [ # Serious angle ('angled_measurement.csv', 'fused', 34), @@ -233,6 +236,7 @@ def test_stop_wo_accel(self, profile): def test_surface(self, profile): assert pytest.approx(profile.surface.nir.index, int(0.01*len(profile.depth))) == 7970 + @pytest.mark.parametrize('fname, attribute, expected_value', [ ('pilots_error.csv', 'surface.force.index', 5758) ]) @@ -240,6 +244,7 @@ def test_force_start_alternate(lyte_profile, fname, attribute, expected_value): result = attrgetter(attribute)(lyte_profile) assert pytest.approx(result, int(0.01 * len(lyte_profile.raw))) == expected_value + @pytest.mark.parametrize("fname, expected", [("open_air.csv", 0)]) def test_surface_indexer_error(lyte_profile, fname, expected): """ @@ -248,3 +253,12 @@ def test_surface_indexer_error(lyte_profile, fname, expected): """ assert lyte_profile.surface.nir.index == expected assert not lyte_profile.nir.empty + + +@pytest.mark.skip('Incomplete work') +def test_app(data_dir): + """Functionality test""" + fname = data_dir + '/ls_app.csv' + profile = ProcessedProfileV6(fname) + print(profile) + assert False # TODO: Add more detailed checking \ No newline at end of file