From 6740eb3fdd619a76db4ff42f93adce0e58803578 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Fri, 26 Sep 2025 14:18:51 -0700 Subject: [PATCH 1/8] refactor: bump to 2.0, upgrade test examples, modify QC dataframe helper --- pyproject.toml | 2 +- .../helpers/data_schema.py | 22 +- tests/resources/helpers/quality_control.json | 453 +++++++++++------- .../helpers/quality_control_invalid.json | 2 +- 4 files changed, 290 insertions(+), 189 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index db5470e..5654895 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ rds = [ "SQLAlchemy==1.4.49" ] helpers = [ - "aind-data-schema>=1.1.0,<2.0", + "aind-data-schema>=2.0", "pandas", ] full = [ diff --git a/src/aind_data_access_api/helpers/data_schema.py b/src/aind_data_access_api/helpers/data_schema.py index 184a661..7822b05 100644 --- a/src/aind_data_access_api/helpers/data_schema.py +++ b/src/aind_data_access_api/helpers/data_schema.py @@ -137,15 +137,14 @@ def get_quality_control_status_df( for name, qc in zip(names, qcs): qc_metrics_flat = {} qc_metrics_flat["name"] = name - for eval in qc.evaluations: - for metric in eval.metrics: - # Find the most recent status before the given datetime - for status in reversed(metric.status_history): - if status.timestamp <= date: - qc_metrics_flat[f"{eval.name}_{metric.name}"] = ( - status.status - ) - break + for metric in qc.metrics: + # Find the most recent status before the given datetime + for status in reversed(metric.status_history): + if status.timestamp <= date: + qc_metrics_flat[f"{eval.name}_{metric.name}"] = ( + status.status + ) + break data.append(qc_metrics_flat) @@ -175,9 +174,8 @@ def get_quality_control_value_df( for name, qc in zip(names, qcs): qc_metrics_flat = {} qc_metrics_flat["name"] = name - for eval in qc.evaluations: - for metric in eval.metrics: - qc_metrics_flat[f"{eval.name}_{metric.name}"] = metric.value + for metric in qc.metrics: + qc_metrics_flat[f"{eval.name}_{metric.name}"] = metric.value data.append(qc_metrics_flat) diff --git a/tests/resources/helpers/quality_control.json b/tests/resources/helpers/quality_control.json index 83bf6aa..0a95cbd 100644 --- a/tests/resources/helpers/quality_control.json +++ b/tests/resources/helpers/quality_control.json @@ -1,176 +1,279 @@ { - "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/quality_control.py", - "schema_version": "1.1.1", - "evaluations": [ - { - "modality": { - "name": "Extracellular electrophysiology", - "abbreviation": "ecephys" - }, - "stage": "Raw data", - "name": "Drift map", - "description": "Qualitative check that drift map shows minimal movement", - "metrics": [ - { - "name": "Probe A drift", - "value": { - "value": "", - "options": [ - "Low", - "Medium", - "High" - ], - "status": [ - "Pass", - "Fail", - "Fail" - ], - "type": "dropdown" - }, - "description": null, - "reference": "ecephys-drift-map", - "status_history": [ - { - "evaluator": "", - "status": "Pending", - "timestamp": "2022-11-22T00:00:00Z" - } - ] - }, - { - "name": "Probe B drift", - "value": { - "value": "", - "options": [ - "Drift visible in entire session", - "Drift visible in part of session", - "Sudden movement event" - ], - "status": [ - "Fail", - "Pass", - "Fail" - ], - "type": "checkbox" - }, - "description": null, - "reference": "ecephys-drift-map", - "status_history": [ - { - "evaluator": "", - "status": "Pending", - "timestamp": "2022-11-22T00:00:00Z" - } - ] - }, - { - "name": "Probe C drift", - "value": "Low", - "description": null, - "reference": "ecephys-drift-map", - "status_history": [ - { - "evaluator": "Automated", - "status": "Pass", - "timestamp": "2022-11-22T00:00:00Z" - } - ] - } - ], - "notes": "", - "allow_failed_metrics": false - }, - { - "modality": { - "name": "Behavior videos", - "abbreviation": "behavior-videos" - }, - "stage": "Raw data", - "name": "Video frame count check", - "description": null, - "metrics": [ - { - "name": "video_1_num_frames", - "value": 662, - "description": null, - "reference": null, - "status_history": [ - { - "evaluator": "Automated", - "status": "Pass", - "timestamp": "2022-11-22T00:00:00Z" - } - ] - }, - { - "name": "video_2_num_frames", - "value": 662, - "description": null, - "reference": null, - "status_history": [ - { - "evaluator": "Automated", - "status": "Pass", - "timestamp": "2022-11-22T00:00:00Z" - } - ] - } - ], - "notes": "Pass when video_1_num_frames==video_2_num_frames", - "allow_failed_metrics": false - }, - { - "modality": { - "name": "Extracellular electrophysiology", - "abbreviation": "ecephys" - }, - "stage": "Raw data", - "name": "Probes present", - "description": null, - "metrics": [ - { - "name": "ProbeA_success", - "value": true, - "description": null, - "reference": null, - "status_history": [ - { - "evaluator": "Automated", - "status": "Pass", - "timestamp": "2022-11-22T00:00:00Z" - } - ] - }, - { - "name": "ProbeB_success", - "value": true, - "description": null, - "reference": null, - "status_history": [ - { - "evaluator": "Automated", - "status": "Pass", - "timestamp": "2022-11-22T00:00:00Z" - } - ] - }, - { - "name": "ProbeC_success", - "value": true, - "description": null, - "reference": null, - "status_history": [ - { - "evaluator": "Automated", - "status": "Pass", - "timestamp": "2022-11-22T00:00:00Z" - } - ] - } - ], - "notes": null, - "allow_failed_metrics": false - } - ], - "notes": null - } \ No newline at end of file + "object_type": "Quality control", + "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/quality_control.py", + "schema_version": "2.0.6", + "metrics": [ + { + "object_type": "QC metric", + "name": "Probe A drift", + "modality": { + "name": "Extracellular electrophysiology", + "abbreviation": "ecephys" + }, + "stage": "Raw data", + "value": { + "value": "", + "options": [ + "Low", + "Medium", + "High" + ], + "status": [ + "Pass", + "Fail", + "Fail" + ], + "type": "dropdown" + }, + "status_history": [ + { + "object_type": "QC status", + "evaluator": "", + "status": "Pending", + "timestamp": "2022-11-22T00:00:00Z" + } + ], + "description": "Pass when drift map shows minimal movement", + "reference": "ecephys-drift-map", + "tags": [ + "Drift map", + "Probe A" + ], + "evaluated_assets": null + }, + { + "object_type": "QC metric", + "name": "Probe B drift", + "modality": { + "name": "Extracellular electrophysiology", + "abbreviation": "ecephys" + }, + "stage": "Raw data", + "value": { + "value": "", + "options": [ + "No Drift", + "Drift visible in part of acquisition", + "Drift visible in entire acquisition", + "Sudden movement event" + ], + "status": [ + "Pass", + "Pass", + "Fail", + "Fail" + ], + "type": "checkbox" + }, + "status_history": [ + { + "object_type": "QC status", + "evaluator": "", + "status": "Pending", + "timestamp": "2022-11-22T00:00:00Z" + } + ], + "description": "Pass when drift map shows minimal movement", + "reference": "ecephys-drift-map", + "tags": [ + "Drift map", + "Probe B" + ], + "evaluated_assets": null + }, + { + "object_type": "QC metric", + "name": "Probe C drift", + "modality": { + "name": "Extracellular electrophysiology", + "abbreviation": "ecephys" + }, + "stage": "Raw data", + "value": "Low", + "status_history": [ + { + "object_type": "QC status", + "evaluator": "Automated", + "status": "Pass", + "timestamp": "2022-11-22T00:00:00Z" + } + ], + "description": "Pass when drift map shows minimal movement", + "reference": "ecephys-drift-map", + "tags": [ + "Drift map", + "Probe C" + ], + "evaluated_assets": null + }, + { + "object_type": "QC metric", + "name": "Expected frame count", + "modality": { + "name": "Behavior videos", + "abbreviation": "behavior-videos" + }, + "stage": "Raw data", + "value": 662, + "status_history": [ + { + "object_type": "QC status", + "evaluator": "Automated", + "status": "Pass", + "timestamp": "2022-11-22T00:00:00Z" + } + ], + "description": "Expected frame count from experiment length, always pass", + "reference": null, + "tags": [ + "Frame count checks" + ], + "evaluated_assets": null + }, + { + "object_type": "QC metric", + "name": "Video 1 frame count", + "modality": { + "name": "Behavior videos", + "abbreviation": "behavior-videos" + }, + "stage": "Raw data", + "value": 662, + "status_history": [ + { + "object_type": "QC status", + "evaluator": "Automated", + "status": "Pass", + "timestamp": "2022-11-22T00:00:00Z" + } + ], + "description": "Pass when frame count matches expected", + "reference": null, + "tags": [ + "Frame count checks", + "Video 1" + ], + "evaluated_assets": null + }, + { + "object_type": "QC metric", + "name": "Video 2 num frames", + "modality": { + "name": "Behavior videos", + "abbreviation": "behavior-videos" + }, + "stage": "Raw data", + "value": 662, + "status_history": [ + { + "object_type": "QC status", + "evaluator": "Automated", + "status": "Pass", + "timestamp": "2022-11-22T00:00:00Z" + } + ], + "description": "Pass when frame count matches expected", + "reference": null, + "tags": [ + "Frame count checks", + "Video 2" + ], + "evaluated_assets": null + }, + { + "object_type": "QC metric", + "name": "ProbeA", + "modality": { + "name": "Extracellular electrophysiology", + "abbreviation": "ecephys" + }, + "stage": "Raw data", + "value": true, + "status_history": [ + { + "object_type": "QC status", + "evaluator": "Automated", + "status": "Pass", + "timestamp": "2022-11-22T00:00:00Z" + } + ], + "description": "Pass when probe is present in the recording", + "reference": null, + "tags": [ + "Probes present" + ], + "evaluated_assets": null + }, + { + "object_type": "QC metric", + "name": "ProbeB", + "modality": { + "name": "Extracellular electrophysiology", + "abbreviation": "ecephys" + }, + "stage": "Raw data", + "value": true, + "status_history": [ + { + "object_type": "QC status", + "evaluator": "Automated", + "status": "Pass", + "timestamp": "2022-11-22T00:00:00Z" + } + ], + "description": "Pass when probe is present in the recording", + "reference": null, + "tags": [ + "Probes present" + ], + "evaluated_assets": null + }, + { + "object_type": "QC metric", + "name": "ProbeC", + "modality": { + "name": "Extracellular electrophysiology", + "abbreviation": "ecephys" + }, + "stage": "Raw data", + "value": true, + "status_history": [ + { + "object_type": "QC status", + "evaluator": "Automated", + "status": "Pass", + "timestamp": "2022-11-22T00:00:00Z" + } + ], + "description": "Pass when probe is present in the recording", + "reference": null, + "tags": [ + "Probes present" + ], + "evaluated_assets": null + } + ], + "key_experimenters": null, + "notes": null, + "default_grouping": [ + "Drift map", + "Frame count checks", + "Probes present" + ], + "allow_tag_failures": [ + "Video 2" + ], + "status": { + "Probe A": "Pending", + "Probes present": "Pass", + "Video 1": "Pass", + "Probe C": "Pass", + "Video 2": "Pass", + "Probe B": "Pending", + "Frame count checks": "Pass", + "Drift map": "Pending", + "behavior-videos": "Pass", + "ecephys": "Pending", + "Raw data": "Pending" + } +} \ No newline at end of file diff --git a/tests/resources/helpers/quality_control_invalid.json b/tests/resources/helpers/quality_control_invalid.json index 54dfb52..ead71a5 100644 --- a/tests/resources/helpers/quality_control_invalid.json +++ b/tests/resources/helpers/quality_control_invalid.json @@ -1,5 +1,5 @@ { "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/quality_control.py", - "schema_version": "1.1.1", + "schema_version": "2.0.6", "notes": null } \ No newline at end of file From 6a974cfa904691118dd6c157eecce5ebef7bd7ab Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Fri, 26 Sep 2025 14:24:27 -0700 Subject: [PATCH 2/8] refactor: remove eval naming, fix tests --- .../helpers/data_schema.py | 6 ++-- tests/helpers/test_data_schema.py | 33 +++++++------------ 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/aind_data_access_api/helpers/data_schema.py b/src/aind_data_access_api/helpers/data_schema.py index 7822b05..87775b2 100644 --- a/src/aind_data_access_api/helpers/data_schema.py +++ b/src/aind_data_access_api/helpers/data_schema.py @@ -141,9 +141,7 @@ def get_quality_control_status_df( # Find the most recent status before the given datetime for status in reversed(metric.status_history): if status.timestamp <= date: - qc_metrics_flat[f"{eval.name}_{metric.name}"] = ( - status.status - ) + qc_metrics_flat[metric.name] = status.status break data.append(qc_metrics_flat) @@ -175,7 +173,7 @@ def get_quality_control_value_df( qc_metrics_flat = {} qc_metrics_flat["name"] = name for metric in qc.metrics: - qc_metrics_flat[f"{eval.name}_{metric.name}"] = metric.value + qc_metrics_flat[metric.name] = metric.value data.append(qc_metrics_flat) diff --git a/tests/helpers/test_data_schema.py b/tests/helpers/test_data_schema.py index dafc386..5abf4e5 100644 --- a/tests/helpers/test_data_schema.py +++ b/tests/helpers/test_data_schema.py @@ -9,7 +9,6 @@ from unittest.mock import MagicMock, patch from aind_data_schema.core.quality_control import ( QualityControl, - QCEvaluation, QCMetric, QCStatus, Status, @@ -166,22 +165,18 @@ def test_get_qc_value_df( ) metric0 = QCMetric( name="Metric0", + modality=Modality.ECEPHYS, + stage=Stage.RAW, value=0, status_history=[ status, ], ) - eval = QCEvaluation( - name="Evaluation0", - modality=Modality.ECEPHYS, - stage=Stage.RAW, - metrics=[metric0], - ) - mock_get_quality_control_by_names.return_value = [ QualityControl( - evaluations=[eval], + metrics=[metric0], + default_grouping=["test_grouping"], ) ] @@ -190,7 +185,7 @@ def test_get_qc_value_df( test_df = pd.DataFrame( { "name": ["fake_name"], - "Evaluation0_Metric0": [0], + "Metric0": [0], } ) @@ -213,22 +208,18 @@ def test_get_qc_status_df( ) metric0 = QCMetric( name="Metric0", + modality=Modality.ECEPHYS, + stage=Stage.RAW, value=0, status_history=[ status, ], ) - eval = QCEvaluation( - name="Evaluation0", - modality=Modality.ECEPHYS, - stage=Stage.RAW, - metrics=[metric0], - ) - mock_get_quality_control_by_names.return_value = [ QualityControl( - evaluations=[eval], + metrics=[metric0], + default_grouping=["test_grouping"], ) ] @@ -237,7 +228,7 @@ def test_get_qc_status_df( test_df = pd.DataFrame( { "name": ["fake_name"], - "Evaluation0_Metric0": [Status.PASS], + "Metric0": [Status.PASS], } ) @@ -262,8 +253,8 @@ def test_get_quality_control_by_names_valid(self): ) self.assertEqual(len(result), 2) - self.assertEqual(result[0].evaluations[0].name, "Drift map") - self.assertEqual(result[1].evaluations[0].name, "Drift map") + self.assertEqual(result[0].metrics[0].name, "Probe A drift") + self.assertEqual(result[1].metrics[0].name, "Probe A drift") mock_client.fetch_records_by_filter_list.assert_called_once_with( filter_key="name", filter_values=["name1", "name2"], From 4315d3a67d139bd2009627ef13c978c3ff26dd22 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Fri, 26 Sep 2025 14:33:29 -0700 Subject: [PATCH 3/8] tests: adding test coverage using the actual QC example --- tests/helpers/test_data_schema.py | 138 ++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/tests/helpers/test_data_schema.py b/tests/helpers/test_data_schema.py index 5abf4e5..56ecc04 100644 --- a/tests/helpers/test_data_schema.py +++ b/tests/helpers/test_data_schema.py @@ -303,6 +303,144 @@ def test_get_quality_control_by_names_no_records(self): projection={"quality_control": 1}, ) + def test_get_qc_value_df_with_example_data(self): + """Test get_quality_control_value_df with actual example QC data.""" + mock_client = MagicMock() + mock_client.fetch_records_by_filter_list.return_value = [ + {"quality_control": self.example_quality_control.copy()}, + {"quality_control": self.example_quality_control.copy()}, + ] + + result_df = get_quality_control_value_df( + client=mock_client, names=["session1", "session2"] + ) + + # Check that we got the right shape + self.assertEqual(len(result_df), 2) + self.assertEqual(list(result_df["name"]), ["session1", "session2"]) + + # Check specific values from the example QC data + expected_columns = [ + "name", + "Probe A drift", + "Probe B drift", + "Probe C drift", + "Expected frame count", + "Video 1 frame count", + "Video 2 num frames", + "ProbeA", + "ProbeB", + "ProbeC" + ] + self.assertEqual( + sorted(result_df.columns.tolist()), sorted(expected_columns) + ) + + # Check specific values + self.assertEqual(result_df["Probe C drift"].iloc[0], "Low") + self.assertEqual(result_df["Expected frame count"].iloc[0], 662) + self.assertEqual(result_df["Video 1 frame count"].iloc[0], 662) + self.assertEqual(result_df["ProbeA"].iloc[0], True) + self.assertEqual(result_df["ProbeB"].iloc[0], True) + self.assertEqual(result_df["ProbeC"].iloc[0], True) + + # Check that Probe A and B drift have complex value structures + probe_a_value = result_df["Probe A drift"].iloc[0] + self.assertIsInstance(probe_a_value, dict) + self.assertEqual(probe_a_value["value"], "") + self.assertEqual(probe_a_value["type"], "dropdown") + + probe_b_value = result_df["Probe B drift"].iloc[0] + self.assertIsInstance(probe_b_value, dict) + self.assertEqual(probe_b_value["value"], "") + self.assertEqual(probe_b_value["type"], "checkbox") + + def test_get_qc_status_df_with_example_data(self): + """Test get_quality_control_status_df with actual example QC data.""" + mock_client = MagicMock() + mock_client.fetch_records_by_filter_list.return_value = [ + {"quality_control": self.example_quality_control.copy()}, + {"quality_control": self.example_quality_control.copy()}, + ] + + # Use a date after the timestamps in the example data + test_date = datetime( + 2022, 11, 23, tzinfo=datetime.now().astimezone().tzinfo + ) + + result_df = get_quality_control_status_df( + client=mock_client, names=["session1", "session2"], date=test_date + ) + + # Check that we got the right shape + self.assertEqual(len(result_df), 2) + self.assertEqual(list(result_df["name"]), ["session1", "session2"]) + + # Check specific status values from the example QC data + expected_columns = [ + "name", + "Probe A drift", + "Probe B drift", + "Probe C drift", + "Expected frame count", + "Video 1 frame count", + "Video 2 num frames", + "ProbeA", + "ProbeB", + "ProbeC" + ] + self.assertEqual( + sorted(result_df.columns.tolist()), sorted(expected_columns) + ) + + # Check specific status values + self.assertEqual(result_df["Probe C drift"].iloc[0], Status.PASS) + self.assertEqual( + result_df["Expected frame count"].iloc[0], Status.PASS + ) + self.assertEqual(result_df["Video 1 frame count"].iloc[0], Status.PASS) + self.assertEqual(result_df["Video 2 num frames"].iloc[0], Status.PASS) + self.assertEqual(result_df["ProbeA"].iloc[0], Status.PASS) + self.assertEqual(result_df["ProbeB"].iloc[0], Status.PASS) + self.assertEqual(result_df["ProbeC"].iloc[0], Status.PASS) + + # Check that Probe A and B drift have pending status + self.assertEqual(result_df["Probe A drift"].iloc[0], Status.PENDING) + self.assertEqual(result_df["Probe B drift"].iloc[0], Status.PENDING) + + def test_get_qc_status_df_with_date_filtering(self): + """Test get_quality_control_status_df correctly filters by date.""" + mock_client = MagicMock() + mock_client.fetch_records_by_filter_list.return_value = [ + {"quality_control": self.example_quality_control.copy()} + ] + + # Use a date before the timestamps in the example data + early_date = datetime( + 2022, 11, 21, tzinfo=datetime.now().astimezone().tzinfo + ) + + result_df = get_quality_control_status_df( + client=mock_client, names=["session1"], date=early_date + ) + + # Since the date is before all status timestamps, no statuses found + # The function should only include the name column + self.assertEqual(len(result_df), 1) + self.assertEqual(list(result_df["name"]), ["session1"]) + + # All metric columns should be missing since no status history entries + # are before the specified date + metric_columns = [col for col in result_df.columns if col != "name"] + for col in metric_columns: + # If the column exists, it should be NaN/None + if col in result_df.columns: + is_nan_or_none = ( + pd.isna(result_df[col].iloc[0]) or + result_df[col].iloc[0] is None + ) + self.assertTrue(is_nan_or_none) + if __name__ == "__main__": unittest.main() From 399e7800a50d917ae4eb93ec3da5ff81a996747b Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Fri, 26 Sep 2025 14:33:43 -0700 Subject: [PATCH 4/8] chore: lint --- tests/helpers/test_data_schema.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/helpers/test_data_schema.py b/tests/helpers/test_data_schema.py index 56ecc04..eef472b 100644 --- a/tests/helpers/test_data_schema.py +++ b/tests/helpers/test_data_schema.py @@ -330,7 +330,7 @@ def test_get_qc_value_df_with_example_data(self): "Video 2 num frames", "ProbeA", "ProbeB", - "ProbeC" + "ProbeC", ] self.assertEqual( sorted(result_df.columns.tolist()), sorted(expected_columns) @@ -349,7 +349,7 @@ def test_get_qc_value_df_with_example_data(self): self.assertIsInstance(probe_a_value, dict) self.assertEqual(probe_a_value["value"], "") self.assertEqual(probe_a_value["type"], "dropdown") - + probe_b_value = result_df["Probe B drift"].iloc[0] self.assertIsInstance(probe_b_value, dict) self.assertEqual(probe_b_value["value"], "") @@ -367,7 +367,7 @@ def test_get_qc_status_df_with_example_data(self): test_date = datetime( 2022, 11, 23, tzinfo=datetime.now().astimezone().tzinfo ) - + result_df = get_quality_control_status_df( client=mock_client, names=["session1", "session2"], date=test_date ) @@ -387,7 +387,7 @@ def test_get_qc_status_df_with_example_data(self): "Video 2 num frames", "ProbeA", "ProbeB", - "ProbeC" + "ProbeC", ] self.assertEqual( sorted(result_df.columns.tolist()), sorted(expected_columns) @@ -419,7 +419,7 @@ def test_get_qc_status_df_with_date_filtering(self): early_date = datetime( 2022, 11, 21, tzinfo=datetime.now().astimezone().tzinfo ) - + result_df = get_quality_control_status_df( client=mock_client, names=["session1"], date=early_date ) @@ -428,7 +428,7 @@ def test_get_qc_status_df_with_date_filtering(self): # The function should only include the name column self.assertEqual(len(result_df), 1) self.assertEqual(list(result_df["name"]), ["session1"]) - + # All metric columns should be missing since no status history entries # are before the specified date metric_columns = [col for col in result_df.columns if col != "name"] @@ -436,8 +436,8 @@ def test_get_qc_status_df_with_date_filtering(self): # If the column exists, it should be NaN/None if col in result_df.columns: is_nan_or_none = ( - pd.isna(result_df[col].iloc[0]) or - result_df[col].iloc[0] is None + pd.isna(result_df[col].iloc[0]) + or result_df[col].iloc[0] is None ) self.assertTrue(is_nan_or_none) From 39e1313c74a0125d23d2bd147c65e2d06eab30b0 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Fri, 26 Sep 2025 14:35:57 -0700 Subject: [PATCH 5/8] chore: bump minimum python to 3.10 --- .github/workflows/run_dev_tests.yml | 2 +- .github/workflows/run_main_tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_dev_tests.yml b/.github/workflows/run_dev_tests.yml index b1a19b3..76f4f09 100644 --- a/.github/workflows/run_dev_tests.yml +++ b/.github/workflows/run_dev_tests.yml @@ -9,7 +9,7 @@ jobs: ci: strategy: matrix: - python-version: [ '3.8', '3.9', '3.10' ] + python-version: [ '3.10', '3.11', '3.12' ] uses: AllenNeuralDynamics/.github/.github/workflows/test-ci.yml@main with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/run_main_tests.yml b/.github/workflows/run_main_tests.yml index 9eaf522..855ec21 100644 --- a/.github/workflows/run_main_tests.yml +++ b/.github/workflows/run_main_tests.yml @@ -10,7 +10,7 @@ jobs: ci: strategy: matrix: - python-version: [ '3.8', '3.9', '3.10' ] + python-version: [ '3.10', '3.11', '3.12' ] uses: AllenNeuralDynamics/.github/.github/workflows/test-ci.yml@main with: python-version: ${{ matrix.python-version }} From ed15375e68093c253f342ca44bfe13144bf1d49e Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Fri, 26 Sep 2025 14:38:52 -0700 Subject: [PATCH 6/8] tests: fix a bug, no columns exist when no metrics are captured --- tests/helpers/test_data_schema.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/helpers/test_data_schema.py b/tests/helpers/test_data_schema.py index eef472b..269fff5 100644 --- a/tests/helpers/test_data_schema.py +++ b/tests/helpers/test_data_schema.py @@ -432,14 +432,8 @@ def test_get_qc_status_df_with_date_filtering(self): # All metric columns should be missing since no status history entries # are before the specified date metric_columns = [col for col in result_df.columns if col != "name"] - for col in metric_columns: - # If the column exists, it should be NaN/None - if col in result_df.columns: - is_nan_or_none = ( - pd.isna(result_df[col].iloc[0]) - or result_df[col].iloc[0] is None - ) - self.assertTrue(is_nan_or_none) + # No metric columns should exist since no status entries match the date filter + self.assertEqual(len(metric_columns), 0) if __name__ == "__main__": From 8ad0d66548a21c8e74108bda44e4b553c33388bc Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Fri, 26 Sep 2025 14:39:55 -0700 Subject: [PATCH 7/8] chore: lint --- tests/helpers/test_data_schema.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/helpers/test_data_schema.py b/tests/helpers/test_data_schema.py index 269fff5..7c0cb62 100644 --- a/tests/helpers/test_data_schema.py +++ b/tests/helpers/test_data_schema.py @@ -429,10 +429,7 @@ def test_get_qc_status_df_with_date_filtering(self): self.assertEqual(len(result_df), 1) self.assertEqual(list(result_df["name"]), ["session1"]) - # All metric columns should be missing since no status history entries - # are before the specified date metric_columns = [col for col in result_df.columns if col != "name"] - # No metric columns should exist since no status entries match the date filter self.assertEqual(len(metric_columns), 0) From ce63a3067559648147f8f169a9d34c210734c08c Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Mon, 13 Oct 2025 11:36:16 -0700 Subject: [PATCH 8/8] BREAKING CHANGE: bump default client to v2 --- src/aind_data_access_api/document_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aind_data_access_api/document_db.py b/src/aind_data_access_api/document_db.py index 7bd8fb0..81f04af 100644 --- a/src/aind_data_access_api/document_db.py +++ b/src/aind_data_access_api/document_db.py @@ -585,7 +585,7 @@ def __init__( host: str, database: str = "metadata_index", collection: str = "data_assets", - version: str = "v1", + version: str = "v2", boto: Optional[BotoSession] = None, session: Optional[Session] = None, ):