diff --git a/pcpostprocess/hergQC.py b/pcpostprocess/hergQC.py index e248a3ca..f6cb81a8 100644 --- a/pcpostprocess/hergQC.py +++ b/pcpostprocess/hergQC.py @@ -172,8 +172,19 @@ def run_qc(self, voltage_steps, times, if (None in qc_vals_before) or (None in qc_vals_after): return QC - qc1_1 = self.qc1(*qc_vals_before) - qc1_2 = self.qc1(*qc_vals_after) + qc1_1 = True + for i in range(n_sweeps): + _qc1_1 = self.qc1(*qc_vals_before[i, :]) + qc1_1 = qc1_1 and _qc1_1 + if not qc1_1: + break + + qc1_2 = True + for i in range(n_sweeps): + _qc1_2 = self.qc1(*qc_vals_after[i, :]) + if not qc1_2: + break + qc1_2 = qc1_2 and _qc1_2 QC['qc1.rseal'] = [qc1_1[0], qc1_2[0]] QC['qc1.cm'] = [qc1_1[1], qc1_2[1]] @@ -197,14 +208,19 @@ def run_qc(self, voltage_steps, times, QC['qc3.E4031'] = [qc3_2] QC['qc3.subtracted'] = [qc3_3] - rseals = [qc_vals_before[0], qc_vals_after[0]] - cms = [qc_vals_before[1], qc_vals_after[1]] - rseriess = [qc_vals_before[2], qc_vals_after[2]] - qc4 = self.qc4(rseals, cms, rseriess) + qc4 = [] + for i in range(n_sweeps): + rseals = [qc_vals_before[i, 0], qc_vals_after[i, 0]] + cms = [qc_vals_before[i, 1], qc_vals_after[i, 1]] + rseriess = [qc_vals_before[i, 2], qc_vals_after[i, 2]] + qc4.append(self.qc4(rseals, cms, rseriess)) + + # Combine sweepwise QC4 results + qc4 = np.array(qc4) - QC['qc4.rseal'] = [qc4[0]] - QC['qc4.cm'] = [qc4[1]] - QC['qc4.rseries'] = [qc4[2]] + QC['qc4.rseal'] = [(bool(row[0]), row[1]) for row in qc4[:, 0, :]] + QC['qc4.cm'] = [(bool(row[0]), row[1]) for row in qc4[:, 1, :]] + QC['qc4.rseries'] = [(bool(row[0]), row[1]) for row in qc4[:, 2, :]] # indices where hERG peaks qc5 = self.qc5(before[0, :], after[0, :], diff --git a/pcpostprocess/scripts/run_herg_qc.py b/pcpostprocess/scripts/run_herg_qc.py index 9f618aec..3f4e2059 100644 --- a/pcpostprocess/scripts/run_herg_qc.py +++ b/pcpostprocess/scripts/run_herg_qc.py @@ -162,7 +162,7 @@ def main(): # this error is raised if the user has not specified the dictionary of protocols in the export_config.py file if not readnames: - logging.error("No compatible protocols found in export config file.") + logging.error("No compatible protocols found in export config file") return with multiprocessing.Pool(min(args.no_cpus, len(readnames)), @@ -266,7 +266,8 @@ def main(): wells_to_export = wells if args.export_failed else overall_selection - logging.info(f"exporting wells {wells}") + logging.info(f"Wells passing QC so far: {overall_selection}") + logging.info(f"exporting wells {wells_to_export}") no_protocols = len(res_dict) @@ -281,6 +282,7 @@ def main(): if dfs: extract_df = pd.concat(dfs, ignore_index=True) + extract_df['selected'] = extract_df['well'].isin(overall_selection) else: logging.error("Didn't export any data") @@ -424,8 +426,6 @@ def agg_func(x): qc_df['protocol'] = ['staircaseramp1_2' if p == 'staircaseramp2' else p for p in qc_df.protocol] - print(qc_df.protocol.unique()) - fails_dict = {} no_wells = 384 @@ -855,6 +855,8 @@ def run_qc_for_protocol(readname, savename, time_strs, args, output_dir): raw_before_all = before_trace.get_trace_sweeps(sweeps) raw_after_all = after_trace.get_trace_sweeps(sweeps) + logging.info(f"sampling_rate is {sampling_rate}") + selected_wells = [] for well in args.wells: @@ -929,7 +931,6 @@ def run_qc_for_protocol(readname, savename, time_strs, args, output_dir): after_currents[sweep, :] = after_raw logging.info(f"{well} {savename}\n----------") - logging.info(f"sampling_rate is {sampling_rate}") voltage_steps = [tend for tstart, tend, vstart, vend in @@ -941,8 +942,8 @@ def run_qc_for_protocol(readname, savename, time_strs, args, output_dir): times, before_currents_corrected, after_currents_corrected, - np.array(qc_before[well])[0, :], - np.array(qc_after[well])[0, :], + np.array(qc_before[well]), + np.array(qc_after[well]), nsweeps, ) diff --git a/tests/test_herg_qc.py b/tests/test_herg_qc.py index 6ba2cf19..012cb23b 100755 --- a/tests/test_herg_qc.py +++ b/tests/test_herg_qc.py @@ -891,46 +891,46 @@ def test_run_qc(self): hergqc = self.clone_hergqc("test_run_qc") failed_wells = [ - 'A04', 'A05', 'A06', 'A07', 'A08', 'A10', 'A11', 'A12', 'A13', 'A15', - 'A16', 'A19', 'A20', 'A21', 'A22', 'A23', 'A24', 'B02', 'B04', 'B05', - 'B07', 'B09', 'B10', 'B11', 'B12', 'B13', 'B14', 'B15', 'B16', 'B18', - 'B19', 'B21', 'B23', 'C01', 'C02', 'C04', 'C05', 'C07', 'C08', 'C09', - 'C10', 'C11', 'C12', 'C14', 'C17', 'C18', 'C19', 'C20', 'C21', 'C22', - 'C23', 'C24', 'D01', 'D02', 'D03', 'D04', 'D05', 'D09', 'D10', 'D11', - 'D12', 'D13', 'D14', 'D15', 'D16', 'D17', 'D18', 'D19', 'D21', 'D23', - 'E01', 'E02', 'E03', 'E04', 'E06', 'E07', 'E09', 'E10', 'E11', 'E13', - 'E14', 'E15', 'E16', 'E17', 'E18', 'E19', 'E20', 'E21', 'E22', 'E23', - 'E24', 'F01', 'F02', 'F03', 'F04', 'F05', 'F06', 'F07', 'F09', 'F10', - 'F11', 'F12', 'F13', 'F14', 'F15', 'F16', 'F18', 'F19', 'F20', 'F21', - 'F22', 'F23', 'F24', 'G03', 'G06', 'G08', 'G09', 'G10', 'G12', 'G13', - 'G14', 'G15', 'G16', 'G17', 'G18', 'G19', 'G20', 'G21', 'G23', 'G24', - 'H01', 'H02', 'H03', 'H04', 'H06', 'H07', 'H08', 'H09', 'H10', 'H11', - 'H13', 'H14', 'H15', 'H16', 'H17', 'H18', 'H19', 'H20', 'H21', 'H23', - 'H24', 'I01', 'I03', 'I04', 'I05', 'I06', 'I07', 'I08', 'I10', 'I11', - 'I12', 'I13', 'I14', 'I15', 'I16', 'I17', 'I18', 'I19', 'I20', 'I21', - 'J03', 'J06', 'J07', 'J08', 'J09', 'J10', 'J11', 'J12', 'J14', 'J15', - 'J16', 'J17', 'J18', 'J19', 'J20', 'J21', 'J23', 'J24', 'K01', 'K02', - 'K03', 'K05', 'K06', 'K07', 'K09', 'K10', 'K11', 'K12', 'K13', 'K14', - 'K16', 'K17', 'K18', 'K20', 'K22', 'K23', 'K24', 'L01', 'L02', 'L03', - 'L04', 'L05', 'L06', 'L07', 'L08', 'L10', 'L11', 'L12', 'L13', 'L16', - 'L17', 'L18', 'L20', 'L21', 'L23', 'L24', 'M01', 'M02', 'M03', 'M04', - 'M05', 'M06', 'M07', 'M08', 'M09', 'M10', 'M11', 'M12', 'M13', 'M14', - 'M15', 'M16', 'M17', 'M18', 'M19', 'M20', 'M21', 'N01', 'N02', 'N03', - 'N04', 'N06', 'N07', 'N08', 'N09', 'N11', 'N13', 'N14', 'N16', 'N17', - 'N18', 'N19', 'N20', 'N21', 'N22', 'N23', 'N24', 'O01', 'O02', 'O03', - 'O04', 'O05', 'O06', 'O07', 'O08', 'O10', 'O11', 'O12', 'O13', 'O14', - 'O15', 'O16', 'O17', 'O18', 'O19', 'O20', 'O21', 'O22', 'O24', 'P01', - 'P03', 'P05', 'P06', 'P07', 'P08', 'P09', 'P10', 'P11', 'P12', 'P13', - 'P14', 'P15', 'P16', 'P17', 'P18', 'P19', 'P20', 'P21', 'P22', 'P24' + 'A01', 'A04', 'A05', 'A06', 'A07', 'A08', 'A10', 'A11', 'A12', 'A13', + 'A15', 'A16', 'A19', 'A20', 'A21', 'A22', 'A23', 'A24', 'B01', 'B02', + 'B04', 'B05', 'B06', 'B07', 'B09', 'B10', 'B11', 'B12', 'B13', 'B14', + 'B15', 'B16', 'B18', 'B19', 'B21', 'B22', 'B23', 'C01', 'C02', 'C04', + 'C05', 'C07', 'C08', 'C09', 'C10', 'C11', 'C12', 'C14', 'C15', 'C17', + 'C18', 'C19', 'C20', 'C21', 'C22', 'C23', 'C24', 'D01', 'D02', 'D03', + 'D04', 'D05', 'D08', 'D09', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', + 'D16', 'D17', 'D18', 'D19', 'D21', 'D23', 'E01', 'E02', 'E03', 'E04', + 'E06', 'E07', 'E09', 'E10', 'E11', 'E12', 'E13', 'E14', 'E15', 'E16', + 'E17', 'E18', 'E19', 'E20', 'E21', 'E22', 'E23', 'E24', 'F01', 'F02', + 'F03', 'F04', 'F05', 'F06', 'F07', 'F08', 'F09', 'F10', 'F11', 'F12', + 'F13', 'F14', 'F15', 'F16', 'F18', 'F19', 'F20', 'F21', 'F22', 'F23', + 'F24', 'G03', 'G06', 'G08', 'G09', 'G10', 'G12', 'G13', 'G14', 'G15', + 'G16', 'G17', 'G18', 'G19', 'G20', 'G21', 'G23', 'G24', 'H01', 'H02', + 'H03', 'H04', 'H06', 'H07', 'H08', 'H09', 'H10', 'H11', 'H13', 'H14', + 'H15', 'H16', 'H17', 'H18', 'H19', 'H20', 'H21', 'H23', 'H24', 'I01', + 'I03', 'I04', 'I05', 'I06', 'I07', 'I08', 'I10', 'I11', 'I12', 'I13', + 'I14', 'I15', 'I16', 'I17', 'I18', 'I19', 'I20', 'I21', 'I24', 'J03', + 'J06', 'J07', 'J08', 'J09', 'J10', 'J11', 'J12', 'J14', 'J15', 'J16', + 'J17', 'J18', 'J19', 'J20', 'J21', 'J23', 'J24', 'K01', 'K02', 'K03', + 'K05', 'K06', 'K07', 'K09', 'K10', 'K11', 'K12', 'K13', 'K14', 'K16', + 'K17', 'K18', 'K20', 'K22', 'K23', 'K24', 'L01', 'L02', 'L03', 'L04', + 'L05', 'L06', 'L07', 'L08', 'L10', 'L11', 'L12', 'L13', 'L16', 'L17', + 'L18', 'L20', 'L21', 'L23', 'L24', 'M01', 'M02', 'M03', 'M04', 'M05', + 'M06', 'M07', 'M08', 'M09', 'M10', 'M11', 'M12', 'M13', 'M14', 'M15', + 'M16', 'M17', 'M18', 'M19', 'M20', 'M21', 'N01', 'N02', 'N03', 'N04', + 'N06', 'N07', 'N08', 'N09', 'N11', 'N13', 'N14', 'N16', 'N17', 'N18', + 'N19', 'N20', 'N21', 'N22', 'N23', 'N24', 'O01', 'O02', 'O03', 'O04', + 'O05', 'O06', 'O07', 'O08', 'O10', 'O11', 'O12', 'O13', 'O14', 'O15', + 'O16', 'O17', 'O18', 'O19', 'O20', 'O21', 'O22', 'O24', 'P01', 'P03', + 'P05', 'P06', 'P07', 'P08', 'P09', 'P10', 'P11', 'P12', 'P13', 'P14', + 'P15', 'P16', 'P17', 'P18', 'P19', 'P20', 'P21', 'P22', 'P24' ] for well in self.all_wells: before = np.array(self.trace_sweeps_before[well]) after = np.array(self.trace_sweeps_after[well]) - # Take values from the first sweep only - qc_vals_before = np.array(self.qc_vals_before[well])[0, :] - qc_vals_after = np.array(self.qc_vals_after[well])[0, :] + qc_vals_before = np.array(self.qc_vals_before[well]) + qc_vals_after = np.array(self.qc_vals_after[well]) QC = hergqc.run_qc( voltage_steps=self.voltage_steps,