From 91ca6c830e7a81e4ba100d84898b29c48045c705 Mon Sep 17 00:00:00 2001 From: RebeccaBecky Date: Tue, 6 Apr 2021 15:43:27 +0100 Subject: [PATCH 1/8] Add basic reduce --- NDXINTER/reduce.py | 58 ++++++++++++++++++----------------------- NDXINTER/reduce_vars.py | 21 ++++++++------- 2 files changed, 36 insertions(+), 43 deletions(-) diff --git a/NDXINTER/reduce.py b/NDXINTER/reduce.py index 18dc957..d6217c9 100755 --- a/NDXINTER/reduce.py +++ b/NDXINTER/reduce.py @@ -1,41 +1,33 @@ -import os -import shutil import sys -sys.path.append("/isis/NDXMUSR/user/scripts/autoreduction") -import reduce_vars as web_var -def validate(input_file, output_dir): - """ - Function to ensure that the files we want to use in reduction exist. - Please add any files/directories to the required_files/dirs lists. - """ - print("Running validation") - required_files = [input_file] - required_dirs = [output_dir] - for file_path in required_files: - if not os.path.isfile(file_path): - raise RuntimeError("Unable to find file: {}".format(file_path)) - for dir in required_dirs: - if not os.path.isdir(dir): - raise RuntimeError("Unable to find directory: {}".format(dir)) - print("Validation successful") - +AUTOREDUCTION_DIR = r"/autoreduce/data-archive/NDXINTER/user/scripts/autoreduction" +sys.path.append(AUTOREDUCTION_DIR) + +# import mantid algorithms, numpy and matplotlib +from mantid.simpleapi import * +import matplotlib.pyplot as plt +import numpy as np +import reduce_vars as web_var +import os def main(input_file, output_dir): - validate(input_file, output_dir) + standard_params = web_var.standard_vars + advanced_params = web_var.advanced_vars - # Example of printing some stuff which is captured in autoreduction - # output log file - print(web_var) - print("input_file = " + str(input_file)) - print("output_dir = " + str(output_dir)) + config['defaultsave.directory'] = output_dir - # Copy raw data to output dir. - # Note this should only be done if raw files are small and for specific - # purpose such as testing - shutil.copy(input_file, output_dir) - - # And of course, here and below insert your reduction code! + OutputWorkspaceBinned, OutputWorkspace, OutputWorkspaceFirstTransmission, OutputWorkspaceSecondTransmission = ReflectometryISISLoadAndProcess(InputRunList=input_file, + FirstTransmissionRunList=standard_params['first_transmission_run_list'], + SecondTransmissionRunList=standard_params['second_transmission_run_list'], + ThetaIn=standard_params['theta_in'], + DetectorCorrectionType=standard_params['detector_correction_type'], + AnalysisMode=standard_params['analysis_mode'], + TransmissionProcessingInstructions=standard_params['transmission_processing_instructions'], + ProcessingInstructions=standard_params['processing_instructions']) + + SaveNexus(OutputWorkspaceBinned, os.path.join(output_dir, OutputWorkspaceBinned.name()+".nxs")) + SaveNexus(OutputWorkspace, os.path.join(output_dir, OutputWorkspace.name()+".nxs")) if __name__ == "__main__": - main("test", "test") + main('', '') + diff --git a/NDXINTER/reduce_vars.py b/NDXINTER/reduce_vars.py index 097dccd..2293afd 100755 --- a/NDXINTER/reduce_vars.py +++ b/NDXINTER/reduce_vars.py @@ -1,14 +1,15 @@ standard_vars = { - 'example standard var' : [1.0, 2.0, 3.0], # example comment + 'input_run_list' : "INTER00061667", + 'first_transmission_run_list' : "INTER00061705", + 'second_transmission_run_list' : "INTER00061669", + 'theta_in': 0.7, + 'detector_correction_type': "RotateAroundSample", + 'monitor_background_wavelength_min': 0.0, + 'monitor_background_wavelength_max': 1.0, + 'analysis_mode': "MultiDetectorAnalysis", + 'transmission_processing_instructions': "76-85", + 'processing_instructions': "80-84" } advanced_vars = { - 'example advanced var' : '5', } -variable_help = { - 'standard_vars' : { - 'example standard var' : "The list of a, b, c" - }, - 'advanced_vars' : { - 'example advanced var' : "example comment" - }, -} \ No newline at end of file + From 119742dc67f98901adee1c45912e163c102e4d61 Mon Sep 17 00:00:00 2001 From: RebeccaBecky Date: Fri, 9 Apr 2021 13:37:44 +0100 Subject: [PATCH 2/8] Add using default workspace names --- NDXINTER/reduce.py | 50 ++++++++++++++++++++++++++++++----------- NDXINTER/reduce_vars.py | 7 ++++++ 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/NDXINTER/reduce.py b/NDXINTER/reduce.py index d6217c9..d142ab5 100755 --- a/NDXINTER/reduce.py +++ b/NDXINTER/reduce.py @@ -1,5 +1,13 @@ import sys +sys.path.append('/opt/Mantid/scripts') +sys.path.append('/opt/Mantid/scripts/SANS') +sys.path.append('/opt/Mantid/lib') +sys.path.append('/opt/Mantid/scripts/Inelastic') +sys.path.append('/opt/Mantid/scripts/Engineering') +sys.path.append('/opt/Mantid/scripts/Interface') +sys.path.append('/opt/Mantid/scripts/Diffraction') + AUTOREDUCTION_DIR = r"/autoreduce/data-archive/NDXINTER/user/scripts/autoreduction" sys.path.append(AUTOREDUCTION_DIR) @@ -13,20 +21,36 @@ def main(input_file, output_dir): standard_params = web_var.standard_vars advanced_params = web_var.advanced_vars - config['defaultsave.directory'] = output_dir - - OutputWorkspaceBinned, OutputWorkspace, OutputWorkspaceFirstTransmission, OutputWorkspaceSecondTransmission = ReflectometryISISLoadAndProcess(InputRunList=input_file, - FirstTransmissionRunList=standard_params['first_transmission_run_list'], - SecondTransmissionRunList=standard_params['second_transmission_run_list'], - ThetaIn=standard_params['theta_in'], - DetectorCorrectionType=standard_params['detector_correction_type'], - AnalysisMode=standard_params['analysis_mode'], - TransmissionProcessingInstructions=standard_params['transmission_processing_instructions'], - ProcessingInstructions=standard_params['processing_instructions']) - - SaveNexus(OutputWorkspaceBinned, os.path.join(output_dir, OutputWorkspaceBinned.name()+".nxs")) - SaveNexus(OutputWorkspace, os.path.join(output_dir, OutputWorkspace.name()+".nxs")) + + alg=AlgorithmManager.create("ReflectometryISISLoadAndProcess") + properties = { + "InputRunList" : standard_params['input_run_list'], + "FirstTransmissionRunList" : standard_params['first_transmission_run_list'], + "SecondTransmissionRunList" : standard_params['second_transmission_run_list'], + "ThetaIn" : standard_params['theta_in'], + "DetectorCorrectionType" : standard_params['detector_correction_type'], + "MonitorBackgroundWavelengthMin" : standard_params['monitor_background_wavelength_min'], + "MonitorBackgroundWavelengthMax" : standard_params['monitor_background_wavelength_max'], + "MonitorIntegrationWavelengthMin" : standard_params['MonitorIntegrationWavelengthMin'], + "MonitorIntegrationWavelengthMax" : standard_params['MonitorIntegrationWavelengthMax'], + "WavelengthMin" : standard_params['WavelengthMin'], + "WavelengthMax" : standard_params['WavelengthMax'], + "I0MonitorIndex" : standard_params['IZeroMonitorIndex'], + "AnalysisMode" : standard_params['analysis_mode'], + "StartOverlap" : standard_params['StartOverlap'], + "EndOverlap" : standard_params['EndOverlap'], + "TransmissionProcessingInstructions" : standard_params['transmission_processing_instructions'], + "ProcessingInstructions" : standard_params['processing_instructions'] + } + alg.setProperties(properties) + alg.execute() + + OutputWorkspace=alg.getPropertyValue("OutputWorkspace") + OutputWorkspaceBinned=alg.getPropertyValue("OutputWorkspaceBinned") + + SaveNexus(OutputWorkspace, os.path.join(output_dir, OutputWorkspace+".nxs")) + SaveNexus(OutputWorkspaceBinned, os.path.join(output_dir, OutputWorkspaceBinned+".nxs")) if __name__ == "__main__": main('', '') diff --git a/NDXINTER/reduce_vars.py b/NDXINTER/reduce_vars.py index 2293afd..5a1087c 100755 --- a/NDXINTER/reduce_vars.py +++ b/NDXINTER/reduce_vars.py @@ -9,6 +9,13 @@ 'analysis_mode': "MultiDetectorAnalysis", 'transmission_processing_instructions': "76-85", 'processing_instructions': "80-84" + 'MonitorIntegrationWavelengthMin' : 4.0, + 'MonitorIntegrationWavelengthMax' : 10.0, + 'WavelengthMin' : 1.5, + 'WavelengthMax' : 17.0, + 'StartOverlap' : 10.0, + 'EndOverlap' : 12.0, + 'IZeroMonitorIndex' : 2 } advanced_vars = { } From 903043317649174decdbc87ec193b7a3e6cff56d Mon Sep 17 00:00:00 2001 From: RebeccaBecky Date: Fri, 9 Apr 2021 17:26:37 +0100 Subject: [PATCH 3/8] Add parsing settings from JSON and reading angle --- NDXINTER/reduce.py | 116 +++++++++++++++++++++++++++++++++------- NDXINTER/reduce_vars.py | 27 +++++----- 2 files changed, 112 insertions(+), 31 deletions(-) diff --git a/NDXINTER/reduce.py b/NDXINTER/reduce.py index d142ab5..5afc07a 100755 --- a/NDXINTER/reduce.py +++ b/NDXINTER/reduce.py @@ -17,32 +17,38 @@ import numpy as np import reduce_vars as web_var import os +import json def main(input_file, output_dir): standard_params = web_var.standard_vars advanced_params = web_var.advanced_vars + config['defaultsave.directory'] = output_dir + angle = get_angle() + analysis_mode,first_transmission_run_list,second_transmission_run_list,transmission_processing_instructions,processing_instructions,start_overlap,end_overlap,monitor_integration_wavelength_min,monitor_integration_wavelength_max,monitor_background_wavelength_min,monitor_background_wavelength_max,wavelength_min,wavelength_max,i_zero_monitor_index,detector_correction_type = parse_json_settings(angle) + alg=AlgorithmManager.create("ReflectometryISISLoadAndProcess") properties = { - "InputRunList" : standard_params['input_run_list'], - "FirstTransmissionRunList" : standard_params['first_transmission_run_list'], - "SecondTransmissionRunList" : standard_params['second_transmission_run_list'], - "ThetaIn" : standard_params['theta_in'], - "DetectorCorrectionType" : standard_params['detector_correction_type'], - "MonitorBackgroundWavelengthMin" : standard_params['monitor_background_wavelength_min'], - "MonitorBackgroundWavelengthMax" : standard_params['monitor_background_wavelength_max'], - "MonitorIntegrationWavelengthMin" : standard_params['MonitorIntegrationWavelengthMin'], - "MonitorIntegrationWavelengthMax" : standard_params['MonitorIntegrationWavelengthMax'], - "WavelengthMin" : standard_params['WavelengthMin'], - "WavelengthMax" : standard_params['WavelengthMax'], - "I0MonitorIndex" : standard_params['IZeroMonitorIndex'], - "AnalysisMode" : standard_params['analysis_mode'], - "StartOverlap" : standard_params['StartOverlap'], - "EndOverlap" : standard_params['EndOverlap'], - "TransmissionProcessingInstructions" : standard_params['transmission_processing_instructions'], - "ProcessingInstructions" : standard_params['processing_instructions'] + "InputRunList" : input_file, + "FirstTransmissionRunList" : first_transmission_run_list, + "SecondTransmissionRunList" : second_transmission_run_list, + "ThetaIn" : angle, + "DetectorCorrectionType" : detector_correction_type, + "MonitorBackgroundWavelengthMin" : monitor_background_wavelength_min, + "MonitorBackgroundWavelengthMax" : monitor_background_wavelength_max, + "MonitorIntegrationWavelengthMin" : monitor_integration_wavelength_min, + "MonitorIntegrationWavelengthMax" : monitor_integration_wavelength_max, + "WavelengthMin" : wavelength_min, + "WavelengthMax" : wavelength_max, + "I0MonitorIndex" : i_zero_monitor_index, + "AnalysisMode" : analysis_mode, + "StartOverlap" : start_overlap, + "EndOverlap" : end_overlap, + "TransmissionProcessingInstructions" : transmission_processing_instructions, + "ProcessingInstructions" : processing_instructions } + alg.setProperties(properties) alg.execute() @@ -52,6 +58,80 @@ def main(input_file, output_dir): SaveNexus(OutputWorkspace, os.path.join(output_dir, OutputWorkspace+".nxs")) SaveNexus(OutputWorkspaceBinned, os.path.join(output_dir, OutputWorkspaceBinned+".nxs")) +def get_angle(): + ws=Load('INTER61668') + title = (ws.run().getProperty('run_title').value) + angle = title.split(" th=")[1].split()[0] + return angle + +def parse_json_settings(angle): + """ + Autoreduction get settings from JSON file + """ + json_input = r"C:\Users\wyf59278\Documents\repos\reflectometry_reduction\settings.json" + + with open(json_input, "r") as read_file: + data = json.load(read_file) + + experimentView = data["experimentView"] + + if experimentView["analysisModeComboBox"] == 1: + analysis_mode = "MultiDetectorAnalysis" + elif experimentView["analysisModeComboBox"] == 0: + analysis_mode = "PointDetectorAnalysis" + else: + raise Exception # If the value isn't 1 or 0 then it isn't valid + + perAngleDefaults = experimentView["perAngleDefaults"] + rows = perAngleDefaults["rows"] + + # This looks for the run angle and set other parameters accordingly + angle_found = False + for i in rows: + if i[0] == angle: + angle_found = True + first_transmission_run_list = i[1] + second_transmission_run_list = i[2] + transmission_processing_instructions = i[3] + processing_instructions = i[8] + break + + # This is the default case + if not angle_found: + for i in rows: + if i[0] == "": + angle_found = True + first_transmission_run_list = i[1] + second_transmission_run_list = i[2] + transmission_processing_instructions = i[3] + processing_instructions = i[8] + break + + if not angle_found: + raise Exception # Excpetion for if neither a pre-defined angle or the default case are found + + start_overlap = experimentView["startOverlapEdit"] + + end_overlap = experimentView["endOverlapEdit"] + + instrumentView = data["instrumentView"] + + monitor_integration_wavelength_min = instrumentView["monIntMinEdit"] + monitor_integration_wavelength_max = instrumentView["monIntMaxEdit"] + monitor_background_wavelength_min = instrumentView["monBgMinEdit"] + monitor_background_wavelength_max = instrumentView["monBgMaxEdit"] + wavelength_min = instrumentView["lamMinEdit"] + wavelength_max = instrumentView["lamMaxEdit"] + i_zero_monitor_index = instrumentView["I0MonitorIndex"] + + if instrumentView["detectorCorrectionTypeComboBox"] == 1: + detector_correction_type = "RotateAroundSample" + elif instrumentView["detectorCorrectionTypeComboBox"] == 0: + detector_correction_type = "VerticalShift" + else: + raise Exception # If the value isn't 1 or 0 then it isn't valid + + return analysis_mode,first_transmission_run_list,second_transmission_run_list,transmission_processing_instructions,processing_instructions,start_overlap,end_overlap,monitor_integration_wavelength_min,monitor_integration_wavelength_max,monitor_background_wavelength_min,monitor_background_wavelength_max,wavelength_min,wavelength_max,i_zero_monitor_index,detector_correction_type + if __name__ == "__main__": main('', '') - diff --git a/NDXINTER/reduce_vars.py b/NDXINTER/reduce_vars.py index 5a1087c..6cc1881 100755 --- a/NDXINTER/reduce_vars.py +++ b/NDXINTER/reduce_vars.py @@ -1,22 +1,23 @@ standard_vars = { 'input_run_list' : "INTER00061667", - 'first_transmission_run_list' : "INTER00061705", - 'second_transmission_run_list' : "INTER00061669", 'theta_in': 0.7, - 'detector_correction_type': "RotateAroundSample", - 'monitor_background_wavelength_min': 0.0, - 'monitor_background_wavelength_max': 1.0, + 'analysis_mode': "MultiDetectorAnalysis", + 'first_transmission_run_list' : "INTER00061705", + 'second_transmission_run_list' : "INTER00061669", 'transmission_processing_instructions': "76-85", - 'processing_instructions': "80-84" - 'MonitorIntegrationWavelengthMin' : 4.0, - 'MonitorIntegrationWavelengthMax' : 10.0, - 'WavelengthMin' : 1.5, - 'WavelengthMax' : 17.0, - 'StartOverlap' : 10.0, - 'EndOverlap' : 12.0, - 'IZeroMonitorIndex' : 2 + 'processing_instructions': "80-84", + 'start_overlap' : 10.0, + 'end_overlap' : 12.0 } advanced_vars = { + 'monitor_integration_wavelength_min' : 4.0, + 'monitor_integration_wavelength_max' : 10.0, + 'monitor_background_wavelength_min': 17.0, + 'monitor_background_wavelength_max': 18.0, + 'wavelength_min' : 1.5, + 'wavelength_max' : 17.0, + 'i_zero_monitor_index' : 2, + 'detector_correction_type': "VerticalShift" } From 7c41c560d179ecdc72a83cfdf383bd5aab8835dd Mon Sep 17 00:00:00 2001 From: RebeccaBecky Date: Mon, 12 Apr 2021 16:36:28 +0100 Subject: [PATCH 4/8] Add reading real angle from motor position logs --- NDXINTER/reduce.py | 51 ++++++++++++++++++++++++++++++----------- NDXINTER/reduce_vars.py | 3 --- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/NDXINTER/reduce.py b/NDXINTER/reduce.py index 5afc07a..8d252e5 100755 --- a/NDXINTER/reduce.py +++ b/NDXINTER/reduce.py @@ -19,15 +19,18 @@ import os import json +# Main funcion that gets called by the reduction def main(input_file, output_dir): standard_params = web_var.standard_vars advanced_params = web_var.advanced_vars - config['defaultsave.directory'] = output_dir - angle = get_angle() + # Get the angle + angle = get_angle(input_file) + # Parse settings from JSON file analysis_mode,first_transmission_run_list,second_transmission_run_list,transmission_processing_instructions,processing_instructions,start_overlap,end_overlap,monitor_integration_wavelength_min,monitor_integration_wavelength_max,monitor_background_wavelength_min,monitor_background_wavelength_max,wavelength_min,wavelength_max,i_zero_monitor_index,detector_correction_type = parse_json_settings(angle) + # Run reduction alg=AlgorithmManager.create("ReflectometryISISLoadAndProcess") properties = { "InputRunList" : input_file, @@ -48,33 +51,45 @@ def main(input_file, output_dir): "TransmissionProcessingInstructions" : transmission_processing_instructions, "ProcessingInstructions" : processing_instructions } - alg.setProperties(properties) alg.execute() + # Save reduced data as Nexus files OutputWorkspace=alg.getPropertyValue("OutputWorkspace") OutputWorkspaceBinned=alg.getPropertyValue("OutputWorkspaceBinned") - SaveNexus(OutputWorkspace, os.path.join(output_dir, OutputWorkspace+".nxs")) SaveNexus(OutputWorkspaceBinned, os.path.join(output_dir, OutputWorkspaceBinned+".nxs")) -def get_angle(): - ws=Load('INTER61668') - title = (ws.run().getProperty('run_title').value) - angle = title.split(" th=")[1].split()[0] - return angle + +def get_angle(input_file): + """ + Get the average angle from logs of motor position + :param input_file: The input Nexus file + :return: Average angle from motor position readback + """ + ws=Load(input_file) + # Filter the logs for all angles starting from time 0 and use the average of the returned angles + (angle_list, average_angle) = FilterLogByTime(ws, 'Theta', StartTime=0) + return average_angle def parse_json_settings(angle): """ - Autoreduction get settings from JSON file + Get experiment settings and instrument settings from JSON file + :param angle: Angle passed in and used to select "per angle defaults" + :return: Returns all of the parameters needed to do the reduction """ json_input = r"C:\Users\wyf59278\Documents\repos\reflectometry_reduction\settings.json" with open(json_input, "r") as read_file: data = json.load(read_file) + #======================================================================================== + # Experiment Settings + #======================================================================================== + experimentView = data["experimentView"] + # Set a string based on what integer value is found if experimentView["analysisModeComboBox"] == 1: analysis_mode = "MultiDetectorAnalysis" elif experimentView["analysisModeComboBox"] == 0: @@ -86,13 +101,18 @@ def parse_json_settings(angle): rows = perAngleDefaults["rows"] # This looks for the run angle and set other parameters accordingly + # Using a tolerance of +-0.5% of the motor readback angle + min = angle * 0.995 + max = angle * 1.005 angle_found = False for i in rows: - if i[0] == angle: + # If the value is within -0.5% to +0.5% it is counted as a match + if min <= float(i[0]) <= max: angle_found = True first_transmission_run_list = i[1] second_transmission_run_list = i[2] transmission_processing_instructions = i[3] + # Skipping over parameters that are present in the JSON file but not currently used in the reduction processing_instructions = i[8] break @@ -104,16 +124,20 @@ def parse_json_settings(angle): first_transmission_run_list = i[1] second_transmission_run_list = i[2] transmission_processing_instructions = i[3] + # Skipping over parameters that are present in the JSON file but not currently used in the reduction processing_instructions = i[8] break if not angle_found: - raise Exception # Excpetion for if neither a pre-defined angle or the default case are found + raise Exception # Excpetion for if neither a pre-defined angle nor the default case are found start_overlap = experimentView["startOverlapEdit"] - end_overlap = experimentView["endOverlapEdit"] + #======================================================================================== + # Instrument Settings + #======================================================================================== + instrumentView = data["instrumentView"] monitor_integration_wavelength_min = instrumentView["monIntMinEdit"] @@ -124,6 +148,7 @@ def parse_json_settings(angle): wavelength_max = instrumentView["lamMaxEdit"] i_zero_monitor_index = instrumentView["I0MonitorIndex"] + # Set a string based on what integer value is found if instrumentView["detectorCorrectionTypeComboBox"] == 1: detector_correction_type = "RotateAroundSample" elif instrumentView["detectorCorrectionTypeComboBox"] == 0: diff --git a/NDXINTER/reduce_vars.py b/NDXINTER/reduce_vars.py index 6cc1881..c627d55 100755 --- a/NDXINTER/reduce_vars.py +++ b/NDXINTER/reduce_vars.py @@ -1,7 +1,4 @@ standard_vars = { - 'input_run_list' : "INTER00061667", - 'theta_in': 0.7, - 'analysis_mode': "MultiDetectorAnalysis", 'first_transmission_run_list' : "INTER00061705", 'second_transmission_run_list' : "INTER00061669", From 1aead8e62bbd1cd00ce1df83c0ddf12318fe8729 Mon Sep 17 00:00:00 2001 From: RebeccaBecky Date: Tue, 13 Apr 2021 15:29:34 +0100 Subject: [PATCH 5/8] Add load input under correct name to prevent loading same file twice --- NDXINTER/reduce.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/NDXINTER/reduce.py b/NDXINTER/reduce.py index 8d252e5..240e050 100755 --- a/NDXINTER/reduce.py +++ b/NDXINTER/reduce.py @@ -26,14 +26,17 @@ def main(input_file, output_dir): config['defaultsave.directory'] = output_dir # Get the angle - angle = get_angle(input_file) + angle, input_run = get_angle(input_file) # Parse settings from JSON file - analysis_mode,first_transmission_run_list,second_transmission_run_list,transmission_processing_instructions,processing_instructions,start_overlap,end_overlap,monitor_integration_wavelength_min,monitor_integration_wavelength_max,monitor_background_wavelength_min,monitor_background_wavelength_max,wavelength_min,wavelength_max,i_zero_monitor_index,detector_correction_type = parse_json_settings(angle) + analysis_mode, first_transmission_run_list, second_transmission_run_list, transmission_processing_instructions, \ + processing_instructions, start_overlap, end_overlap, monitor_integration_wavelength_min, \ + monitor_integration_wavelength_max, monitor_background_wavelength_min, monitor_background_wavelength_max, wavelength_min, \ + wavelength_max, i_zero_monitor_index, detector_correction_type = parse_json_settings(angle) # Run reduction alg=AlgorithmManager.create("ReflectometryISISLoadAndProcess") properties = { - "InputRunList" : input_file, + "InputRunList" : input_run, "FirstTransmissionRunList" : first_transmission_run_list, "SecondTransmissionRunList" : second_transmission_run_list, "ThetaIn" : angle, @@ -65,12 +68,15 @@ def get_angle(input_file): """ Get the average angle from logs of motor position :param input_file: The input Nexus file - :return: Average angle from motor position readback + :return: Average (mean) angle from motor position readback """ - ws=Load(input_file) + run_str = input_file.split("INTER")[1].split(".")[0].strip("0") + instrument='INTER' + name=instrument+run_str + ws=Load(Filename=name, OutputWorkspace='TOF_'+run_str) # Filter the logs for all angles starting from time 0 and use the average of the returned angles (angle_list, average_angle) = FilterLogByTime(ws, 'Theta', StartTime=0) - return average_angle + return average_angle, name def parse_json_settings(angle): """ @@ -156,7 +162,10 @@ def parse_json_settings(angle): else: raise Exception # If the value isn't 1 or 0 then it isn't valid - return analysis_mode,first_transmission_run_list,second_transmission_run_list,transmission_processing_instructions,processing_instructions,start_overlap,end_overlap,monitor_integration_wavelength_min,monitor_integration_wavelength_max,monitor_background_wavelength_min,monitor_background_wavelength_max,wavelength_min,wavelength_max,i_zero_monitor_index,detector_correction_type + return analysis_mode, first_transmission_run_list, second_transmission_run_list, transmission_processing_instructions, \ + processing_instructions, start_overlap, end_overlap, monitor_integration_wavelength_min, \ + monitor_integration_wavelength_max, monitor_background_wavelength_min, monitor_background_wavelength_max, wavelength_min, \ + wavelength_max, i_zero_monitor_index,detector_correction_type if __name__ == "__main__": main('', '') From 71c1e8a2a122d2b7bf25c62363ae539375555027 Mon Sep 17 00:00:00 2001 From: RebeccaBecky Date: Fri, 16 Apr 2021 11:29:57 +0100 Subject: [PATCH 6/8] Add copying json to output dir and change reduce_vars to just json path --- NDXINTER/reduce.py | 40 ++++++++++++++++++++-------------------- NDXINTER/reduce_vars.py | 32 ++++++++++++++++---------------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/NDXINTER/reduce.py b/NDXINTER/reduce.py index 240e050..e76007e 100755 --- a/NDXINTER/reduce.py +++ b/NDXINTER/reduce.py @@ -1,13 +1,5 @@ import sys -sys.path.append('/opt/Mantid/scripts') -sys.path.append('/opt/Mantid/scripts/SANS') -sys.path.append('/opt/Mantid/lib') -sys.path.append('/opt/Mantid/scripts/Inelastic') -sys.path.append('/opt/Mantid/scripts/Engineering') -sys.path.append('/opt/Mantid/scripts/Interface') -sys.path.append('/opt/Mantid/scripts/Diffraction') - AUTOREDUCTION_DIR = r"/autoreduce/data-archive/NDXINTER/user/scripts/autoreduction" sys.path.append(AUTOREDUCTION_DIR) @@ -18,6 +10,9 @@ import reduce_vars as web_var import os import json +from shutil import copy + +instrument='INTER' # Main funcion that gets called by the reduction def main(input_file, output_dir): @@ -28,10 +23,11 @@ def main(input_file, output_dir): # Get the angle angle, input_run = get_angle(input_file) # Parse settings from JSON file + json_input = standard_params['path_to_json_settings_file'] analysis_mode, first_transmission_run_list, second_transmission_run_list, transmission_processing_instructions, \ - processing_instructions, start_overlap, end_overlap, monitor_integration_wavelength_min, \ + processing_instructions, start_overlap, end_overlap, scale_rhs_workspace, monitor_integration_wavelength_min, \ monitor_integration_wavelength_max, monitor_background_wavelength_min, monitor_background_wavelength_max, wavelength_min, \ - wavelength_max, i_zero_monitor_index, detector_correction_type = parse_json_settings(angle) + wavelength_max, i_zero_monitor_index, detector_correction_type = parse_json_settings(json_input, angle) # Run reduction alg=AlgorithmManager.create("ReflectometryISISLoadAndProcess") @@ -51,6 +47,7 @@ def main(input_file, output_dir): "AnalysisMode" : analysis_mode, "StartOverlap" : start_overlap, "EndOverlap" : end_overlap, + "ScaleRHSWorkspace" : scale_rhs_workspace, "TransmissionProcessingInstructions" : transmission_processing_instructions, "ProcessingInstructions" : processing_instructions } @@ -63,6 +60,9 @@ def main(input_file, output_dir): SaveNexus(OutputWorkspace, os.path.join(output_dir, OutputWorkspace+".nxs")) SaveNexus(OutputWorkspaceBinned, os.path.join(output_dir, OutputWorkspaceBinned+".nxs")) + # Save a copy of the .json settings file + copy(json_input, output_dir) + def get_angle(input_file): """ @@ -70,21 +70,20 @@ def get_angle(input_file): :param input_file: The input Nexus file :return: Average (mean) angle from motor position readback """ - run_str = input_file.split("INTER")[1].split(".")[0].strip("0") - instrument='INTER' + filename = os.path.basename(input_file) + run_str = filename.split("INTER")[1].split(".")[0].strip("0") name=instrument+run_str ws=Load(Filename=name, OutputWorkspace='TOF_'+run_str) # Filter the logs for all angles starting from time 0 and use the average of the returned angles (angle_list, average_angle) = FilterLogByTime(ws, 'Theta', StartTime=0) return average_angle, name -def parse_json_settings(angle): +def parse_json_settings(json_input, angle): """ Get experiment settings and instrument settings from JSON file :param angle: Angle passed in and used to select "per angle defaults" :return: Returns all of the parameters needed to do the reduction """ - json_input = r"C:\Users\wyf59278\Documents\repos\reflectometry_reduction\settings.json" with open(json_input, "r") as read_file: data = json.load(read_file) @@ -115,8 +114,8 @@ def parse_json_settings(angle): # If the value is within -0.5% to +0.5% it is counted as a match if min <= float(i[0]) <= max: angle_found = True - first_transmission_run_list = i[1] - second_transmission_run_list = i[2] + first_transmission_run_list = instrument+i[1] + second_transmission_run_list = instrument+i[2] transmission_processing_instructions = i[3] # Skipping over parameters that are present in the JSON file but not currently used in the reduction processing_instructions = i[8] @@ -127,8 +126,8 @@ def parse_json_settings(angle): for i in rows: if i[0] == "": angle_found = True - first_transmission_run_list = i[1] - second_transmission_run_list = i[2] + first_transmission_run_list = instrument+i[1] + second_transmission_run_list = instrument+i[2] transmission_processing_instructions = i[3] # Skipping over parameters that are present in the JSON file but not currently used in the reduction processing_instructions = i[8] @@ -139,6 +138,7 @@ def parse_json_settings(angle): start_overlap = experimentView["startOverlapEdit"] end_overlap = experimentView["endOverlapEdit"] + scale_rhs_workspace = experimentView["transScaleRHSCheckBox"] #======================================================================================== # Instrument Settings @@ -163,9 +163,9 @@ def parse_json_settings(angle): raise Exception # If the value isn't 1 or 0 then it isn't valid return analysis_mode, first_transmission_run_list, second_transmission_run_list, transmission_processing_instructions, \ - processing_instructions, start_overlap, end_overlap, monitor_integration_wavelength_min, \ + processing_instructions, start_overlap, end_overlap, scale_rhs_workspace, monitor_integration_wavelength_min, \ monitor_integration_wavelength_max, monitor_background_wavelength_min, monitor_background_wavelength_max, wavelength_min, \ wavelength_max, i_zero_monitor_index,detector_correction_type if __name__ == "__main__": - main('', '') + main('','') diff --git a/NDXINTER/reduce_vars.py b/NDXINTER/reduce_vars.py index c627d55..5fd105e 100755 --- a/NDXINTER/reduce_vars.py +++ b/NDXINTER/reduce_vars.py @@ -1,20 +1,20 @@ standard_vars = { - 'analysis_mode': "MultiDetectorAnalysis", - 'first_transmission_run_list' : "INTER00061705", - 'second_transmission_run_list' : "INTER00061669", - 'transmission_processing_instructions': "76-85", - 'processing_instructions': "80-84", - 'start_overlap' : 10.0, - 'end_overlap' : 12.0 + 'path_to_json_settings_file' : '/autoreduce/scripts/settings.json' + # 'analysis_mode': "MultiDetectorAnalysis", + # 'first_transmission_run_list' : "INTER00061705", + # 'second_transmission_run_list' : "INTER00061669", + # 'transmission_processing_instructions': "76-85", + # 'processing_instructions': "80-84", + # 'start_overlap' : 10.0, + # 'end_overlap' : 12.0 } advanced_vars = { - 'monitor_integration_wavelength_min' : 4.0, - 'monitor_integration_wavelength_max' : 10.0, - 'monitor_background_wavelength_min': 17.0, - 'monitor_background_wavelength_max': 18.0, - 'wavelength_min' : 1.5, - 'wavelength_max' : 17.0, - 'i_zero_monitor_index' : 2, - 'detector_correction_type': "VerticalShift" + # 'monitor_integration_wavelength_min' : 4.0, + # 'monitor_integration_wavelength_max' : 10.0, + # 'monitor_background_wavelength_min': 17.0, + # 'monitor_background_wavelength_max': 18.0, + # 'wavelength_min' : 1.5, + # 'wavelength_max' : 17.0, + # 'i_zero_monitor_index' : 2, + # 'detector_correction_type': "VerticalShift" } - From bd727ede8694a07e3429b49c32999e609ff917cf Mon Sep 17 00:00:00 2001 From: RebeccaBecky Date: Fri, 16 Apr 2021 12:18:37 +0100 Subject: [PATCH 7/8] Add setting per-angle defaults params in a separate function --- NDXINTER/reduce.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/NDXINTER/reduce.py b/NDXINTER/reduce.py index e76007e..fa011f7 100755 --- a/NDXINTER/reduce.py +++ b/NDXINTER/reduce.py @@ -110,27 +110,17 @@ def parse_json_settings(json_input, angle): min = angle * 0.995 max = angle * 1.005 angle_found = False - for i in rows: + for row in rows: # If the value is within -0.5% to +0.5% it is counted as a match - if min <= float(i[0]) <= max: - angle_found = True - first_transmission_run_list = instrument+i[1] - second_transmission_run_list = instrument+i[2] - transmission_processing_instructions = i[3] - # Skipping over parameters that are present in the JSON file but not currently used in the reduction - processing_instructions = i[8] + if min <= float(row[0]) <= max: + angle_found, first_transmission_run_list, second_transmission_run_list, transmission_processing_instructions, processing_instructions = get_per_angle_defaults_params(row) break # This is the default case if not angle_found: - for i in rows: - if i[0] == "": - angle_found = True - first_transmission_run_list = instrument+i[1] - second_transmission_run_list = instrument+i[2] - transmission_processing_instructions = i[3] - # Skipping over parameters that are present in the JSON file but not currently used in the reduction - processing_instructions = i[8] + for row in rows: + if row[0] == "": + angle_found, first_transmission_run_list, second_transmission_run_list, transmission_processing_instructions, processing_instructions = get_per_angle_defaults_params(row) break if not angle_found: @@ -167,5 +157,20 @@ def parse_json_settings(json_input, angle): monitor_integration_wavelength_max, monitor_background_wavelength_min, monitor_background_wavelength_max, wavelength_min, \ wavelength_max, i_zero_monitor_index,detector_correction_type +def get_per_angle_defaults_params(row): + """ + Get parameters that are dependant on the angle + :param row: The row for the angle that has been selected (or the row for the default case if no angle was matched) + :return: Returns all of the parameters that are dependant on the angle + """ + angle_found = True + first_transmission_run_list = instrument+row[1] + second_transmission_run_list = instrument+row[2] + transmission_processing_instructions = row[3] + # Skipping over parameters that are present in the JSON file but not currently used in the reduction + processing_instructions = row[8] + return angle_found, first_transmission_run_list, second_transmission_run_list, transmission_processing_instructions, processing_instructions + if __name__ == "__main__": main('','') + From 895a80abe7cd50329827620842d8d5ca32bc02ea Mon Sep 17 00:00:00 2001 From: RebeccaBecky Date: Fri, 16 Apr 2021 16:17:04 +0100 Subject: [PATCH 8/8] Add class for the params extracted from JSON --- NDXINTER/reduce.py | 106 +++++++++++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 47 deletions(-) diff --git a/NDXINTER/reduce.py b/NDXINTER/reduce.py index fa011f7..47a971a 100755 --- a/NDXINTER/reduce.py +++ b/NDXINTER/reduce.py @@ -24,32 +24,29 @@ def main(input_file, output_dir): angle, input_run = get_angle(input_file) # Parse settings from JSON file json_input = standard_params['path_to_json_settings_file'] - analysis_mode, first_transmission_run_list, second_transmission_run_list, transmission_processing_instructions, \ - processing_instructions, start_overlap, end_overlap, scale_rhs_workspace, monitor_integration_wavelength_min, \ - monitor_integration_wavelength_max, monitor_background_wavelength_min, monitor_background_wavelength_max, wavelength_min, \ - wavelength_max, i_zero_monitor_index, detector_correction_type = parse_json_settings(json_input, angle) + params = parse_json_settings(json_input, angle) # Run reduction alg=AlgorithmManager.create("ReflectometryISISLoadAndProcess") properties = { "InputRunList" : input_run, - "FirstTransmissionRunList" : first_transmission_run_list, - "SecondTransmissionRunList" : second_transmission_run_list, + "FirstTransmissionRunList" : params.first_transmission_run_list, + "SecondTransmissionRunList" : params.second_transmission_run_list, "ThetaIn" : angle, - "DetectorCorrectionType" : detector_correction_type, - "MonitorBackgroundWavelengthMin" : monitor_background_wavelength_min, - "MonitorBackgroundWavelengthMax" : monitor_background_wavelength_max, - "MonitorIntegrationWavelengthMin" : monitor_integration_wavelength_min, - "MonitorIntegrationWavelengthMax" : monitor_integration_wavelength_max, - "WavelengthMin" : wavelength_min, - "WavelengthMax" : wavelength_max, - "I0MonitorIndex" : i_zero_monitor_index, - "AnalysisMode" : analysis_mode, - "StartOverlap" : start_overlap, - "EndOverlap" : end_overlap, - "ScaleRHSWorkspace" : scale_rhs_workspace, - "TransmissionProcessingInstructions" : transmission_processing_instructions, - "ProcessingInstructions" : processing_instructions + "DetectorCorrectionType" : params.detector_correction_type, + "MonitorBackgroundWavelengthMin" : params.monitor_background_wavelength_min, + "MonitorBackgroundWavelengthMax" : params.monitor_background_wavelength_max, + "MonitorIntegrationWavelengthMin" : params.monitor_integration_wavelength_min, + "MonitorIntegrationWavelengthMax" : params.monitor_integration_wavelength_max, + "WavelengthMin" : params.wavelength_min, + "WavelengthMax" : params.wavelength_max, + "I0MonitorIndex" : params.i_zero_monitor_index, + "AnalysisMode" : params.analysis_mode, + "StartOverlap" : params.start_overlap, + "EndOverlap" : params.end_overlap, + "ScaleRHSWorkspace" : params.scale_rhs_workspace, + "TransmissionProcessingInstructions" : params.transmission_processing_instructions, + "ProcessingInstructions" : params.processing_instructions } alg.setProperties(properties) alg.execute() @@ -63,7 +60,6 @@ def main(input_file, output_dir): # Save a copy of the .json settings file copy(json_input, output_dir) - def get_angle(input_file): """ Get the average angle from logs of motor position @@ -84,6 +80,7 @@ def parse_json_settings(json_input, angle): :param angle: Angle passed in and used to select "per angle defaults" :return: Returns all of the parameters needed to do the reduction """ + params = INTERParams() with open(json_input, "r") as read_file: data = json.load(read_file) @@ -96,9 +93,9 @@ def parse_json_settings(json_input, angle): # Set a string based on what integer value is found if experimentView["analysisModeComboBox"] == 1: - analysis_mode = "MultiDetectorAnalysis" + params.analysis_mode = "MultiDetectorAnalysis" elif experimentView["analysisModeComboBox"] == 0: - analysis_mode = "PointDetectorAnalysis" + params.analysis_mode = "PointDetectorAnalysis" else: raise Exception # If the value isn't 1 or 0 then it isn't valid @@ -113,22 +110,22 @@ def parse_json_settings(json_input, angle): for row in rows: # If the value is within -0.5% to +0.5% it is counted as a match if min <= float(row[0]) <= max: - angle_found, first_transmission_run_list, second_transmission_run_list, transmission_processing_instructions, processing_instructions = get_per_angle_defaults_params(row) + angle_found, params = get_per_angle_defaults_params(row, params) break # This is the default case if not angle_found: for row in rows: if row[0] == "": - angle_found, first_transmission_run_list, second_transmission_run_list, transmission_processing_instructions, processing_instructions = get_per_angle_defaults_params(row) + angle_found, params = get_per_angle_defaults_params(row, params) break if not angle_found: raise Exception # Excpetion for if neither a pre-defined angle nor the default case are found - start_overlap = experimentView["startOverlapEdit"] - end_overlap = experimentView["endOverlapEdit"] - scale_rhs_workspace = experimentView["transScaleRHSCheckBox"] + params.start_overlap = experimentView["startOverlapEdit"] + params.end_overlap = experimentView["endOverlapEdit"] + params.scale_rhs_workspace = experimentView["transScaleRHSCheckBox"] #======================================================================================== # Instrument Settings @@ -136,40 +133,55 @@ def parse_json_settings(json_input, angle): instrumentView = data["instrumentView"] - monitor_integration_wavelength_min = instrumentView["monIntMinEdit"] - monitor_integration_wavelength_max = instrumentView["monIntMaxEdit"] - monitor_background_wavelength_min = instrumentView["monBgMinEdit"] - monitor_background_wavelength_max = instrumentView["monBgMaxEdit"] - wavelength_min = instrumentView["lamMinEdit"] - wavelength_max = instrumentView["lamMaxEdit"] - i_zero_monitor_index = instrumentView["I0MonitorIndex"] + params.monitor_integration_wavelength_min = instrumentView["monIntMinEdit"] + params.monitor_integration_wavelength_max = instrumentView["monIntMaxEdit"] + params.monitor_background_wavelength_min = instrumentView["monBgMinEdit"] + params.monitor_background_wavelength_max = instrumentView["monBgMaxEdit"] + params.wavelength_min = instrumentView["lamMinEdit"] + params.wavelength_max = instrumentView["lamMaxEdit"] + params.i_zero_monitor_index = instrumentView["I0MonitorIndex"] # Set a string based on what integer value is found if instrumentView["detectorCorrectionTypeComboBox"] == 1: - detector_correction_type = "RotateAroundSample" + params.detector_correction_type = "RotateAroundSample" elif instrumentView["detectorCorrectionTypeComboBox"] == 0: - detector_correction_type = "VerticalShift" + params.detector_correction_type = "VerticalShift" else: raise Exception # If the value isn't 1 or 0 then it isn't valid - return analysis_mode, first_transmission_run_list, second_transmission_run_list, transmission_processing_instructions, \ - processing_instructions, start_overlap, end_overlap, scale_rhs_workspace, monitor_integration_wavelength_min, \ - monitor_integration_wavelength_max, monitor_background_wavelength_min, monitor_background_wavelength_max, wavelength_min, \ - wavelength_max, i_zero_monitor_index,detector_correction_type + return params -def get_per_angle_defaults_params(row): +def get_per_angle_defaults_params(row, params): """ Get parameters that are dependant on the angle :param row: The row for the angle that has been selected (or the row for the default case if no angle was matched) :return: Returns all of the parameters that are dependant on the angle """ angle_found = True - first_transmission_run_list = instrument+row[1] - second_transmission_run_list = instrument+row[2] - transmission_processing_instructions = row[3] + params.first_transmission_run_list = instrument+row[1] + params.second_transmission_run_list = instrument+row[2] + params.transmission_processing_instructions = row[3] # Skipping over parameters that are present in the JSON file but not currently used in the reduction - processing_instructions = row[8] - return angle_found, first_transmission_run_list, second_transmission_run_list, transmission_processing_instructions, processing_instructions + params.processing_instructions = row[8] + return angle_found, params + +class INTERParams: + analysis_mode : str + first_transmission_run_list : str + second_transmission_run_list : str + transmission_processing_instructions : str + processing_instructions : str + start_overlap : str + end_overlap : str + scale_rhs_workspace : str + monitor_integration_wavelength_min : str + monitor_integration_wavelength_max : str + monitor_background_wavelength_min : str + monitor_background_wavelength_max : str + wavelength_min : str + wavelength_max : str + i_zero_monitor_index : str + detector_correction_type : str if __name__ == "__main__": main('','')