diff --git a/analysis-scripts/analysis_scripts/__init__.py b/analysis-scripts/analysis_scripts/__init__.py index df1eb04..33d6ce2 100644 --- a/analysis-scripts/analysis_scripts/__init__.py +++ b/analysis-scripts/analysis_scripts/__init__.py @@ -1,4 +1,12 @@ import json +from argparse import ArgumentParser +from os import getenv +from pathlib import Path + +from requests import get, post +from yaml import safe_load + +from freanalysis.plugins import run_plugin class AnalysisScript(object): @@ -24,16 +32,195 @@ def requires(self): raise NotImplementedError("you must override this function.") return json.dumps("{json of metadata MDTF format.}") - def run_analysis(self, catalog, png_dir, reference_catalog=None): - """Runs the analysis and generates all plots and associated datasets. + def get_request(url): + """Sends a get request to the input url. Args: - catalog: Path to a model output catalog. - png_dir: Directory to store output png figures in. - reference_catalog: Path to a catalog of reference data. + url: String url to send the get request to. Returns: - A list of png figures. + Dictionary of response body data. + + Raises: + ValueError if the response does not return status 200. """ - raise NotImplementedError("you must override this function.") - return ["figure1.png", "figure2.png",] + response = get(url) + if response.status_code != 200: + print(response.text) + return ValueError("get from {url} failed.") + return response.json() + + + def post_request(url, data): + """Post an http request to dora. + + Args: + url: String url to post the http request to. + data: Dictionary of data that will be passed as json in the body of the request. + + Returns: + String text from the http response. + + Raises: + ValueError if the response does not return status 200. + """ + response = post(url, json=data, auth=(getenv("DORA_TOKEN"), None)) + if response.status_code != 200: + print(response.text) + raise ValueError(f"post to {url} with {data} failed.") + return response.text + + + def parse_experiment_yaml(path): + """Parse the experiment yaml and return a dictionary of the data needed to add the + experiment to dora. + + Args: + path: Path to the experiment yaml. + + Returns: + Dictionary of data needed to add the experiment to dora. + + Raises: + ValueError if the experiment owner cannot be determined. + """ + with open(path) as file_: + yaml_ = safe_load(file_) + + # Determine the username - is this a hack? + history_path_parts = Path(yaml_["directories"]["history_dir"]).parts + user = history_path_parts[2] + if user == "$USER": + user = getenv(user[1:]) + if not user: + raise ValueError(f"Could not identify user {user}.") + + # Expand the paths. + pp_path = yaml_["directories"]["pp_dir"].replace("$USER", user) + database_path = pp_path.replace("archive", "home").replace("pp", "db") # Nasty hack. + analysis_path = yaml_["directories"]["analysis_dir"].replace("$USER", user) + + # Get the model type from the history directory path - is there a better way? + model_type = history_path_parts[3].upper() # Nasty hack. + + # Get the starting and ending years and total length of the experiment. + start = int(yaml_["postprocess"]["settings"]["pp_start"]) + stop = int(yaml_["postprocess"]["settings"]["pp_stop"]) + length = stop - start + 1 + + return { + "expLength": length, + "expName": yaml_["name"], + "expType": yaml_["name"].split("_")[-1].upper(), # Nasty hack. + "expYear": start, + "modelType": model_type, + "owner": user, + "pathAnalysis": analysis_path, + "pathDB": database_path, + "pathPP": pp_path, + "pathXML": path, + "userName": user, + } + + + def get_dora_experiment_id(path): + """Gets the experiment id using a http request after parsing the experiment yaml. + + Args: + path: Path to the experiment yaml. + + Returns: + Integer dora experiment id. + + Raises: + ValueError if the unique experiment (identified by the pp directory path) + cannot be found. + """ + data = parse_experiment_yaml(path) + # url = f"https://dora.gfdl.noaa.gov/api/search?search={data['owner']}" + url = f"http://127.0.0.1:5000/api/search?search={data['owner']}" + response = get_request(url) + for experiment in response.values(): + if experiment["pathPP"] == data["pathPP"]: + return int(experiment["id"]) + raise ValueError("could not find experiment with pp directory - {data['pathPP']}") + + + def add_experiment_to_dora(): + """Adds the experiment to dora using a http request.""" + parser = ArgumentParser() + parser.add_argument("experiment_yaml", help="Path to the experiment yaml.") + args = parser.parse_args() + + # Parse the experiment yaml to get the data needed to add the experiment to dora. + data = parse_experiment_yaml(args.experiment_yaml) + + # Add the experiment to dora. + # url = "https://dora.gfdl.noaa.gov/api/add-experiment" + url = "http://127.0.0.1:5000/api/add-experiment" + post_request(url, data) + + + def run_analysis(): + """Runs the analysis script and writes the paths to the created figures to a yaml file.""" + parser = ArgumentParser() + parser.add_argument("name", help="Name of the analysis script.") + parser.add_argument("catalog", help="Path to the data catalog.") + parser.add_argument("output_directory", help="Path to the output directory.") + parser.add_argument("output_yaml", help="Path to the output yaml.") + parser.add_argument("-c", "--config", help="Path to the configuration yaml.") + args = parser.parse_args() + + # Create the directory for the figures. + Path(args.output_directory).mkdir(parents=True, exist_ok=True) + + # Parse the configuration out of the yaml file. + if args.config: + with open(args.config) as file_: + config_yaml = safe_load(file_) + try: + configuration = config_yaml["analysis"][args.name] + except KeyError: + configuration = None + else: + configuration = None + + # Run the analysis. + figure_paths = run_plugin( + args.name, + args.catalog, + args.output_directory, + config=configuration, + ) + + # Write out the figure paths to a file. + with open(args.output_yaml, "w") as output: + output.write("figure_paths:\n") + for path in figure_paths: + output.write(" -{Path(path).resolve()}\n") + + + def publish_analysis_figures(): + """Uploads the analysis figures to dora.""" + parser = ArgumentParser() + parser.add_argument("name", help="Name of the analysis script.") + parser.add_argument("experiment_yaml", help="Path to the experiment yaml file.") + parser.add_argument("figures_yaml", help="Path to the yaml that contains the figure paths.") + args = parser.parse_args() + + # Check to make sure that the experiment was added to dora and get it id. + dora_id = get_dora_experiment_id(args.experiment_yaml) + + # Parse out the list of paths from the input yaml file and upload them. + url = "https://dora.gfdl.noaa.gov/api/add-png" + data = {"id": dora_id, "name": args.name} + with open(args.figures_yaml) as file_: + paths = safe_load(file_)["figure_paths"] + for path in paths: + data["path"] = path + post_request(url, data) + + +#if __name__ == "__main__": +# add_experiment_to_dora() +# print(get_dora_experiment_id("../../../c96L65_am5f7b12r1_amip.yaml"))