ocw-parallel multicore support Python3 upgrade

CHANGES.md

@@ -1,6 +1,73 @@
 Apache Open Climate Workbench Change Log
 ========================================
 
+# Release Notes - Apache Open Climate Workbench - Version 1.3.0
+# Release Report - https://s.apache.org/TSNM
+
+Sub-task
+
+    [CLIMATE-529] - Verify that Nio dependency has been removed from mccsearch
+
+Bug
+
+    [CLIMATE-338] - RCMED data source loads datasets improperly based on day attribute of time values
+    [CLIMATE-353] - Bounds doesn't cast input values into proper format
+    [CLIMATE-374] - UI runEvaluation doesn't use $window or $location for results transition
+    [CLIMATE-413] - Extend browser support for ocw-ui/frontend/config/karma.conf.js
+    [CLIMATE-480] - plotter.draw_subregions requires outdated subregion object
+    [CLIMATE-744] - Cannot load TRMM data from RCMED
+    [CLIMATE-756] - Error: Invalid LatLng object: (NaN, NaN) when queuing APHRODITE v.1101 precipitation for the Monsoon Asia domain from RCMED
+    [CLIMATE-790] - SSL errors in the esgf example
+    [CLIMATE-797] - Attribute error in mask_missing_data
+    [CLIMATE-826] - ValueError in model_ensemble_to_rcmed.py
+    [CLIMATE-891] - Make (non-ESGF and pydap) examples script python 3 compatible
+    [CLIMATE-913] - Bugs in CLI
+    [CLIMATE-914] - Update dataset_processor.spatial_regrid module
+    [CLIMATE-915] - Updates for the NARCCAP example configuration files
+    [CLIMATE-917] - Bocumentation build error with Python3
+    [CLIMATE-919] - load_dataset_from_multiple_netcdf_files() does not have default variable units
+    [CLIMATE-922] - Optimize traversing in mccSearch
+    [CLIMATE-929] - Incorrect indices used for temporal subset after trimming
+    [CLIMATE-930] - multiple file loader forces 2D lats and lons
+    [CLIMATE-931] - Documentation build error - missing module
+    [CLIMATE-943] - TypeError: Cannot set property 'uiDateValidator' of undefined
+    [CLIMATE-945] - Grunt Errors On Build Task
+    [CLIMATE-946] - CORDEX script does not necessarily use correct interpreter when calling RCMES
+
+Epic
+
+    [CLIMATE-924] - Changes to RCMES and ocw for CORDEX pilot project
+
+New Feature
+
+    [CLIMATE-316] - Add ESGF Download Script to repository
+    [CLIMATE-366] - Add 'full' temporal rebin option to UI
+    [CLIMATE-426] - Create documentation for working with OCW in Eclipse
+    [CLIMATE-925] - CORDEX Config File Template
+    [CLIMATE-926] - Metadata Extractors
+    [CLIMATE-927] - RCMES script for running multiple evaluations
+
+Improvement
+
+    [CLIMATE-230] - Line colors in time series cycles through too few colors
+    [CLIMATE-348] - Add better default naming for local files
+    [CLIMATE-388] - Make UI backend 'parameters' endpoint more RESTful
+    [CLIMATE-465] - Add class level documentation to all OCW python examples
+    [CLIMATE-625] - Update draw_histogram and draw_marker_on_map documentation
+    [CLIMATE-626] - Update doc strings in Downscaling class
+    [CLIMATE-863] - Address warning, deprecation, etc. within Travis-CI log output
+    [CLIMATE-887] - test_get_netcdf_variable_names setUp to run only once for the test case
+    [CLIMATE-912] - Upgrade mccSearch code from Python2 > 3
+    [CLIMATE-920] - Make examples Python 3 compatible
+    [CLIMATE-928] - temporal_subset should trim edges of dataset times to ensure months divide evenly into years
+    [CLIMATE-938] - Unify format of the time information from a netCDF file
+
+Task
+
+    [CLIMATE-631] - Add documentation on ESGF credential issues
+    [CLIMATE-762] - Automate symlinking ocw-vm frontend to backend to allow for serving of files with bottle.
+    [CLIMATE-881] - Add podaacpy to ocw-conda-dependencies.txt
+
 # Release Notes - Apache Open Climate Workbench - Version 1.2.0
 # Release Report - https://s.apache.org/120release
 

NOTICE.txt

@@ -1,5 +1,5 @@
 Apache Open Climate Workbench
-Copyright 2014 The Apache Software Foundation
+Copyright 2018 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

RCMES/CORDEX/cordex.py

@@ -0,0 +1,62 @@
+import os
+import sys
+import subprocess
+import jinja2
+from metadata_extractor import CORDEXMetadataExtractor, obs4MIPSMetadataExtractor
+
+# These should be modified. TODO: domains can also be made into separate group
+# CORDEX domain
+
+user_input = sys.argv[1:]
+if len(user_input) == 4:
+    domain, workdir, obs_dir, models_dir = user_input[:]
+else:
+    domain = 'NAM-44'
+
+    # The output directory
+    workdir = os.getcwd()+'/'+domain+'_analysis'
+
+    # Location of osb4Mips files
+    obs_dir = '/proj3/data/obs4mips'
+
+    # Location of CORDEX files
+    models_dir = '/proj3/data/CORDEX/{domain}/*'.format(domain=domain)
+
+# Extract metadata from model and obs files, pairing up files with the same
+# variables for separate evaluations
+obs_extractor = obs4MIPSMetadataExtractor(obs_dir)
+models_extractor = CORDEXMetadataExtractor(models_dir)
+groups = obs_extractor.group(models_extractor, 'variable')
+
+# Configuration file template, to be rendered repeatedly for each evaluation
+# run
+env =  jinja2.Environment(loader=jinja2.FileSystemLoader('./templates'),
+                          trim_blocks=True, lstrip_blocks=True)
+t = env.get_template('CORDEX.yaml.template')
+
+# Each group represents a single evaluation. Repeat the evaluation for
+# three seasons: Summer, Winter, and Annual.
+seasons = ['annual', 'winter', 'summer']
+errored = []
+for group in groups:
+    obs_info, models_info = group
+    instrument = obs_info['instrument']
+    variable = obs_info['variable']
+    for season in seasons:
+        configfile_basename = '_'.join([domain, instrument, variable, season]) + '.yaml'
+        configfile_path = os.path.join(workdir, domain, instrument,
+                                       variable, season)
+        if not os.path.exists(configfile_path):
+            os.makedirs(configfile_path)
+        configfile_path = os.path.join(configfile_path, configfile_basename)
+        with open(configfile_path, 'w') as configfile:
+            configfile.write(t.render(obs_info=obs_info, models_info=models_info,
+                                      season=season, output_dir=workdir))
+
+        # TODO: Do this in parallel. Will change this once this approach
+        # is well tested.
+        code = subprocess.call([sys.executable, '../run_RCMES.py', configfile_path])
+        if code:
+            errored.append(configfile_path)
+
+print("All runs done. The following ended with an error: {}".format(errored))

RCMES/CORDEX/metadata_extractor.py

@@ -0,0 +1,245 @@
+import glob
+import os
+
+
+class MetadataExtractor(object):
+    def __init__(self, *paths):
+        """Extracts metadata from data filenames.
+
+        Instances of MetadataExtractor are used to extract metadata from
+        filenames in bulk. Example usage:
+        >>> extractor = MetadataExtractor('/path/to/data')
+
+        Suppose the data in this directory had the following files:
+        pr_*.nc, uas_*.nc, vas_*.nc
+
+        All of the metadata lies in the data attribute:
+        >>> extractor.data
+        [{'filename': /path/to/data/pr_*.nc, 'variable': 'pr'},
+         {'filename': /path/to/data/vas_*.nc, 'variable': 'vas'},
+         {'filename': /path/to/data/uas_*.nc, 'variable': 'uas'}]
+
+        Results can be narrowed down by specifying values for a field:
+        >>> extractor.query(variable='pr')
+        [{'filename': /path/to/data/pr_*.nc, 'variable': 'pr'}]
+
+        Finally, metadata from two sets of extractors can be grouped together
+        based on common field name as follows:
+        >>> extractor.group(extractor2, 'variable')
+
+        This class should only be used as a starting point. We recommend using
+        the included obs4MIPSMetadataExtractor and CORDEXMetadataExtractor
+        subclasses or creating your own subclass for your usecase.
+        """
+        self.paths = paths
+
+    @property
+    def data(self):
+        """
+        The extracted metadata for each file, with all fields listed in
+        the fields attribute included.
+        """
+        return self._data
+
+    @property
+    def paths(self):
+        """
+        Search paths containing the dataset files.
+        """
+        return self._paths
+
+    @paths.setter
+    def paths(self, paths):
+        """
+        Extracts the metadata from scratch when paths are reset.
+        """
+        self._paths = paths
+        self._extract()
+
+    @property
+    def fields(self):
+        """
+        The name of field in the filename, assuming the fully filtered
+        filename conforms to the following convention:
+        filename = <field[0]>_<field[1]>_..._<field[n]>.nc. Using fewer fields
+        than the filename defines is allowed.
+        """
+        fields = ['variable']
+        return fields
+
+    @property
+    def files(self):
+        """
+        List of files (or regular expressions) for each dataset.
+        """
+        files = []
+        for path in self.paths:
+            files.extend(glob.glob(os.path.join(path, '*.nc')))
+        return list(set(self.get_pattern(fname) for fname in files))
+
+    @property
+    def variables(self):
+        """
+        Get the list of variables included accross all the datasets.
+        """
+        return self.get_field('variable')
+
+    @property
+    def field_filters(self):
+        """
+        Override this to filter out specific characters contained in a field.
+        """
+        return dict()
+
+    def query(self, **kwargs):
+        """
+        Narrow down the list of files by field names.
+        """
+        fields = kwargs.keys()
+        if not set(fields).issubset(set(self.fields)):
+            raise ValueError("Invalid fields: {}. Must be subset of: {}"
+                             .format(fields, self.fields))
+        data = self.data
+        for field, value in kwargs.items():
+            value = value if isinstance(value, list) else [value]
+            data = [meta for meta in data
+                    if self._match_filter(meta, field) in value]
+        return data
+
+    def group(self, extractor, field):
+        """
+        Compare the data of this extractor with another extractor instance
+        and group each of their metadata together by given field.
+        """
+        # First we only want to consider values of field which are contained
+        # in both extractors
+        subset = self.get_field(field)
+        other_subset = extractor.get_field(field)
+        intersection = list(subset.intersection(other_subset))
+
+        # Next we will group the datasets in each extractor together by common
+        # field values
+        kwargs = {field: intersection}
+        results = self.query(**kwargs)
+
+        groups = []
+        for meta in results:
+            val = self._match_filter(meta, field)
+            kwargs.update({field: val})
+            match = extractor.query(**kwargs)
+            groups.append((meta, match))
+
+        return groups
+
+    def get_field(self, field):
+        """
+        Returns only the selected field of the extracted data.
+        """
+        if field not in self.fields:
+            raise ValueError("Invalid field: {}. Must be one of: {}"
+                             .format(field, self.fields))
+        sub = set(meta[field] for meta in self.data)
+        return sub
+
+    def filter_filename(self, fname):
+        """
+        Applies a filter to each individual filename contained in the _files
+        attribute, which is useful if some files within a data set are known
+        to not follow conventions, and "fix" them so that they do.
+        """
+        return os.path.basename(fname)
+
+    def get_pattern(self, fname):
+        """
+        Used to group multiple file datasets together via regular expresssions.
+        The most common convention is to split files by time periods, which
+        are generally the last field in a filename.
+        """
+        base = fname.split('_')
+        pattern = '_'.join(base[:len(self.fields)] + ['*.nc'])
+        return pattern
+
+    def _match_filter(self, meta, field):
+        """
+        Filter (ignore) certain character patterns when matching a field.
+        """
+        val = meta[field]
+        if field in self.field_filters:
+            for pattern in self.field_filters[field]:
+                val = val.replace(pattern, '')
+        return val
+
+    def _extract(self):
+        """
+        Do the actual metadata extraction from the list of filename given
+        via filter_filelist(). Additionally, filenames can also be filtered
+        via filter_filename() to remove unwanted characters from the extraction.
+        """
+        self._data = []
+        for fname in self.files:
+            meta = dict(filename=fname)
+
+            # Perform the actual metadata extraction
+            fname = self.filter_filename(fname)
+            meta.update(dict(zip(self.fields, fname.split('_')[:-1])))
+            self._data.append(meta)
+
+
+class obs4MIPSMetadataExtractor(MetadataExtractor):
+    @property
+    def instruments(self):
+        """
+        Get the list of instruments accross all the datasets.
+        """
+        return self.get_field('instrument')
+
+    @property
+    def fields(self):
+        """
+        obs4MIPs fields
+        """
+        fields = ['variable', 'instrument', 'processing_level', 'version']
+        return fields
+
+    @property
+    def field_filters(self):
+        """
+        Field filters for CALIPSO
+        """
+        return dict(variable=['calipso', 'Lidarsr532'])
+
+    def filter_filename(self, fname):
+        """
+        CALIPSO files have odd naming conventions, so we will use
+        a modified version to conform to standard obs4MIPs conventions.
+        """
+        fname = os.path.basename(fname)
+        fname = fname.replace('_obs4MIPs_', '_')
+        return fname
+
+    def get_pattern(self, fname):
+        """
+        Overriden to deal with CALIPSO filenames
+        """
+        base = fname.split('_')
+        offset = -2 if len(base) != 5 else -1
+        pattern = '_'.join(base[:offset] + ['*.nc'])
+        return pattern
+
+
+class CORDEXMetadataExtractor(MetadataExtractor):
+    @property
+    def models(self):
+        """
+        Get the list of models accross all the datasets.
+        """
+        return self.get_field('model')
+
+    @property
+    def fields(self):
+        """
+        obs4MIPs fields
+        """
+        fields = ['variable', 'domain', 'driving_model', 'experiment',
+                  'ensemble', 'model', 'version', 'time_step']
+        return fields