cgi initial commit

.gitignore

@@ -1,6 +1,7 @@
 
 # pixi environments
 .pixi
+.idea
 *.egg-info
 *__pycache__*
 *.json

earthcode/__init__.py

@@ -0,0 +1,2 @@
+from .generators.stac_generator import generate_stac
+from .generators.template_generator import generate_template

earthcode/generators/__init__.py

@@ -0,0 +1,6 @@
+from .experiment_generator import create_experiment_stac_from_template
+from .product_generator import create_product_stac_from_template
+from .project_generator import create_project_stac_from_template
+from .stac_generator import generate_stac
+from .template_generator import generate_template
+from .workflow_generator import create_workflow_stac_from_template

earthcode/generators/experiment_generator.py

@@ -0,0 +1,53 @@
+from pathlib import Path
+from datetime import datetime
+import logging
+import sys
+
+import yaml
+
+from earthcode.static import create_experiment_record, ExperimentMetadata
+from earthcode.git_add import save_experiment_record_to_osc
+
+logging.basicConfig(stream=sys.stdout, encoding='utf-8', level=logging.INFO)
+log = logging.getLogger()
+
+
+def create_experiment_stac_from_template(experiment_yaml, osc_path):
+    with open(experiment_yaml, 'r') as file:
+        data = yaml.safe_load(file)
+
+        for k, v in data.items():
+            if v is None:
+                log.error(f"The Project YAML contains an empty value for the following field: {k}")
+                raise Exception(f"The Project YAML contains an empty value for the following field: {k}")
+
+        temporal_extent = data.get('temporal_extent', None)
+        if temporal_extent is not None:
+            temporal_start = datetime.strptime(temporal_extent['start'], "%Y-%m-%dT%H:%M:%SZ")
+            temporal_end = datetime.strptime(temporal_extent['end'], "%Y-%m-%dT%H:%M:%SZ")
+        else:
+            temporal_start = temporal_end = None
+
+        experiment_metadata = ExperimentMetadata(
+            experiment_id=data['id'],
+            experiment_title=data['title'],
+            experiment_description=data['description'],
+            experiment_license=data['license'],
+            experiment_keywords=data['keywords'],
+            experiment_formats=data['formats'],
+            experiment_themes=data['themes'],
+            experiment_input_parameters_link=data['link_params'],
+            experiment_enviroment_link=data['link_env'],
+            workflow_id=data['workflow'],
+            workflow_title=data['workflow-title'],
+            product_id=data['product'],
+            product_title=data['product-title'],
+            contacts=data.get('contacts', None),
+            experiment_bbox=data.get('spatial_extent', None),
+            experiment_start_datetime=temporal_start,
+            experiment_end_datetime=temporal_end,
+        )
+
+    experiment_record = create_experiment_record(experiment_metadata)
+
+    save_experiment_record_to_osc(experiment_record, Path(osc_path))

earthcode/generators/product_generator.py

@@ -0,0 +1,72 @@
+from pathlib import Path
+from datetime import datetime
+import sys
+import logging
+
+import pystac
+import yaml
+
+from earthcode.static import create_product_collection, ProductCollectionMetadata
+from earthcode.git_add import save_product_collection_to_catalog
+
+
+logging.basicConfig(stream=sys.stdout, encoding="utf-8", level=logging.INFO)
+log = logging.getLogger()
+
+
+def create_product_stac_from_template(project_yaml, osc_path):
+    with open(project_yaml, "r") as file:
+        data = yaml.safe_load(file)
+
+        for k, v in data.items():
+            if v is None:
+                log.error(
+                    f"The Project YAML contains an empty value for the following field: {k}"
+                )
+                raise Exception(
+                    f"The Project YAML contains an empty value for the following field: {k}"
+                )
+
+        # Define spatial and temporal extent
+        spatial_extent = pystac.SpatialExtent(data["extent"]["spatial"]["bbox"]).bboxes
+        temporal_start = datetime.strptime(
+            data["extent"]["temporal"]["start"], "%Y-%m-%dT%H:%M:%SZ"
+        )
+        temporal_end = datetime.strptime(
+            data["extent"]["temporal"]["end"], "%Y-%m-%dT%H:%M:%SZ"
+        )
+
+        # optional
+        product_license = data["license"]
+        if product_license == "other":
+            product_license_link = data["license_link"]
+        else:
+            product_license_link = None
+
+        via_link = data.get("via_link", None)
+
+    product_metadata = ProductCollectionMetadata(
+        product_id=data["id"],
+        product_title=data["title"],
+        product_description=data["description"],
+        product_keywords=data["keywords"],
+        product_status=data["status"],
+        product_region=data["region"],
+        product_themes=data["themes"],
+        product_missions=data["missions"],
+        product_variables=data["variables"],
+        project_id=data["project"],
+        project_title=data["project-title"],
+        product_parameters=data["cf_parameters"],
+        product_doi=data["sci:doi"],
+        product_bbox=spatial_extent,
+        product_start_datetime=temporal_start,
+        product_end_datetime=temporal_end,
+        product_license=product_license,
+        license_link=product_license_link,
+        access_link=via_link,
+    )
+
+    product_collection = create_product_collection(product_metadata)
+
+    save_product_collection_to_catalog(product_collection, Path(osc_path))

earthcode/generators/project_generator.py

@@ -0,0 +1,52 @@
+from pathlib import Path
+from datetime import datetime
+import logging
+import sys
+
+import pystac
+import yaml
+
+from earthcode.static import create_project_collection, ProjectCollectionMetadata
+from earthcode.git_add import save_project_collection_to_osc
+
+logging.basicConfig(stream=sys.stdout, encoding='utf-8', level=logging.INFO)
+log = logging.getLogger()
+
+
+def create_project_stac_from_template(project_yaml, osc_path):
+    with open(project_yaml, 'r') as file:
+        data = yaml.safe_load(file)
+
+        for k, v in data.items():
+            if v is None:
+                log.error(f"The Project YAML contains an empty value for the following field: {k}")
+                raise Exception(f"The Project YAML contains an empty value for the following field: {k}")
+
+        # read spatial and temporal extent
+        spatial_extent = pystac.SpatialExtent(data['extent']['spatial']['bbox']).bboxes
+        temporal_start = datetime.strptime(data['extent']['temporal']['start'], "%Y-%m-%dT%H:%M:%SZ")
+        temporal_end = datetime.strptime(data['extent']['temporal']['end'], "%Y-%m-%dT%H:%M:%SZ")
+
+        # read consortium contacts
+        project_cms = []
+        [project_cms.append((member['name'], member['email'])) for member in data['consortium_members']]
+
+    project_metadata = ProjectCollectionMetadata(
+            project_id=data['id'] ,
+            project_title=data['title'],
+            project_description=data['description'],
+            project_status=data['status'],
+            project_license=data['license'],
+            project_bbox=spatial_extent,
+            project_start_datetime=temporal_start,
+            project_end_datetime=temporal_end,
+            project_themes=data['themes'],
+            to_name=data['to_name'],
+            to_email=data['to_email'],
+            consortium_members=project_cms,
+            website_link=data['link_website'],
+            eo4society_link=data['link_eo4society']
+        )
+    project_collection = create_project_collection(project_metadata)
+
+    save_project_collection_to_osc(project_collection, Path(osc_path))

earthcode/generators/stac_generator.py

@@ -0,0 +1,64 @@
+import logging
+import sys
+import argparse
+import os
+
+from .experiment_generator import create_experiment_stac_from_template
+from .product_generator import create_product_stac_from_template
+from .project_generator import create_project_stac_from_template
+from .workflow_generator import create_workflow_stac_from_template
+
+logging.basicConfig(stream=sys.stdout, encoding='utf-8', level=logging.INFO)
+log = logging.getLogger()
+
+
+def generate_stac(osc_path, project=None, workflow=None, experiment=None, product=None):
+    """
+    Generates the requested STAC json files at the specified OSC repo.
+
+    :param osc_path: OSC repo where the STAC json will be created.
+    :param project: Path to the Project YAML template, if empty no Project STAC will be generated
+    :param workflow: Path to the Workflow YAML template, if empty no Workflow STAC will be generated
+    :param experiment: Path to the Experiment YAML template, if empty no Experiment STAC will be generated
+    :param product: Path to the Product YAML template, if empty no Product STAC will be generated
+    """
+
+    if project is not None:
+        log.info("Generating Project STAC json in OSC @ \"" + osc_path + "\"")
+        create_project_stac_from_template(project, osc_path)
+    if workflow is not None:
+        log.info("Generating Workflow STAC json in OSC @ \"" + osc_path + "\"")
+        create_workflow_stac_from_template(workflow, osc_path)
+    if experiment is not None:
+        log.info("Generating Experiment STAC json in OSC @ \"" + osc_path + "\"")
+        create_experiment_stac_from_template(experiment, osc_path)
+    if product is not None:
+        log.info("Generating Product STAC json in OSC @ \"" + osc_path + "\"")
+        create_product_stac_from_template(product, osc_path)
+
+    if project is None and workflow is None and experiment is None and product is None:
+        log.warning("No template provided."
+                    "Run again with at least a provided template to produce the relative STAC json."
+                    "For additional help invoke with -h.")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-p", "--project", type=str,
+                        help="Project YAML template location")
+    parser.add_argument("-w", "--workflow", type=str,
+                        help="Workflow YAML template location")
+    parser.add_argument("-e", "--experiment", type=str,
+                        help="Experiment YAML template location")
+    parser.add_argument("-o", "--product", type=str,
+                        help="Product YAML template location")
+    parser.add_argument("-m", "--oscm", type=str,
+                        help="The target OSC location where the STAC jsons will be created.")
+
+    args = parser.parse_args()
+
+    generate_stac(args.oscm, args.project, args.workflow, args.experiment, args.product)
+
+
+if __name__ == "__main__":
+    main()

earthcode/generators/template_generator.py

@@ -0,0 +1,78 @@
+import argparse
+import os
+import shutil
+from importlib import resources
+
+import logging
+import sys
+
+logging.basicConfig(stream=sys.stdout, encoding='utf-8', level=logging.INFO)
+log = logging.getLogger()
+
+def generate_template(project=False, workflow=False, experiment=False, product=False, target=os.getcwd()):
+    """
+    Creates requested yaml templates at the desired target folder.
+    If the folder does not exist it will be created.
+    If no folder is specified the PWD where the program is run will be selected.
+
+    :param project: If True: generates the Project yaml template
+    :param workflow: If True: generates the Workflow yaml template
+    :param experiment: If True: generates the Experiment yaml template
+    :param product: If True: generates the Product yaml template
+    :param target: target directory where the templates will be generated.
+    """
+    # If empty use PWD as target directory
+    if target is None:
+        log.warning("No target folder specified, the templates will be generated in the PWD")
+        target = os.getcwd()
+
+    # Create target directory if it doesn't exist
+    if not os.path.isdir(target):
+        os.makedirs(target, exist_ok=True)
+
+    if project:
+        log.info("Generating Project template at \""+target+"\"")
+        with resources.as_file(resources.files("earthcode.generators").joinpath("templates").joinpath("project.yaml")) as path:
+            shutil.copy(path, target)
+
+    if workflow:
+        log.info("Generating Workflow template at \""+target+"\"")
+        with resources.as_file(resources.files("earthcode.generators").joinpath("templates").joinpath("workflow.yaml")) as path:
+            shutil.copy(path, target)
+
+    if experiment:
+        log.info("Generating Experiment template at \""+target+"\"")
+        with resources.as_file(resources.files("earthcode.generators").joinpath("templates").joinpath("experiment.yaml")) as path:
+            shutil.copy(path, target)
+
+    if product:
+        log.info("Generating Product template at \""+target+"\"")
+        with resources.as_file(resources.files("earthcode.generators").joinpath("templates").joinpath("product.yaml")) as path:
+            shutil.copy(path, target)
+
+    if not project and not workflow and not experiment and not product:
+        log.warning("No options selected."
+                    "Run again with at least one option to produce the templates."
+                    "For additional help invoke with -h")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-p", "--project", action='store_true',
+                        help="If present generate a project template")
+    parser.add_argument("-w", "--workflow", action='store_true',
+                        help="If present generate a workflow template")
+    parser.add_argument("-e", "--experiment", action='store_true',
+                        help="If present generate an experiment template")
+    parser.add_argument("-o", "--product", action='store_true',
+                        help="If present generate a product template")
+    parser.add_argument("-t", "--target", type=str,
+                        help="The target location where the templates will be generated.")
+
+    args = parser.parse_args()
+
+    generate_template(args.project, args.workflow, args.experiment, args.product, args.target)
+
+
+if __name__ == "__main__":
+    main()

earthcode/generators/templates/experiment.yaml

@@ -0,0 +1,52 @@
+# Define experiment properties
+id: cool-project-experiment
+title: Experiment related to Cool Project
+description: The first experiment with data from the Cool Project   # Short and meaningful experiment description.
+license: proprietary        # should be one of https:#github.com/ESA-EarthCODE/open-science-catalog-validation/blob/main/schemas/license.json
+keywords:                   # experiment keywords (to enhance the findability of the experiment)
+  - agriculture
+  - crops
+formats:                     # format of experiment output
+  - GeoTIFF
+  - PNG
+workflow: cool-project-workflow                   # id of the workflow used for this experiment
+workflow-title: Workflow to analyze Cool Project  # title of the workflow used for this experiment
+product: cool-project-product                      # id of the output product produced by this experiment
+product-title: Cool Project Product                  # title of the output product produced by this experiment
+
+# Define links, link relations and link titles: e.g. link to service used to run the experiment etc.
+link_env: https://myplatform.com
+link_params: https://github.com/MyExperiment/parameters.git
+
+# Define experiment themes. The fields are restricted to the themes available in the OCS and having at least one theme is mandatory. Check available themes here: https:#opensciencedata.esa.int/themes/catalog
+themes:
+  - land
+  - atmosphere
+
+# Optional contacts
+contacts:
+- name: Technical Officer
+  position: researcher
+  roles:
+    - technical_officer
+  organization: CGI
+  links:
+    - rel: about
+      type: text/html
+      href: https://cgi.com/
+  contact_instructions: Contact preferably through project support page
+
+- name: Junior Staff
+  roles:
+    - assistant
+
+# Optional extent
+spatial_extent:
+    - - -180
+      - -90
+      - 180
+      - 90
+
+temporal_extent:
+    start: '2021-01-01T00:00:00Z'
+    end: '2021-12-31T23:59:59Z'