Added template guide, added validation to stac generators

.gitignore

@@ -12,4 +12,10 @@ htmlcov/
 _build/
 dist/
 dev.ipynb
-earthcode/models
+earthcode/models
+guide/project.yaml
+guide/.ipynb_checkpoints/
+guide/product.yaml
+earthcode/generators/project_generator.py
+guide/workflow.yaml
+guide/experiment.yaml

_toc.yml

@@ -11,6 +11,7 @@ parts:
       - file: guide/2.1.Product_files_self_hosted
       - file: guide/3.Workflow
       - file: guide/4.Experiment
+      - file: guide/5.Templates
   - caption: Examples
     chapters:
       - file: examples/earthcode_data_discovery

earthcode/generators/experiment_generator.py

@@ -1,12 +1,14 @@
 from pathlib import Path
 from datetime import datetime
+import json
 import logging
 import sys
 
 import yaml
 
 from earthcode.static import create_experiment_record, ExperimentMetadata
 from earthcode.git_add import save_experiment_record_to_osc
+from earthcode.validator import validateOSCEntry
 
 logging.basicConfig(stream=sys.stdout, encoding='utf-8', level=logging.INFO)
 log = logging.getLogger()
@@ -51,3 +53,11 @@ def create_experiment_stac_from_template(experiment_yaml, osc_path):
     experiment_record = create_experiment_record(experiment_metadata)
 
     save_experiment_record_to_osc(experiment_record, Path(osc_path))
+
+    # validate the saved record, since catalog links are updated when written to disk
+    experiment_path = Path(osc_path) / "experiments" / experiment_record["id"] / "record.json"
+    with open(experiment_path, "r", encoding="utf-8") as f:
+        json_experiment = json.load(f)
+        errors = validateOSCEntry(json_experiment, Path(osc_path))
+        if errors:
+            raise AssertionError(f"Catalog validation failed. errors={len(errors)}\n{errors}")

earthcode/generators/product_generator.py

@@ -1,3 +1,4 @@
+import json
 from pathlib import Path
 from datetime import datetime
 import sys
@@ -8,6 +9,7 @@
 
 from earthcode.static import create_product_collection, ProductCollectionMetadata
 from earthcode.git_add import save_product_collection_to_catalog
+from earthcode.validator import validateOSCEntry
 
 
 logging.basicConfig(stream=sys.stdout, encoding="utf-8", level=logging.INFO)
@@ -70,3 +72,10 @@ def create_product_stac_from_template(project_yaml, osc_path):
     product_collection = create_product_collection(product_metadata)
 
     save_product_collection_to_catalog(product_collection, Path(osc_path))
+
+    # need to run validation on the saved product collection to validate the final json in the OSC repo, not the one created in memory
+    with open(Path(osc_path) / f'products/{product_collection.id}/collection.json', "r", encoding="utf-8") as f:
+        json_product = json.load(f)
+        errors = validateOSCEntry(json_product, Path(osc_path))
+        if errors:
+            raise AssertionError(f"Catalog validation failed. errors={len(errors)}\n{errors}")

earthcode/generators/project_generator.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 from datetime import datetime
+import json
 import logging
 import sys
 
@@ -8,6 +9,7 @@
 
 from earthcode.static import create_project_collection, ProjectCollectionMetadata
 from earthcode.git_add import save_project_collection_to_osc
+from earthcode.validator import validateOSCEntry
 
 logging.basicConfig(stream=sys.stdout, encoding='utf-8', level=logging.INFO)
 log = logging.getLogger()
@@ -32,7 +34,7 @@ def create_project_stac_from_template(project_yaml, osc_path):
         [project_cms.append((member['name'], member['email'])) for member in data['consortium_members']]
 
     project_metadata = ProjectCollectionMetadata(
-            project_id=data['id'] ,
+            project_id=data['id'],
             project_title=data['title'],
             project_description=data['description'],
             project_status=data['status'],
@@ -50,3 +52,11 @@ def create_project_stac_from_template(project_yaml, osc_path):
     project_collection = create_project_collection(project_metadata)
 
     save_project_collection_to_osc(project_collection, Path(osc_path))
+
+    # validate the saved collection, since catalog links are updated when written to disk
+    project_path = Path(osc_path) / "projects" / project_collection.id / "collection.json"
+    with open(project_path, "r", encoding="utf-8") as f:
+        json_project = json.load(f)
+        errors = validateOSCEntry(json_project, Path(osc_path))
+        if errors:
+            raise AssertionError(f"Catalog validation failed. errors={len(errors)}\n{errors}")

earthcode/generators/stac_generator.py

@@ -1,20 +1,20 @@
 import logging
 import sys
 import argparse
-import os
 
 from .experiment_generator import create_experiment_stac_from_template
 from .product_generator import create_product_stac_from_template
 from .project_generator import create_project_stac_from_template
 from .workflow_generator import create_workflow_stac_from_template
+from earthcode.validator import validate_catalog
 
 logging.basicConfig(stream=sys.stdout, encoding='utf-8', level=logging.INFO)
 log = logging.getLogger()
 
 
 def generate_stac(osc_path, project=None, workflow=None, experiment=None, product=None):
     """
-    Generates the requested STAC json files at the specified OSC repo.
+    Generates the requested STAC json files at the specified OSC repo and performs a validation check.
 
     :param osc_path: OSC repo where the STAC json will be created.
     :param project: Path to the Project YAML template, if empty no Project STAC will be generated
@@ -23,6 +23,12 @@ def generate_stac(osc_path, project=None, workflow=None, experiment=None, produc
     :param product: Path to the Product YAML template, if empty no Product STAC will be generated
     """
 
+    if all(t is None for t in [project, workflow, experiment, product]):
+        log.warning("No template provided."
+                    "Run again with at least a provided template to produce the relative STAC json."
+                    "For additional help invoke with -h.")
+        return
+
     if project is not None:
         log.info("Generating Project STAC json in OSC @ \"" + osc_path + "\"")
         create_project_stac_from_template(project, osc_path)
@@ -36,10 +42,10 @@ def generate_stac(osc_path, project=None, workflow=None, experiment=None, produc
         log.info("Generating Product STAC json in OSC @ \"" + osc_path + "\"")
         create_product_stac_from_template(product, osc_path)
 
-    if project is None and workflow is None and experiment is None and product is None:
-        log.warning("No template provided."
-                    "Run again with at least a provided template to produce the relative STAC json."
-                    "For additional help invoke with -h.")
+    # OPTIONAL full catalogue validation
+    # errors, error_files = validate_catalog(osc_path)
+    # if errors or error_files:
+    #     raise AssertionError(f"Catalog validation failed. errors={len(errors)} {errors} files={len(error_files)}")
 
 
 def main():
@@ -52,8 +58,8 @@ def main():
                         help="Experiment YAML template location")
     parser.add_argument("-o", "--product", type=str,
                         help="Product YAML template location")
-    parser.add_argument("-m", "--oscm", type=str,
-                        help="The target OSC location where the STAC jsons will be created.")
+    parser.add_argument("-m", "--oscm", type=str, required=True,
+                        help="REQUIRED The target OSC location where the STAC jsons will be created.")
 
     args = parser.parse_args()
 

earthcode/generators/template_generator.py

@@ -67,7 +67,7 @@ def main():
     parser.add_argument("-o", "--product", action='store_true',
                         help="If present generate a product template")
     parser.add_argument("-t", "--target", type=str,
-                        help="The target location where the templates will be generated.")
+                        help="The target location where the templates will be generated. If empty the CWD will be used")
 
     args = parser.parse_args()
 

earthcode/generators/workflow_generator.py

@@ -1,12 +1,14 @@
 from pathlib import Path
 from datetime import datetime
+import json
 import logging
 import sys
 
 import yaml
 
 from earthcode.git_add import save_workflow_record_to_osc
 from earthcode.static import create_workflow_record, WorkflowMetadata
+from earthcode.validator import validateOSCEntry
 
 logging.basicConfig(stream=sys.stdout, encoding='utf-8', level=logging.INFO)
 log = logging.getLogger()
@@ -50,3 +52,11 @@ def create_workflow_stac_from_template(project_yaml, osc_path):
     workflow_record = create_workflow_record(workflow_metadata)
 
     save_workflow_record_to_osc(workflow_record, Path(osc_path))
+
+    # validate the saved record, since catalog links are updated when written to disk
+    workflow_path = Path(osc_path) / "workflows" / workflow_record["id"] / "record.json"
+    with open(workflow_path, "r", encoding="utf-8") as f:
+        json_workflow = json.load(f)
+        errors = validateOSCEntry(json_workflow, Path(osc_path))
+        if errors:
+            raise AssertionError(f"Catalog validation failed. errors={len(errors)}\n{errors}")