Merge pull request #454 from unity-sds/453-smce-deployment

453 smce deployment

Author: grallewellyn

airflow/config/webserver_config.py

@@ -25,13 +25,11 @@
 from airflow.utils.trigger_rule import TriggerRule
 from kubernetes.client import models as k8s
 from unity_sps_utils import (
-    CS_SHARED_SERVICES_ACCOUNT_ID,
-    CS_SHARED_SERVICES_ACCOUNT_REGION,
     DEFAULT_LOG_LEVEL,
-    DS_COGNITO_CLIENT_ID,
     DS_S3_BUCKET_PARAM,
     EC2_TYPES,
     LOG_LEVEL_TYPE,
+    MDPS_CLIENT_ID,
     NODE_POOL_DEFAULT,
     NODE_POOL_HIGH_WORKLOAD,
     POD_LABEL,
@@ -175,16 +173,9 @@ def select_stage_in(ti, stac_json, unity_stac_auth_type):
     """Retrieve stage in arguments based on authentication type parameter."""
     stage_in_args = {"stac_json": stac_json, "stac_auth_type": "NONE"}
     if unity_stac_auth_type:
-        shared_services_account = SSM_CLIENT.get_parameter(
-            Name=CS_SHARED_SERVICES_ACCOUNT_ID, WithDecryption=True
-        )["Parameter"]["Value"]
-        shared_services_region = SSM_CLIENT.get_parameter(
-            Name=CS_SHARED_SERVICES_ACCOUNT_REGION, WithDecryption=True
-        )["Parameter"]["Value"]
-        unity_client_id = SSM_CLIENT.get_parameter(
-            Name=f"arn:aws:ssm:{shared_services_region}:{shared_services_account}:parameter{DS_COGNITO_CLIENT_ID}",
-            WithDecryption=True,
-        )["Parameter"]["Value"]
+        unity_client_id = SSM_CLIENT.get_parameter(Name=MDPS_CLIENT_ID, WithDecryption=True)["Parameter"][
+            "Value"
+        ]
         stage_in_args["unity_client_id"] = unity_client_id
         stage_in_args["stac_auth_type"] = "UNITY"
 

airflow/dags/cwl_dag_modular.py

@@ -19,13 +19,13 @@
 from kubernetes.client import models as k8s
 from unity_sps_utils import POD_LABEL, POD_NAMESPACE, get_affinity
 
-PROCESSES_ENDPOINT = "https://api.dit.maap-project.org/api/ogc/processes"
+API_HOST = "https://api.dit.maap-project.org/api/"
 
 
 def fetch_ogc_processes():
     """Fetch available processes from the OGC API and create mapping."""
     try:
-        response = requests.get(PROCESSES_ENDPOINT, timeout=30)
+        response = requests.get(API_HOST + "ogc/processes", timeout=30)
         response.raise_for_status()
 
         processes_data = response.json()
@@ -65,7 +65,8 @@ def fetch_ogc_processes():
 
 # Constants
 K8S_SECRET_NAME = "sps-app-credentials"
-DOCKER_IMAGE = "jplmdps/ogc-job-runner:latest"
+# This docker image is generated by the files in docker/run_ogc_process
+DOCKER_IMAGE = "jplmdps/ogc-job-runner:v1.0.0"
 PROCESS_MAPPING, DROPDOWN_OPTIONS = fetch_ogc_processes()
 
 # SPS-specific secrets
@@ -155,7 +156,7 @@ def _build_submit_env_vars(self):
         return [
             k8s.V1EnvVar(
                 name="SUBMIT_JOB_URL",
-                value="https://api.dit.maap-project.org/api/ogc/processes/{process_id}/execution",
+                value=API_HOST + "ogc/processes/{process_id}/execution",
             ),
             k8s.V1EnvVar(name="PROCESS_ID", value=str(numerical_process_id)),
             k8s.V1EnvVar(name="JOB_INPUTS", value=self.job_inputs or "{}"),
@@ -168,7 +169,7 @@ def _build_monitor_env_vars(self):
         return [
             k8s.V1EnvVar(
                 name="MONITOR_JOB_URL",
-                value="https://api.dit.maap-project.org/api/ogc/jobs/{job_id}",
+                value=API_HOST + "ogc/jobs/{job_id}",
             ),
             k8s.V1EnvVar(name="JOB_ID", value=self.job_id),
             k8s.V1EnvVar(name="SUBMIT_JOB", value="false"),

airflow/dags/run_ogc_process.py

@@ -19,7 +19,7 @@ if [ "$SUBMIT_JOB" = "true" ] || [ "$SUBMIT_JOB" = "True" ]; then
     --data "${SUBMIT_JOB_ARGUMENTS}")
 
     echo "API Response: $response"
-    job_id=$(echo "$response" | jq -r .id)
+    job_id=$(echo "$response" | jq -r .jobID)
 
     if [ "$job_id" = "null" ] || [ -z "$job_id" ]; then
         echo "Failed to get jobID from response."

airflow/docker/run_ogc_process/run_ogc_process_entrypoint.sh

@@ -165,10 +165,6 @@ webserverSecretKeySecretName: ${webserver_secret_name}
 webserver:
   replicas: 3
 
-  # Issue 404: DISABLE AIRRLOW AUTHENTICATION (https://github.com/unity-sds/unity-sps/issues/404)
-  webserverConfig: |-
-    ${webserver_config}
-
   startupProbe:
     timeoutSeconds: 20
     failureThreshold: 60 # Number of tries before giving up (10 minutes with periodSeconds of 10)

airflow/helm/values.tmpl.yaml

@@ -165,10 +165,6 @@ webserverSecretKeySecretName: ${webserver_secret_name}
 webserver:
   replicas: 3
 
-  # Issue 404: DISABLE AIRRLOW AUTHENTICATION (https://github.com/unity-sds/unity-sps/issues/404)
-  webserverConfig: |-
-    ${webserver_config}
-
   startupProbe:
     timeoutSeconds: 20
     failureThreshold: 60 # Number of tries before giving up (10 minutes with periodSeconds of 10)

airflow/helm/values_high_load.tmpl.yaml

@@ -20,8 +20,8 @@
 
 CS_SHARED_SERVICES_ACCOUNT_ID = "/unity/shared-services/aws/account"
 CS_SHARED_SERVICES_ACCOUNT_REGION = "/unity/shared-services/aws/account/region"
-DS_COGNITO_CLIENT_ID = "/unity/shared-services/dapa/client-id"
-DS_S3_BUCKET_PARAM = f"/unity/unity/{os.environ['AIRFLOW_VAR_UNITY_VENUE']}/ds/datastore-bucket"
+MDPS_CLIENT_ID = "/sps/processing/workflows/unity_client_id"
+DS_S3_BUCKET_PARAM = f"/smce/mdps/{os.environ['AIRFLOW_VAR_UNITY_VENUE']}/ds/datastore-bucket"
 
 DEFAULT_LOG_LEVEL = "INFO"
 LOG_LEVEL_TYPE = {"DEBUG": 10, "INFO": 20, "WARNING": 30, "ERROR": 40, "CRITICAL": 50}

airflow/plugins/unity_sps_utils.py

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "unity-sps"
-version = "3.1.0"
+version = "3.1.1"
 authors = [
     { name = "Luca Cinquini", email = "luca.cinquini@jpl.nasa.gov" },
     { name = "Grace Llewellyn", email = "grace.llewellyn@jpl.nasa.gov" },