diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ba7b2521..86982a64 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -111,10 +111,10 @@ jobs: ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }} - KEYCLOAK_STAGING_SM2A_FAB_CLIENT_SECRET: ${{ secrets.KEYCLOAK_STAGING_SM2A_FAB_CLIENT_SECRET }} + STAGING_SM2A_ADMIN_USERNAME: ${{ secrets.STAGING_SM2A_ADMIN_USERNAME }} + STAGING_SM2A_ADMIN_PASSWORD: ${{ secrets.STAGING_SM2A_ADMIN_PASSWORD }} STAGING_SM2A_API_URL: ${{ vars.STAGING_SM2A_API_URL }} DATASET_DAG_NAME: ${{ vars.DATASET_DAG_NAME }} - KEYCLOAK_STAGING_URL: ${{ vars.KEYCLOAK_STAGING_URL }} run: | pip install -r ./scripts/requirements.txt diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index 54c19044..7bd3641e 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -61,10 +61,10 @@ jobs: - name: Publish to production on PR merge env: + SM2A_ADMIN_USERNAME: ${{ secrets.SM2A_ADMIN_USERNAME }} + SM2A_ADMIN_PASSWORD: ${{ secrets.SM2A_ADMIN_PASSWORD }} SM2A_API_URL: ${{ vars.SM2A_API_URL }} PROMOTION_DAG: ${{ vars.PROMOTION_DAG_NAME }} - KEYCLOAK_PROD_URL: ${{ vars.KEYCLOAK_PROD_URL }} - KEYCLOAK_PROD_SM2A_FAB_CLIENT_SECRET: ${{ secrets.KEYCLOAK_PROD_SM2A_FAB_CLIENT_SECRET }} run: | pip install -r ./scripts/requirements.txt diff --git a/scripts/promote_collection.py b/scripts/promote_collection.py index cb780c11..abc26e69 100644 --- a/scripts/promote_collection.py +++ b/scripts/promote_collection.py @@ -5,7 +5,7 @@ import sys import os import uuid -import requests +from base64 import b64encode def trigger_collection_dag(payload: Dict[str, Any], stage: str): @@ -16,41 +16,34 @@ def trigger_collection_dag(payload: Dict[str, Any], stage: str): if stage == "staging": api_url_env = "STAGING_SM2A_API_URL" - token_url = f"https://{os.getenv('KEYCLOAK_STAGING_URL')}/realms/veda/protocol/openid-connect/token" - client_id = "airflow-webserver-fab" - client_secret = os.getenv("KEYCLOAK_STAGING_SM2A_FAB_CLIENT_SECRET") + username_env = "STAGING_SM2A_ADMIN_USERNAME" + password_env = "STAGING_SM2A_ADMIN_PASSWORD" elif stage == "production": api_url_env = "SM2A_API_URL" - token_url = f"https://{os.getenv('KEYCLOAK_PROD_URL')}/realms/veda/protocol/openid-connect/token" - client_id = "airflow-webserver-fab" - client_secret = os.getenv("KEYCLOAK_PROD_SM2A_FAB_CLIENT_SECRET") + username_env = "SM2A_ADMIN_USERNAME" + password_env = "SM2A_ADMIN_PASSWORD" else: raise ValueError( f"Invalid stage provided: {stage}. Must be 'staging' or 'production'." ) base_api_url = os.getenv(api_url_env) + username = os.getenv(username_env) + password = os.getenv(password_env) - response = requests.post( - token_url, - data={ - "client_id": client_id, - "client_secret": client_secret, - "grant_type": "client_credentials", - }, - ) - access_token = response.json()["access_token"] - - if not all([base_api_url, access_token]): + if not all([base_api_url, username, password]): raise ValueError( f"Missing one or more environment variables: " f"stage is None={stage is None}, " - f"access_token is None={access_token is None}" + f"username is None={username_env is None}, " + f"password is None={password_env is None}" ) + api_token = b64encode(f"{username}:{password}".encode()).decode() + headers = { "Content-Type": "application/json", - "Authorization": "Bearer " + access_token, + "Authorization": "Basic " + api_token, } body = { diff --git a/scripts/promote_dataset.py b/scripts/promote_dataset.py index 6ee4b633..e2ea38fd 100644 --- a/scripts/promote_dataset.py +++ b/scripts/promote_dataset.py @@ -5,7 +5,7 @@ import sys import os import uuid -import requests +from base64 import b64encode class MissingFieldError(Exception): @@ -35,30 +35,21 @@ def validate_discovery_item_config(item: Dict[str, Any]) -> Dict[str, Any]: def publish_to_staging(payload): base_api_url = os.getenv("STAGING_SM2A_API_URL") dataset_pipeline_dag = os.getenv("DATASET_DAG_NAME", "veda_dataset_pipeline") + username = os.getenv("STAGING_SM2A_ADMIN_USERNAME") + password = os.getenv("STAGING_SM2A_ADMIN_PASSWORD") - token_url = f"https://{os.getenv('KEYCLOAK_STAGING_URL')}/realms/veda/protocol/openid-connect/token" - client_id = "airflow-webserver-fab" - client_secret = os.getenv("KEYCLOAK_STAGING_SM2A_FAB_CLIENT_SECRET") - - response = requests.post( - token_url, - data={ - "client_id": client_id, - "client_secret": client_secret, - "grant_type": "client_credentials", - }, - ) - access_token = response.json()["access_token"] + api_token = b64encode(f"{username}:{password}".encode()).decode() - if not base_api_url or not access_token: + if not base_api_url or not api_token: raise ValueError( - "STAGING_SM2A_API_URL or KEYCLOAK_STAGING_SM2A_FAB_CLIENT_SECRET is not" + "STAGING_SM2A_API_URL or STAGING_SM2A_ADMIN_USERNAME" + + " or STAGING_SM2A_ADMIN_PASSWORD is not" + " set in the environment variables." ) headers = { "Content-Type": "application/json", - "Authorization": "Bearer " + access_token, + "Authorization": "Basic " + api_token, } body = { @@ -85,30 +76,20 @@ def publish_to_staging(payload): def promote_to_production(payload): base_api_url = os.getenv("SM2A_API_URL") promotion_dag = os.getenv("PROMOTION_DAG_NAME", "veda_promotion_pipeline") + username = os.getenv("SM2A_ADMIN_USERNAME") + password = os.getenv("SM2A_ADMIN_PASSWORD") - token_url = f"https://{os.getenv('KEYCLOAK_PROD_URL')}/realms/veda/protocol/openid-connect/token" - client_id = "airflow-webserver-fab" - client_secret = os.getenv("KEYCLOAK_PROD_SM2A_FAB_CLIENT_SECRET") - - response = requests.post( - token_url, - data={ - "client_id": client_id, - "client_secret": client_secret, - "grant_type": "client_credentials", - }, - ) - access_token = response.json()["access_token"] + api_token = b64encode(f"{username}:{password}".encode()).decode() - if not base_api_url or not access_token: + if not base_api_url or not api_token: raise ValueError( - "SM2A_API_URL or KEYCLOAK_PRODUCTION_SM2A_FAB_CLIENT_SECRET is not" + "SM2A_API_URL or SM2A_ADMIN_USERNAME or SM2A_ADMIN_PASSWORD is not" + " set in the environment variables." ) headers = { "Content-Type": "application/json", - "Authorization": "Bearer " + access_token, + "Authorization": "Basic " + api_token, } payload["conf"]["transfer"] = True diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 1c6d8b40..4818cc54 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,2 +1 @@ -pyyaml -requests +pyyaml \ No newline at end of file