From c85818ccb2ca2495ca03102966c5abc782471b39 Mon Sep 17 00:00:00 2001 From: skyflow-srivyshnavi Date: Wed, 19 Nov 2025 13:06:36 +0530 Subject: [PATCH 1/2] IE-521: updated pipeline workflow, added code comments --- .github/workflows/migrate_pipelines.yml | 29 ++++--------------------- README.md | 5 +++-- migrate_connections.py | 6 +++++ migrate_pipelines.py | 13 ++++++----- migrate_policies.py | 4 ++++ migrate_roles.py | 10 +++++++++ migrate_service_accounts.py | 8 ++++++- migrate_vault_roles_and_policies.py | 2 ++ migrate_vault_schema.py | 5 ++++- 9 files changed, 47 insertions(+), 35 deletions(-) diff --git a/.github/workflows/migrate_pipelines.yml b/.github/workflows/migrate_pipelines.yml index c5dca2a..8aa6b56 100644 --- a/.github/workflows/migrate_pipelines.yml +++ b/.github/workflows/migrate_pipelines.yml @@ -16,41 +16,21 @@ on: description: "Source datastore JSON. Provide either an ftpServer or s3Bucket object." required: true default: | - { - "ftpServer": { - "transferProtocol": "FTPS", - "plainText": { - "hostname": "", - "port": "", - "username": "", - "password": "", - }, - "skyflowHosted": false - } - } + {"ftpServer":{"transferProtocol": "FTPS","plainText": {"hostname": "","port": "","username": "","password": ""},"skyflowHosted": false}} target_datastore_config: description: "Destination datastore JSON. Provide either an ftpServer or s3Bucket object." required: true default: | - { - "s3Bucket": { - "name": "", - "region": "", - "assumedRoleARN": "" - } - } - source_vault_id: - description: "Source Vault ID." - required: false + {"s3Bucket": {"name": "","region": "","assumedRoleARN": ""}} pipeline_id: description: "PipelineID to be migrated." - required: false + required: true default: "" target_vault_id: description: "Target Vault ID" required: true source_account_access_token: - description: "Access token of the Source Account. (Not required, if config file is selected)" + description: "Access token of the Source Account" required: false target_account_access_token: description: "Access token of the Target Account" @@ -108,7 +88,6 @@ jobs: PIPELINE_ID: ${{ github.event.inputs.pipeline_id }} SOURCE_DATASTORE_CONFIG: ${{ github.event.inputs.source_datastore_config }} TARGET_DATASTORE_CONFIG: ${{ github.event.inputs.target_datastore_config }} - SOURCE_VAULT_ID: ${{ github.event.inputs.source_vault_id }} TARGET_VAULT_ID: ${{ github.event.inputs.target_vault_id }} SOURCE_ACCOUNT_AUTH: ${{ github.event.inputs.source_account_access_token }} TARGET_ACCOUNT_AUTH: ${{ github.event.inputs.target_account_access_token }} diff --git a/README.md b/README.md index 7dc9584..f6312c5 100644 --- a/README.md +++ b/README.md @@ -117,11 +117,12 @@ Note: Please note that if all values are provided `config_file` will take the pr ### Pipelines Migration -Migrates a pipeline definition from the source vault to the target vault. +Migrates a pipeline from the source vault to the target vault. ##### Parameters: - **`source_and_target_env`**: Source and Target Env's. -- **`pipeline_id`**: Pipeline ID to migrate. Get the pipeline ID from Studio. +- **`pipeline_id`**: Pipeline ID to migrate. +- - **`target_vault_id`**: Target Vault ID. - **`source_datastore_config`**: JSON object that replaces the source datastore configuration. Provide either an `ftpServer` or `s3Bucket` object with the required credentials. - **`target_datastore_config`**: JSON object that replaces the destination datastore configuration. Provide either an `ftpServer` or `s3Bucket` object with the required credentials. - **`source_account_access_token`**: Access token of the source account. diff --git a/migrate_connections.py b/migrate_connections.py index 831f68a..443391c 100644 --- a/migrate_connections.py +++ b/migrate_connections.py @@ -29,6 +29,7 @@ def list_connections(vault_id): + """Lists inbound + outbound connections for a vault.""" connections = [] response = requests.get( f"{SOURCE_ENV_URL}/v1/gateway/outboundRoutes?vaultID={vault_id}", @@ -45,6 +46,7 @@ def list_connections(vault_id): return connections def get_connection(connection_id): + """Fetches a single connection""" # /inboundRoutes can also fetch outbound connection details response = requests.get( f"{SOURCE_ENV_URL}/v1/gateway/inboundRoutes/{connection_id}", @@ -54,6 +56,7 @@ def get_connection(connection_id): return response.json() def create_connection(connection): + """Creates connection""" route = "outboundRoutes" if connection["mode"] == "EGRESS" else "inboundRoutes" response = requests.post( f"{TARGET_ENV_URL}/v1/gateway/{route}", @@ -64,8 +67,10 @@ def create_connection(connection): def transform_connection_payload(source_resource): + """Transforms source connection payload to target payload.""" transformed_resource = source_resource transformed_resource["vaultID"] = TARGET_VAULT_ID + # drop basic audit and invocation URL if "BasicAudit" in transformed_resource.keys(): del transformed_resource["BasicAudit"] for route in transformed_resource["routes"]: @@ -74,6 +79,7 @@ def transform_connection_payload(source_resource): def main(connection_ids=None): + """Migrates connections""" try: print("-- Initiating Connections migration --") connections = [] diff --git a/migrate_pipelines.py b/migrate_pipelines.py index 092f5a5..2604262 100644 --- a/migrate_pipelines.py +++ b/migrate_pipelines.py @@ -4,9 +4,10 @@ import requests from typing import Any, Dict, List, Optional +from dotenv import load_dotenv +load_dotenv() PIPELINE_ID = os.getenv("PIPELINE_ID") -SOURCE_VAULT_ID = os.getenv("SOURCE_VAULT_ID") TARGET_VAULT_ID = os.getenv("TARGET_VAULT_ID") SOURCE_ACCOUNT_ID = os.getenv("SOURCE_ACCOUNT_ID") TARGET_ACCOUNT_ID = os.getenv("TARGET_ACCOUNT_ID") @@ -35,7 +36,7 @@ } def list_pipelines(vault_id: str) -> List[Dict[str, Any]]: - """Return all pipelines in the supplied vault.""" + """Lists Pipelines""" pipelines = [] response = requests.get( f"{SOURCE_ENV_URL}/v1/pipelines?vaultID={vault_id}", @@ -46,7 +47,7 @@ def list_pipelines(vault_id: str) -> List[Dict[str, Any]]: return pipelines def get_pipeline(pipeline_id: str) -> Dict[str, Any]: - """Fetch a single pipeline definition from the source environment.""" + """Fetches a single pipeline""" response = requests.get( f"{SOURCE_ENV_URL}/v1/pipelines/{pipeline_id}", headers=SOURCE_ACCOUNT_HEADERS, @@ -55,7 +56,7 @@ def get_pipeline(pipeline_id: str) -> Dict[str, Any]: return response.json()[PIPELINE] def create_pipeline(pipeline: Dict[str, Any]) -> requests.Response: - """Create a pipeline in the target environment.""" + """Creates a pipeline""" response = requests.post( f"{TARGET_ENV_URL}/v1/pipelines", json=pipeline, @@ -170,7 +171,7 @@ def transform_pipeline_payload( source_datastore_input: Optional[Dict[str, Any]] = None, target_datastore_input: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: - """Prepare the payload for the target pipeline.""" + """Transforms source pipeline payload to target payload.""" transformed_resource = copy.deepcopy(source_resource) if "ID" in transformed_resource: del transformed_resource["ID"] # remove pipeline ID @@ -187,7 +188,7 @@ def transform_pipeline_payload( def main(pipeline_id: str) -> None: - """pipeline migration""" + """Migrates pipeline""" try: print("-- Initiating Pipelines migration --") source_datastore_input = load_datastore_input(SOURCE_DATASTORE_CONFIG, "source") diff --git a/migrate_policies.py b/migrate_policies.py index 17d4488..547ca5b 100644 --- a/migrate_policies.py +++ b/migrate_policies.py @@ -26,6 +26,7 @@ def get_policy(policy_id): + """Fetches a policy""" response = requests.get( f"{SOURCE_ENV_URL}/v1/policies/{policy_id}", headers=SOURCE_ACCOUNT_HEADERS ) @@ -34,6 +35,7 @@ def get_policy(policy_id): def create_policy(policy_data): + """Creates a policy""" response = requests.post( f"{TARGET_ENV_URL}/v1/policies", json=policy_data, headers=TARGET_ACCOUNT_HEADERS ) @@ -42,6 +44,7 @@ def create_policy(policy_data): def transform_policy_payload(source_resource): + """Transforms source policy payload to target payload.""" transformed_resource = source_resource["policy"] transformed_resource["resource"] = {"ID": TARGET_VAULT_ID, "type": "VAULT"} policy_rules = transformed_resource["rules"] @@ -95,6 +98,7 @@ def transform_policy_payload(source_resource): def main(policy_ids=None): + """Migrates policies""" try: policy_ids = policy_ids if policy_ids else ast.literal_eval(POLICY_IDS) policies_created = [] diff --git a/migrate_roles.py b/migrate_roles.py index 54ca5ec..0346e87 100644 --- a/migrate_roles.py +++ b/migrate_roles.py @@ -32,6 +32,7 @@ def get_role(role_id): + """Fetch a single role definition from the source account.""" response = requests.get( f"{SOURCE_ENV_URL}/v1/roles/{role_id}", headers=SOURCE_ACCOUNT_HEADERS ) @@ -39,6 +40,7 @@ def get_role(role_id): return response.json() def get_system_role(role_name): + """Return a system role present in the target vault.""" response = requests.get( f"{TARGET_ENV_URL}/v1/roles?name={role_name}&resource.type=VAULT&resource.ID={TARGET_VAULT_ID}", headers=TARGET_ACCOUNT_HEADERS ) @@ -46,6 +48,7 @@ def get_system_role(role_name): return response.json() def create_role(role): + """Create a custom role in the target vault.""" response = requests.post( f"{TARGET_ENV_URL}/v1/roles", json=role, headers=TARGET_ACCOUNT_HEADERS ) @@ -54,6 +57,7 @@ def create_role(role): def get_role_policies(role_id): + """List all policies attached to the given role.""" response = requests.get( f"{SOURCE_ENV_URL}/v1/roles/{role_id}/policies", headers=SOURCE_ACCOUNT_HEADERS ) @@ -62,6 +66,7 @@ def get_role_policies(role_id): def get_role_by_role_name(role_name): + """Search the target vault for an existing custom role by name.""" response = requests.get( f"{TARGET_ENV_URL}/v1/roles?name={role_name}&resource.type=VAULT&resource.ID={TARGET_VAULT_ID}", headers=TARGET_ACCOUNT_HEADERS, @@ -71,6 +76,7 @@ def get_role_by_role_name(role_name): def assign_policy_to_role(policy_ids, role_id: list): + """Assign the provided policies to the role.""" for policy_id in policy_ids: assign_request = {"ID": policy_id, "roleIDs": role_id} response = requests.post( @@ -82,6 +88,7 @@ def assign_policy_to_role(policy_ids, role_id: list): # return response.json() def list_all_roles() -> list: + """Lists custom roles""" response = requests.get( f"{SOURCE_ENV_URL}/v1/roles?type=CUSTOM&resource.ID={SOURCE_VAULT_ID}&resource.type=VAULT", headers=SOURCE_ACCOUNT_HEADERS, @@ -91,6 +98,7 @@ def list_all_roles() -> list: def transform_role_payload(source_resource): + """Transforms source role payload to target payload.""" transformed_resource = {} transformed_resource["roleDefinition"] = source_resource["role"]["definition"] permissions: list = source_resource["role"]["definition"]["permissions"] @@ -104,11 +112,13 @@ def transform_role_payload(source_resource): "vaults.read:upstream", ] ] + # remove upstream read permissions that are implicitly granted in the target account transformed_resource["roleDefinition"]["permissions"] = new_permissions transformed_resource["resource"] = {"ID": TARGET_VAULT_ID, "type": "VAULT"} return transformed_resource def main(role_ids=None): + """Migrates roles and their associated policies.""" try: print("-- Initializing Roles migration --") should_enable_custom_role_check = SKIP_ROLE_CREATION_IF_ROLE_EXISTS diff --git a/migrate_service_accounts.py b/migrate_service_accounts.py index 9d9edf9..814a4b1 100644 --- a/migrate_service_accounts.py +++ b/migrate_service_accounts.py @@ -27,6 +27,7 @@ def list_service_account_roles(service_account_id): + """Return every role assigned to the given service account.""" response = requests.get( f"{SOURCE_ENV_URL}/v1/members/{service_account_id}/roles?member.type=SERVICE_ACCOUNT", headers=SOURCE_ACCOUNT_HEADERS, @@ -36,6 +37,7 @@ def list_service_account_roles(service_account_id): def get_service_account(service_account_id): + """Fetch a service account definition from the source account.""" response = requests.get( f"{SOURCE_ENV_URL}/v1/serviceAccounts/{service_account_id}", headers=SOURCE_ACCOUNT_HEADERS, @@ -45,6 +47,7 @@ def get_service_account(service_account_id): def create_service_account(service_account): + """Create the supplied service account in the target account.""" response = requests.post( f"{TARGET_ENV_URL}/v1/serviceAccounts", json=service_account, @@ -55,6 +58,7 @@ def create_service_account(service_account): def assign_roles_to_service_account(role_ids, service_account_id): + """Attach the specified roles to the new service account.""" for role_id in role_ids: assign_request = { "ID": role_id, @@ -69,15 +73,17 @@ def assign_roles_to_service_account(role_ids, service_account_id): def transform_service_account_payload(source_resource): + """Strip source-only metadata before creating the service account.""" transformed_resource = source_resource # transformed_resource["resource"] = {"ID": TARGET_VAULT_ID, "type": "VAULT"} // not require due to SA flattening change - del transformed_resource["serviceAccount"]["ID"] + del transformed_resource["serviceAccount"]["ID"] # IDs are regenerated per account del transformed_resource["serviceAccount"]["namespace"] del transformed_resource["serviceAccount"]["BasicAudit"] return transformed_resource def main(service_accounts_ids=None): + """Migrates service accounts and associated roles.""" try: print("-- Initializing Service accounts migration --") service_accounts_ids = ( diff --git a/migrate_vault_roles_and_policies.py b/migrate_vault_roles_and_policies.py index c9e1f61..4ebc91f 100644 --- a/migrate_vault_roles_and_policies.py +++ b/migrate_vault_roles_and_policies.py @@ -16,6 +16,7 @@ def list_all_vault_custom_roles() -> list: + """Return all custom roles of the source vault.""" response = requests.get( f"{SOURCE_ENV_URL}/v1/roles?type=CUSTOM&resource.ID={SOURCE_VAULT_ID}&resource.type=VAULT", headers=SOURCE_ACCOUNT_HEADERS, @@ -25,6 +26,7 @@ def list_all_vault_custom_roles() -> list: def main(): + """Migrates all the custom roles and policies""" try: print(f"-- Fetching roles for the vault {SOURCE_VAULT_ID} --") roles = list_all_vault_custom_roles() diff --git a/migrate_vault_schema.py b/migrate_vault_schema.py index 3a6d4ea..9651036 100644 --- a/migrate_vault_schema.py +++ b/migrate_vault_schema.py @@ -26,18 +26,20 @@ "Authorization": f"Bearer {TARGET_ACCOUNT_AUTH}", "Content-Type": "application/json", } - def get_vault_details(vaultID: str): + """Return the vault metadata and schema""" response = requests.get(f"{SOURCE_ENV_URL}/v1/vaults/{vaultID}", headers=SOURCE_ACCOUNT_HEADERS) response.raise_for_status() return response.json() def create_vault(create_vault_request_payload): + """Creates a vault""" response = requests.post(f"{TARGET_ENV_URL}/v1/vaults", json=create_vault_request_payload, headers=TARGET_ACCOUNT_HEADERS) response.raise_for_status() return response.json() def transform_payload(vault_details): + """Transforms source vault payload to target payload.""" create_vault_payload = { "name": VAULT_NAME if VAULT_NAME else f"UntitledVault{random.randint(0,1000)}" if VAULT_SCHEMA_CONFIG else vault_details["name"], "description": VAULT_DESCRIPTION if VAULT_DESCRIPTION else "" if VAULT_SCHEMA_CONFIG else vault_details["description"], @@ -50,6 +52,7 @@ def transform_payload(vault_details): return create_vault_payload def main(): + """Migrates vault schema""" try: print("-- Initializing Vault migration --") if VAULT_SCHEMA_CONFIG is not None and VAULT_SCHEMA_CONFIG == "config_file": From 95260d995c201eea85d4cbe1b569c26d9a78aff3 Mon Sep 17 00:00:00 2001 From: Srivyshnavi Koduri <121154323+skyflow-srivyshnavi@users.noreply.github.com> Date: Wed, 19 Nov 2025 13:34:48 +0530 Subject: [PATCH 2/2] update README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f6312c5..07b0056 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ Migrates a pipeline from the source vault to the target vault. ##### Parameters: - **`source_and_target_env`**: Source and Target Env's. - **`pipeline_id`**: Pipeline ID to migrate. -- - **`target_vault_id`**: Target Vault ID. +- **`target_vault_id`**: Target Vault ID. - **`source_datastore_config`**: JSON object that replaces the source datastore configuration. Provide either an `ftpServer` or `s3Bucket` object with the required credentials. - **`target_datastore_config`**: JSON object that replaces the destination datastore configuration. Provide either an `ftpServer` or `s3Bucket` object with the required credentials. - **`source_account_access_token`**: Access token of the source account.