From dce4a0d4090846ae3c1d3e67b75958e1e3945f69 Mon Sep 17 00:00:00 2001 From: basseche Date: Thu, 5 Jun 2025 15:18:24 +0200 Subject: [PATCH 1/6] add script to update cases with empty metadata in database Signed-off-by: basseche --- scripts/constant.py | 4 +- scripts/functions/networks/networks.py | 10 ++++ scripts/migrate_cases_empty_metadata.py | 67 +++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 scripts/migrate_cases_empty_metadata.py diff --git a/scripts/constant.py b/scripts/constant.py index 62da2d3..bdc4951 100644 --- a/scripts/constant.py +++ b/scripts/constant.py @@ -28,7 +28,7 @@ GRAFANA_PROVISIONING = "/provisioning" # TODO add command parameter to set this DEV -DEV = False +DEV = True DEV_HOSTNAME = "172.17.0.1" # NOSONAR S1313 : this local IP is not excluded from this sonar issue @@ -89,6 +89,8 @@ GET_NETWORK = NETWORK_STORE_SERVER_URL + "/networks/{networkId}" MIGRATE_V211_LIMITS = NETWORK_STORE_SERVER_URL + "/migration/v211limits/{networkId}/{variantNum}" MIGRATE_V214_TAP_CHANGER_STEPS = NETWORK_STORE_SERVER_URL + "/migration/v214tapChangeSteps/{networkId}/{variantNum}" +GET_CASES_EMPTY_METADATA = CASE_SERVER_URL + "/migration/cases/emptyMetadata" +MIGRATE_CASE_UPDATE_METADATA = CASE_SERVER_URL + "/migration/cases/{uuid}/metadata" DELETE_NETWORKS = NETWORK_STORE_SERVER_URL + "/networks" diff --git a/scripts/functions/networks/networks.py b/scripts/functions/networks/networks.py index 98587a5..892f30b 100644 --- a/scripts/functions/networks/networks.py +++ b/scripts/functions/networks/networks.py @@ -24,3 +24,13 @@ def migrate_v211_limits(network_id, variant_num): def migrate_v214_tapchangersteps(network_id, variant_num): response = requests.put(constant.MIGRATE_V214_TAP_CHANGER_STEPS.format(networkId=network_id, variantNum=variant_num)) response.raise_for_status() + +def get_cases_empty_metadata(): + response = requests.get(constant.GET_CASES_EMPTY_METADATA) + response.raise_for_status() + print(response.json()) + return response.json() + +def migrate_cases_updateMetadata(case_uuid): + response = requests.put(constant.MIGRATE_CASE_UPDATE_METADATA.format(uuid=case_uuid)) + response.raise_for_status() \ No newline at end of file diff --git a/scripts/migrate_cases_empty_metadata.py b/scripts/migrate_cases_empty_metadata.py new file mode 100644 index 0000000..8cea11b --- /dev/null +++ b/scripts/migrate_cases_empty_metadata.py @@ -0,0 +1,67 @@ +# +# Copyright (c) 2025, RTE (http://www.rte-france.com) +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +import requests +import argparse +import constant +import sys +from tqdm import tqdm + +from functions.plateform.plateform import check_server_status +from functions.plateform.plateform import get_plateform_info +from functions.networks.networks import get_cases_empty_metadata +from functions.networks.networks import migrate_cases_updateMetadata + +parser = argparse.ArgumentParser(description='Send requests to the gridsuite services to update cases metadata by filling missing information') +parser.add_argument("-n", "--dry-run", help="test mode (default) will not execute any update request", action='store_true') + +args = parser.parse_args() +dry_run = args.dry_run + +print("---------------------------------------------------------") +print("Cases metadata migration script") +if dry_run: + print("dry-run=" + str(dry_run) + " -> will run without modifying anything (test mode)") + +# Check powsybl-case-server +if not check_server_status(constant.CASE_SERVER_HOSTNAME): sys.exit() +print("\n") + +# Just getting an enlightening url opportunistically from here because it exists +plateformName = get_plateform_info()['redirect_uri'] +print("\n") + +print("---------------------------------------------------------") +print("This script will apply on plateform = " + plateformName ) +print("\n") +print("===> powsybl-case-server seems OK ! The script can proceed") +print("\n") +casesUuid = get_cases_empty_metadata() +print("table content : ") +print(str(casesUuid)) +## print("For a total of " + str(len(casesUuid)) + " casesUuid") +print("---------------------------------------------------------") + +print("cases metadata update migration (dry-run=" + str(dry_run) + ") in processing...") +failCount = 0 +successCount = 0 + +for caseUuid in tqdm(casesUuid): + if not dry_run: + try: + migrate_cases_updateMetadata(caseUuid) + successCount += 1 + except Exception as e: + failCount += 1 + # print only str(e) instead of the full traceback because we call this method from a simple for loop script + tqdm.write("case " + caseUuid + " => migration failed: "+ str(e)) + if isinstance(e, requests.exceptions.RequestException) and e.response is not None: + tqdm.write("Response body: " + repr(e.response.text)) # repr for cheap escaping + tqdm.write("") # emtpy newline between errors for legibility +print("End of case metadata update migration") +print("cases migration sucesses : " + str(successCount)) +print("cases migration failures : " + str(failCount)) From eb1771e766cf2224a1543e06adf1b2387110ab42 Mon Sep 17 00:00:00 2001 From: basseche Date: Thu, 5 Jun 2025 16:14:09 +0200 Subject: [PATCH 2/6] resolve sonar issue Signed-off-by: basseche --- scripts/functions/networks/networks.py | 2 +- scripts/migrate_cases_empty_metadata.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/functions/networks/networks.py b/scripts/functions/networks/networks.py index 892f30b..30e5a41 100644 --- a/scripts/functions/networks/networks.py +++ b/scripts/functions/networks/networks.py @@ -31,6 +31,6 @@ def get_cases_empty_metadata(): print(response.json()) return response.json() -def migrate_cases_updateMetadata(case_uuid): +def migrate_cases_update_metadata(case_uuid): response = requests.put(constant.MIGRATE_CASE_UPDATE_METADATA.format(uuid=case_uuid)) response.raise_for_status() \ No newline at end of file diff --git a/scripts/migrate_cases_empty_metadata.py b/scripts/migrate_cases_empty_metadata.py index 8cea11b..7147401 100644 --- a/scripts/migrate_cases_empty_metadata.py +++ b/scripts/migrate_cases_empty_metadata.py @@ -15,6 +15,7 @@ from functions.plateform.plateform import get_plateform_info from functions.networks.networks import get_cases_empty_metadata from functions.networks.networks import migrate_cases_updateMetadata +from scripts.functions.networks.networks import migrate_cases_update_metadata parser = argparse.ArgumentParser(description='Send requests to the gridsuite services to update cases metadata by filling missing information') parser.add_argument("-n", "--dry-run", help="test mode (default) will not execute any update request", action='store_true') @@ -53,7 +54,7 @@ for caseUuid in tqdm(casesUuid): if not dry_run: try: - migrate_cases_updateMetadata(caseUuid) + migrate_cases_update_metadata(caseUuid) successCount += 1 except Exception as e: failCount += 1 From 586d9aaecba5f3e36ef4d012a6525c790fb30469 Mon Sep 17 00:00:00 2001 From: basseche Date: Fri, 13 Jun 2025 10:57:05 +0200 Subject: [PATCH 3/6] Update scripts/constant.py Co-authored-by: etiennehomer --- scripts/constant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/constant.py b/scripts/constant.py index bdc4951..ce2cb6a 100644 --- a/scripts/constant.py +++ b/scripts/constant.py @@ -89,7 +89,7 @@ GET_NETWORK = NETWORK_STORE_SERVER_URL + "/networks/{networkId}" MIGRATE_V211_LIMITS = NETWORK_STORE_SERVER_URL + "/migration/v211limits/{networkId}/{variantNum}" MIGRATE_V214_TAP_CHANGER_STEPS = NETWORK_STORE_SERVER_URL + "/migration/v214tapChangeSteps/{networkId}/{variantNum}" -GET_CASES_EMPTY_METADATA = CASE_SERVER_URL + "/migration/cases/emptyMetadata" +GET_CASES_WITH_EMPTY_METADATA = CASE_SERVER_URL + "/migration/cases/emptyMetadata" MIGRATE_CASE_UPDATE_METADATA = CASE_SERVER_URL + "/migration/cases/{uuid}/metadata" DELETE_NETWORKS = NETWORK_STORE_SERVER_URL + "/networks" From 84a81ac96f1db8d18e98baa66bcd3bbdef92da94 Mon Sep 17 00:00:00 2001 From: basseche Date: Fri, 13 Jun 2025 10:52:21 +0200 Subject: [PATCH 4/6] review Signed-off-by: basseche --- scripts/constant.py | 2 +- scripts/functions/cases/cases.py | 10 ++++++++++ scripts/functions/networks/networks.py | 10 ---------- scripts/migrate_cases_empty_metadata.py | 15 +++++++-------- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/scripts/constant.py b/scripts/constant.py index ce2cb6a..8d6fc90 100644 --- a/scripts/constant.py +++ b/scripts/constant.py @@ -28,7 +28,7 @@ GRAFANA_PROVISIONING = "/provisioning" # TODO add command parameter to set this DEV -DEV = True +DEV = False DEV_HOSTNAME = "172.17.0.1" # NOSONAR S1313 : this local IP is not excluded from this sonar issue diff --git a/scripts/functions/cases/cases.py b/scripts/functions/cases/cases.py index acbf471..59e6fb2 100644 --- a/scripts/functions/cases/cases.py +++ b/scripts/functions/cases/cases.py @@ -33,3 +33,13 @@ def copy_to_s3_storage(case_uuid, case_name, case): def exists_case_on_s3(case_uuid): return requests.get(constant.S3_EXISTS_CASE.format(caseUuid = case_uuid)).text == 'true' + +def get_cases_with_empty_metadata(): + response = requests.get(constant.GET_CASES_EMPTY_METADATA) + response.raise_for_status() + print(response.json()) + return response.json() + +def complete_cases_metadata(case_uuid): + response = requests.put(constant.MIGRATE_CASE_UPDATE_METADATA.format(uuid=case_uuid)) + response.raise_for_status() \ No newline at end of file diff --git a/scripts/functions/networks/networks.py b/scripts/functions/networks/networks.py index 30e5a41..98587a5 100644 --- a/scripts/functions/networks/networks.py +++ b/scripts/functions/networks/networks.py @@ -24,13 +24,3 @@ def migrate_v211_limits(network_id, variant_num): def migrate_v214_tapchangersteps(network_id, variant_num): response = requests.put(constant.MIGRATE_V214_TAP_CHANGER_STEPS.format(networkId=network_id, variantNum=variant_num)) response.raise_for_status() - -def get_cases_empty_metadata(): - response = requests.get(constant.GET_CASES_EMPTY_METADATA) - response.raise_for_status() - print(response.json()) - return response.json() - -def migrate_cases_update_metadata(case_uuid): - response = requests.put(constant.MIGRATE_CASE_UPDATE_METADATA.format(uuid=case_uuid)) - response.raise_for_status() \ No newline at end of file diff --git a/scripts/migrate_cases_empty_metadata.py b/scripts/migrate_cases_empty_metadata.py index 7147401..d775ce1 100644 --- a/scripts/migrate_cases_empty_metadata.py +++ b/scripts/migrate_cases_empty_metadata.py @@ -13,9 +13,8 @@ from functions.plateform.plateform import check_server_status from functions.plateform.plateform import get_plateform_info -from functions.networks.networks import get_cases_empty_metadata -from functions.networks.networks import migrate_cases_updateMetadata -from scripts.functions.networks.networks import migrate_cases_update_metadata +from scripts.functions.cases.cases import get_cases_with_empty_metadata +from scripts.functions.cases.cases import complete_cases_metadata parser = argparse.ArgumentParser(description='Send requests to the gridsuite services to update cases metadata by filling missing information') parser.add_argument("-n", "--dry-run", help="test mode (default) will not execute any update request", action='store_true') @@ -41,20 +40,20 @@ print("\n") print("===> powsybl-case-server seems OK ! The script can proceed") print("\n") -casesUuid = get_cases_empty_metadata() +casesUuids = get_cases_with_empty_metadata() print("table content : ") -print(str(casesUuid)) -## print("For a total of " + str(len(casesUuid)) + " casesUuid") +print(str(casesUuids)) +print("For a total of " + str(len(casesUuids)) + " casesUuids ") print("---------------------------------------------------------") print("cases metadata update migration (dry-run=" + str(dry_run) + ") in processing...") failCount = 0 successCount = 0 -for caseUuid in tqdm(casesUuid): +for caseUuid in tqdm(casesUuids): if not dry_run: try: - migrate_cases_update_metadata(caseUuid) + complete_cases_metadata(caseUuid) successCount += 1 except Exception as e: failCount += 1 From cef0467e511abedaa7faca5229139efcc7edde47 Mon Sep 17 00:00:00 2001 From: basseche Date: Fri, 13 Jun 2025 11:11:07 +0200 Subject: [PATCH 5/6] review 2 Signed-off-by: basseche --- scripts/functions/cases/cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/functions/cases/cases.py b/scripts/functions/cases/cases.py index 59e6fb2..ebab6e7 100644 --- a/scripts/functions/cases/cases.py +++ b/scripts/functions/cases/cases.py @@ -35,7 +35,7 @@ def exists_case_on_s3(case_uuid): return requests.get(constant.S3_EXISTS_CASE.format(caseUuid = case_uuid)).text == 'true' def get_cases_with_empty_metadata(): - response = requests.get(constant.GET_CASES_EMPTY_METADATA) + response = requests.get(constant.GET_CASES_WITH_EMPTY_METADATA) response.raise_for_status() print(response.json()) return response.json() From 8d75d69be55376ca053e8cc6def269217fa48f55 Mon Sep 17 00:00:00 2001 From: basseche Date: Fri, 13 Jun 2025 11:24:40 +0200 Subject: [PATCH 6/6] rename script Signed-off-by: basseche --- ...migrate_cases_empty_metadata.py => complete_cases_metadata.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{migrate_cases_empty_metadata.py => complete_cases_metadata.py} (100%) diff --git a/scripts/migrate_cases_empty_metadata.py b/scripts/complete_cases_metadata.py similarity index 100% rename from scripts/migrate_cases_empty_metadata.py rename to scripts/complete_cases_metadata.py