From 24cfa8c6c66a98964523f962ca309b2f0ca4426a Mon Sep 17 00:00:00 2001 From: LE SAULNIER Kevin Date: Wed, 12 Jul 2023 09:50:57 +0200 Subject: [PATCH 1/4] feat: clean orphan elements : networks, fiters and contingency-lists for now Signed-off-by: LE SAULNIER Kevin --- .gitignore | 1 + README.md | 19 ++++++ .../clean_orphans_elements.py | 29 +++++++++ clean_orphan_elements/constant.py | 32 ++++++++++ .../clean_orphan_contingency_lists.py | 59 ++++++++++++++++++ .../functions/clean_orphan_filters.py | 59 ++++++++++++++++++ .../functions/clean_orphan_networks.py | 61 +++++++++++++++++++ clean_orphan_elements/script_mode.py | 5 ++ 8 files changed, 265 insertions(+) create mode 100644 .gitignore create mode 100644 clean_orphan_elements/clean_orphans_elements.py create mode 100644 clean_orphan_elements/constant.py create mode 100644 clean_orphan_elements/functions/clean_orphan_contingency_lists.py create mode 100644 clean_orphan_elements/functions/clean_orphan_filters.py create mode 100644 clean_orphan_elements/functions/clean_orphan_networks.py create mode 100644 clean_orphan_elements/script_mode.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ba0430d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__/ \ No newline at end of file diff --git a/README.md b/README.md index 9f0271f..3aed75b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,21 @@ # admin-tools Gridsuite admin tools + +# Clean orphan elements +This python script will send requests to gridsuite services configured in "constant.py" in order to clean orphan elements + +Developed with Python version 3.8.10 + +## Script modes + +Two executions modes are available : +- **exec** : this mode will actually remove orphan elements by executing DELETE requests to services +- **test** (default) : this mode will not modify nor remove any element. It will only display which element will be delete if script is ran with "exec" mode + +## Execution + +Command line to run script with "exec" mode : +
+    python clean_oprhans_element.py --mode=exec
+
+ diff --git a/clean_orphan_elements/clean_orphans_elements.py b/clean_orphan_elements/clean_orphans_elements.py new file mode 100644 index 0000000..0cc6141 --- /dev/null +++ b/clean_orphan_elements/clean_orphans_elements.py @@ -0,0 +1,29 @@ +import argparse + +from functions.clean_orphan_contingency_lists import delete_orphan_contingency_lists +from functions.clean_orphan_filters import delete_orphan_filters +from functions.clean_orphan_networks import delete_orphan_network + +from script_mode import ScriptMode + +parser=argparse.ArgumentParser(description='Send requests to the gridsuite services to remove orphan elements',) + +parser.add_argument(("--mode"), help="test mode (default) will not execute any modification request", choices=[ScriptMode.TEST.value, ScriptMode.EXEC.value], nargs='?', type=str, const=ScriptMode.TEST, default=ScriptMode.TEST) + +args=parser.parse_args() +script_mode = args.mode + +if(script_mode == ScriptMode.TEST): + print("Orphans deletion script will run without deleting anything (test mode)") +else: + print("Orphans deletion script (exec mode)") + +print("\n\n") + +delete_orphan_network(script_mode) + +delete_orphan_contingency_lists(script_mode) + +delete_orphan_filters(script_mode) + + diff --git a/clean_orphan_elements/constant.py b/clean_orphan_elements/constant.py new file mode 100644 index 0000000..1487946 --- /dev/null +++ b/clean_orphan_elements/constant.py @@ -0,0 +1,32 @@ +# ENV FOLDERS +DEV_FOLDER="sql/dev/" +PROD_FOLDER="sql/prod/" + +# SQL FILES +GET_EXISTING_NETWORKS_SQL="get_existing_networks.sql" +GET_ORPHAN_NETWORKS_SQL="get_orphan_networks.sql" +DELETE_ORPHAN_NETWORKS_SQL="delete_orphan_networks.sql" + +# SERVICES URL +HTTP_PROTOCOL="http://" +API_VERSION="/v1" + +STUDY_SERVER_URL=HTTP_PROTOCOL + "localhost:5001" + API_VERSION +NETWORK_STORE_SERVER_URL=HTTP_PROTOCOL + "localhost:8080" + API_VERSION +DIRECTORY_SERVER_URL=HTTP_PROTOCOL + "localhost:5026" + API_VERSION +ACTIONS_SERVER_URL=HTTP_PROTOCOL + "localhost:5022" + API_VERSION +FILTER_SERVER_URL=HTTP_PROTOCOL + "localhost:5027" + API_VERSION + +# PATHS +GET_STUDIES=STUDY_SERVER_URL + "/supervision/studies" + +GET_NETWORKS=NETWORK_STORE_SERVER_URL + "/networks" +DELETE_NETWORKS=NETWORK_STORE_SERVER_URL + "/networks" + +GET_DIRECTORY_ELEMENTS=DIRECTORY_SERVER_URL + "/supervision/elements" + +GET_CONTINGENCY_LISTS=ACTIONS_SERVER_URL + "/contingency-lists" +DELETE_CONTINGENCY_LISTS=ACTIONS_SERVER_URL + "/contingency-lists" + +GET_FILTERS=FILTER_SERVER_URL + "/filters" +DELETE_FILTERS=FILTER_SERVER_URL + "/filters" \ No newline at end of file diff --git a/clean_orphan_elements/functions/clean_orphan_contingency_lists.py b/clean_orphan_elements/functions/clean_orphan_contingency_lists.py new file mode 100644 index 0000000..b080478 --- /dev/null +++ b/clean_orphan_elements/functions/clean_orphan_contingency_lists.py @@ -0,0 +1,59 @@ +import requests +import constant +from script_mode import ScriptMode + +def get_directory_element_uuid(element): + return element["elementUuid"] + +def get_actions_element_uuid(element): + return element["id"] + + +def delete_contingency_lists(contingency_list_uuids, script_mode): + if(script_mode == ScriptMode.TEST): + for orphan_cl in contingency_list_uuids: + print("DELETE " + constant.DELETE_CONTINGENCY_LISTS + "/" + orphan_cl) + else: + for orphan_cl in contingency_list_uuids: + requests.delete(constant.DELETE_CONTINGENCY_LISTS + "/" + orphan_cl) + +def delete_orphan_contingency_lists(script_mode): + # DELETING ORPHAN ACTIONS IN ACTIONS SERVER + print("/// Orphan actions deletion ///") + ### GET EXISTING ACTIONS FROM DIRECTORY SERVER + print("Getting existing contingency lists from directory-server") + get_directory_contingency_lists_response = requests.get(constant.GET_DIRECTORY_ELEMENTS, params={"elementType": "CONTINGENCY_LIST"}) + get_directory_contingency_lists_response_json = get_directory_contingency_lists_response.json() + get_directory_contingency_lists_response_json_uuid = map(get_directory_element_uuid, get_directory_contingency_lists_response_json) + existing_contingency_lists_uuid = list(get_directory_contingency_lists_response_json_uuid) + + print("Done") + + ### GET CONTINGENCY LISTS FROM ACTIONS SERVER + print("Getting all contingency lists from actions-server") + get_actions_contingency_lists_response = requests.get(constant.GET_CONTINGENCY_LISTS) + get_actions_contingency_lists_json = get_actions_contingency_lists_response.json() + get_actions_contingency_lists_uuid = map(get_actions_element_uuid, get_actions_contingency_lists_json) + all_contingency_lists_uuid = list(get_actions_contingency_lists_uuid) + + print("Done") + + ### GET ORPHANS CONTINGENCY LISTS - CONTINGENCY LISTS IN ACTIONS SERVER WHICH ARE NOT KNOWN IN DIRECTORY SERVER + print("Computing orphan contingency lists") + orphan_contingency_lists = [] + for element_uuid in all_contingency_lists_uuid: + if(element_uuid not in existing_contingency_lists_uuid): + orphan_contingency_lists.append(element_uuid) + + print("Done") + + ### DELETING OPRHANS + print("Deleting the following orphan contingency lists : ") + for orphan_cl in orphan_contingency_lists: + print(" - ", orphan_cl) + + delete_contingency_lists(orphan_contingency_lists, script_mode) + + print("Done") + + print("\n\n") \ No newline at end of file diff --git a/clean_orphan_elements/functions/clean_orphan_filters.py b/clean_orphan_elements/functions/clean_orphan_filters.py new file mode 100644 index 0000000..86fb169 --- /dev/null +++ b/clean_orphan_elements/functions/clean_orphan_filters.py @@ -0,0 +1,59 @@ +import constant +import requests + +from script_mode import ScriptMode + +def get_directory_element_uuid(element): + return element["elementUuid"] + +def delete_filters(filter_uuids, script_mode): + if(script_mode == ScriptMode.TEST): + for orphan_f in filter_uuids: + print("DELETE " + constant.DELETE_FILTERS + "/" + orphan_f) + else: + for orphan_f in filter_uuids: + requests.delete(constant.DELETE_FILTERS + "/" + orphan_f) + +def get_element_id(element): + return element["id"] + +def delete_orphan_filters(script_mode): + # DELETING ORPHAN FILTERS IN FILTER SERVER + print("/// Orphan filters deletion ///") + ### GET EXISTING FILTERS FROM DIRECTORY SERVER + print("Getting existing filters from directory-server") + get_directory_filters_response = requests.get(constant.GET_DIRECTORY_ELEMENTS, params={"elementType": "FILTER"}) + get_directory_filters_response_json = get_directory_filters_response.json() + get_directory_filters_response_json_uuid = map(get_directory_element_uuid, get_directory_filters_response_json) + existing_filters_uuid = list(get_directory_filters_response_json_uuid) + + print("Done") + + ### GET CONTINGENCY LISTS FROM ACTIONS SERVER + print("Getting all filters from filter-server") + get_actions_filters_response = requests.get(constant.GET_FILTERS) + get_actions_filters_json = get_actions_filters_response.json() + get_actions_filters_uuid = map(get_element_id, get_actions_filters_json) + all_filters_uuid = list(get_actions_filters_uuid) + + print("Done") + + ### GET ORPHANS CONTINGENCY LISTS - CONTINGENCY LISTS IN ACTIONS SERVER WHICH ARE NOT KNOWN IN DIRECTORY SERVER + print("Computing orphan filters") + orphan_filters = [] + for element_uuid in all_filters_uuid: + if(element_uuid not in existing_filters_uuid): + orphan_filters.append(element_uuid) + + print("Done") + + ### DELETING OPRHANS + print("Deleting the following orphan filters : ") + for orphan_cl in orphan_filters: + print(" - ", orphan_cl) + + delete_filters(orphan_filters, script_mode) + + print("Done") + + print("\n\n") \ No newline at end of file diff --git a/clean_orphan_elements/functions/clean_orphan_networks.py b/clean_orphan_elements/functions/clean_orphan_networks.py new file mode 100644 index 0000000..6b5b0bd --- /dev/null +++ b/clean_orphan_elements/functions/clean_orphan_networks.py @@ -0,0 +1,61 @@ +import requests +import constant + +from script_mode import ScriptMode + +def get_network_uuid_from_study(study): + return study["networkUuid"] + + +def get_network_uuid_from_network(network): + return network["uuid"] + + + +def delete_networks(network_uuids, script_mode): + if(script_mode == ScriptMode.TEST): + for orphan_n in network_uuids: + print("DELETE " + constant.DELETE_NETWORKS + "/" + orphan_n) + else: + for orphan_n in network_uuids: + requests.delete(constant.DELETE_NETWORKS + "/" + orphan_n) + +def delete_orphan_network(script_mode): + # DELETING ORPHAN NETWORKS IN NETWORK STORE SERVER + print("/// Orphan networks deletion ///") + ### GET EXISTING NETWORKS AMONG EXISTING STUDIES + print("Getting existing networks from existing studies") + get_studies_response = requests.get(constant.GET_STUDIES) + + get_studies_response_json = get_studies_response.json() + get_studies_response_json_network_uuid = map(get_network_uuid_from_study, get_studies_response_json) + existing_networks_uuid = list(get_studies_response_json_network_uuid) + + print("Done") + ### GET NETWORKS SAVED IN NETWORK STORE SERVER + print("Getting networks from network store server") + get_networks_response = requests.get(constant.GET_NETWORKS) + + get_networks_response_json = get_networks_response.json() + get_networks_response_json_uuid = map(get_network_uuid_from_network, get_networks_response_json) + all_networks_uuid = list(get_networks_response_json_uuid) + + print("Done") + ### GET ORPHANS NETWORKS - NETWORKS IN NETWORK STORE SERVER WHICH ARE NOT KNOWN IN STUDY SERVER + print("Computing orphan networks") + orphan_networks = [] + for network_uuid in all_networks_uuid: + if(network_uuid not in existing_networks_uuid): + orphan_networks.append(network_uuid) + + print("Done") + ### DELETING OPRHANS + print("Deleting the following orphan networks : ") + for orphan_n in orphan_networks: + print(" - ", orphan_n) + + delete_networks(orphan_networks, script_mode) + + print("Done") + + print("\n\n") \ No newline at end of file diff --git a/clean_orphan_elements/script_mode.py b/clean_orphan_elements/script_mode.py new file mode 100644 index 0000000..79c327d --- /dev/null +++ b/clean_orphan_elements/script_mode.py @@ -0,0 +1,5 @@ +from enum import Enum + +class ScriptMode(Enum): + TEST = "test" + EXEC = "exec" \ No newline at end of file From 7f4ad6e828b5fec47e05e8dcd62a2e49aabd922c Mon Sep 17 00:00:00 2001 From: Slimane AMAR Date: Wed, 2 Aug 2023 16:38:31 +0200 Subject: [PATCH 2/4] Code reorganization --- .gitignore | 8 +++++++- ...clean_orphans_elements.py => clean_orphans_elements.py | 7 +++---- clean_orphan_elements/constant.py => constant.py | 0 functions/__init__.py | 0 functions/clean_orphan_elements/__init__.py | 0 .../clean_orphan_contingency_lists.py | 0 .../clean_orphan_elements}/clean_orphan_filters.py | 0 .../clean_orphan_elements}/clean_orphan_networks.py | 3 ++- clean_orphan_elements/script_mode.py => script_mode.py | 0 9 files changed, 12 insertions(+), 6 deletions(-) rename clean_orphan_elements/clean_orphans_elements.py => clean_orphans_elements.py (72%) rename clean_orphan_elements/constant.py => constant.py (100%) create mode 100644 functions/__init__.py create mode 100644 functions/clean_orphan_elements/__init__.py rename {clean_orphan_elements/functions => functions/clean_orphan_elements}/clean_orphan_contingency_lists.py (100%) rename {clean_orphan_elements/functions => functions/clean_orphan_elements}/clean_orphan_filters.py (100%) rename {clean_orphan_elements/functions => functions/clean_orphan_elements}/clean_orphan_networks.py (99%) rename clean_orphan_elements/script_mode.py => script_mode.py (100%) diff --git a/.gitignore b/.gitignore index ba0430d..3802f2a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,7 @@ -__pycache__/ \ No newline at end of file +# Python +__pycache__/ +/venv + +# IntelliJ +/.idea +*.iml \ No newline at end of file diff --git a/clean_orphan_elements/clean_orphans_elements.py b/clean_orphans_elements.py similarity index 72% rename from clean_orphan_elements/clean_orphans_elements.py rename to clean_orphans_elements.py index 0cc6141..4593c5d 100644 --- a/clean_orphan_elements/clean_orphans_elements.py +++ b/clean_orphans_elements.py @@ -1,9 +1,8 @@ import argparse -from functions.clean_orphan_contingency_lists import delete_orphan_contingency_lists -from functions.clean_orphan_filters import delete_orphan_filters -from functions.clean_orphan_networks import delete_orphan_network - +from functions.clean_orphan_elements.clean_orphan_contingency_lists import delete_orphan_contingency_lists +from functions.clean_orphan_elements.clean_orphan_filters import delete_orphan_filters +from functions.clean_orphan_elements.clean_orphan_networks import delete_orphan_network from script_mode import ScriptMode parser=argparse.ArgumentParser(description='Send requests to the gridsuite services to remove orphan elements',) diff --git a/clean_orphan_elements/constant.py b/constant.py similarity index 100% rename from clean_orphan_elements/constant.py rename to constant.py diff --git a/functions/__init__.py b/functions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/functions/clean_orphan_elements/__init__.py b/functions/clean_orphan_elements/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clean_orphan_elements/functions/clean_orphan_contingency_lists.py b/functions/clean_orphan_elements/clean_orphan_contingency_lists.py similarity index 100% rename from clean_orphan_elements/functions/clean_orphan_contingency_lists.py rename to functions/clean_orphan_elements/clean_orphan_contingency_lists.py diff --git a/clean_orphan_elements/functions/clean_orphan_filters.py b/functions/clean_orphan_elements/clean_orphan_filters.py similarity index 100% rename from clean_orphan_elements/functions/clean_orphan_filters.py rename to functions/clean_orphan_elements/clean_orphan_filters.py diff --git a/clean_orphan_elements/functions/clean_orphan_networks.py b/functions/clean_orphan_elements/clean_orphan_networks.py similarity index 99% rename from clean_orphan_elements/functions/clean_orphan_networks.py rename to functions/clean_orphan_elements/clean_orphan_networks.py index 6b5b0bd..b919d9f 100644 --- a/clean_orphan_elements/functions/clean_orphan_networks.py +++ b/functions/clean_orphan_elements/clean_orphan_networks.py @@ -1,8 +1,9 @@ import requests -import constant +import constant from script_mode import ScriptMode + def get_network_uuid_from_study(study): return study["networkUuid"] diff --git a/clean_orphan_elements/script_mode.py b/script_mode.py similarity index 100% rename from clean_orphan_elements/script_mode.py rename to script_mode.py From 749b4cfe057d79e7cafa9692492910c95d567bc5 Mon Sep 17 00:00:00 2001 From: Slimane AMAR Date: Wed, 2 Aug 2023 16:45:04 +0200 Subject: [PATCH 3/4] Apply PEP 8 rules --- .gitignore | 2 +- clean_orphans_elements.py | 12 +++--- constant.py | 40 +++++++++---------- .../clean_orphan_contingency_lists.py | 23 ++++++----- .../clean_orphan_filters.py | 18 +++++---- .../clean_orphan_networks.py | 16 ++++---- script_mode.py | 3 +- 7 files changed, 62 insertions(+), 52 deletions(-) diff --git a/.gitignore b/.gitignore index 3802f2a..4489ffb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ __pycache__/ # IntelliJ /.idea -*.iml \ No newline at end of file +*.iml diff --git a/clean_orphans_elements.py b/clean_orphans_elements.py index 4593c5d..b3af79c 100644 --- a/clean_orphans_elements.py +++ b/clean_orphans_elements.py @@ -5,14 +5,16 @@ from functions.clean_orphan_elements.clean_orphan_networks import delete_orphan_network from script_mode import ScriptMode -parser=argparse.ArgumentParser(description='Send requests to the gridsuite services to remove orphan elements',) +parser = argparse.ArgumentParser(description='Send requests to the gridsuite services to remove orphan elements', ) -parser.add_argument(("--mode"), help="test mode (default) will not execute any modification request", choices=[ScriptMode.TEST.value, ScriptMode.EXEC.value], nargs='?', type=str, const=ScriptMode.TEST, default=ScriptMode.TEST) +parser.add_argument("--mode", help="test mode (default) will not execute any modification request", + choices=[ScriptMode.TEST.value, ScriptMode.EXEC.value], nargs='?', type=str, const=ScriptMode.TEST, + default=ScriptMode.TEST) -args=parser.parse_args() +args = parser.parse_args() script_mode = args.mode -if(script_mode == ScriptMode.TEST): +if script_mode == ScriptMode.TEST: print("Orphans deletion script will run without deleting anything (test mode)") else: print("Orphans deletion script (exec mode)") @@ -24,5 +26,3 @@ delete_orphan_contingency_lists(script_mode) delete_orphan_filters(script_mode) - - diff --git a/constant.py b/constant.py index 1487946..847ea67 100644 --- a/constant.py +++ b/constant.py @@ -1,32 +1,32 @@ # ENV FOLDERS -DEV_FOLDER="sql/dev/" -PROD_FOLDER="sql/prod/" +DEV_FOLDER = "sql/dev/" +PROD_FOLDER = "sql/prod/" # SQL FILES -GET_EXISTING_NETWORKS_SQL="get_existing_networks.sql" -GET_ORPHAN_NETWORKS_SQL="get_orphan_networks.sql" -DELETE_ORPHAN_NETWORKS_SQL="delete_orphan_networks.sql" +GET_EXISTING_NETWORKS_SQL = "get_existing_networks.sql" +GET_ORPHAN_NETWORKS_SQL = "get_orphan_networks.sql" +DELETE_ORPHAN_NETWORKS_SQL = "delete_orphan_networks.sql" # SERVICES URL -HTTP_PROTOCOL="http://" -API_VERSION="/v1" +HTTP_PROTOCOL = "http://" +API_VERSION = "/v1" -STUDY_SERVER_URL=HTTP_PROTOCOL + "localhost:5001" + API_VERSION -NETWORK_STORE_SERVER_URL=HTTP_PROTOCOL + "localhost:8080" + API_VERSION -DIRECTORY_SERVER_URL=HTTP_PROTOCOL + "localhost:5026" + API_VERSION -ACTIONS_SERVER_URL=HTTP_PROTOCOL + "localhost:5022" + API_VERSION -FILTER_SERVER_URL=HTTP_PROTOCOL + "localhost:5027" + API_VERSION +STUDY_SERVER_URL = HTTP_PROTOCOL + "localhost:5001" + API_VERSION +NETWORK_STORE_SERVER_URL = HTTP_PROTOCOL + "localhost:8080" + API_VERSION +DIRECTORY_SERVER_URL = HTTP_PROTOCOL + "localhost:5026" + API_VERSION +ACTIONS_SERVER_URL = HTTP_PROTOCOL + "localhost:5022" + API_VERSION +FILTER_SERVER_URL = HTTP_PROTOCOL + "localhost:5027" + API_VERSION # PATHS -GET_STUDIES=STUDY_SERVER_URL + "/supervision/studies" +GET_STUDIES = STUDY_SERVER_URL + "/supervision/studies" -GET_NETWORKS=NETWORK_STORE_SERVER_URL + "/networks" -DELETE_NETWORKS=NETWORK_STORE_SERVER_URL + "/networks" +GET_NETWORKS = NETWORK_STORE_SERVER_URL + "/networks" +DELETE_NETWORKS = NETWORK_STORE_SERVER_URL + "/networks" -GET_DIRECTORY_ELEMENTS=DIRECTORY_SERVER_URL + "/supervision/elements" +GET_DIRECTORY_ELEMENTS = DIRECTORY_SERVER_URL + "/supervision/elements" -GET_CONTINGENCY_LISTS=ACTIONS_SERVER_URL + "/contingency-lists" -DELETE_CONTINGENCY_LISTS=ACTIONS_SERVER_URL + "/contingency-lists" +GET_CONTINGENCY_LISTS = ACTIONS_SERVER_URL + "/contingency-lists" +DELETE_CONTINGENCY_LISTS = ACTIONS_SERVER_URL + "/contingency-lists" -GET_FILTERS=FILTER_SERVER_URL + "/filters" -DELETE_FILTERS=FILTER_SERVER_URL + "/filters" \ No newline at end of file +GET_FILTERS = FILTER_SERVER_URL + "/filters" +DELETE_FILTERS = FILTER_SERVER_URL + "/filters" diff --git a/functions/clean_orphan_elements/clean_orphan_contingency_lists.py b/functions/clean_orphan_elements/clean_orphan_contingency_lists.py index b080478..0818825 100644 --- a/functions/clean_orphan_elements/clean_orphan_contingency_lists.py +++ b/functions/clean_orphan_elements/clean_orphan_contingency_lists.py @@ -2,34 +2,39 @@ import constant from script_mode import ScriptMode + def get_directory_element_uuid(element): return element["elementUuid"] + def get_actions_element_uuid(element): return element["id"] def delete_contingency_lists(contingency_list_uuids, script_mode): - if(script_mode == ScriptMode.TEST): + if script_mode == ScriptMode.TEST: for orphan_cl in contingency_list_uuids: print("DELETE " + constant.DELETE_CONTINGENCY_LISTS + "/" + orphan_cl) else: for orphan_cl in contingency_list_uuids: requests.delete(constant.DELETE_CONTINGENCY_LISTS + "/" + orphan_cl) + def delete_orphan_contingency_lists(script_mode): # DELETING ORPHAN ACTIONS IN ACTIONS SERVER print("/// Orphan actions deletion ///") - ### GET EXISTING ACTIONS FROM DIRECTORY SERVER + # GET EXISTING ACTIONS FROM DIRECTORY SERVER print("Getting existing contingency lists from directory-server") - get_directory_contingency_lists_response = requests.get(constant.GET_DIRECTORY_ELEMENTS, params={"elementType": "CONTINGENCY_LIST"}) + get_directory_contingency_lists_response = requests.get(constant.GET_DIRECTORY_ELEMENTS, + params={"elementType": "CONTINGENCY_LIST"}) get_directory_contingency_lists_response_json = get_directory_contingency_lists_response.json() - get_directory_contingency_lists_response_json_uuid = map(get_directory_element_uuid, get_directory_contingency_lists_response_json) + get_directory_contingency_lists_response_json_uuid = map(get_directory_element_uuid, + get_directory_contingency_lists_response_json) existing_contingency_lists_uuid = list(get_directory_contingency_lists_response_json_uuid) print("Done") - ### GET CONTINGENCY LISTS FROM ACTIONS SERVER + # GET CONTINGENCY LISTS FROM ACTIONS SERVER print("Getting all contingency lists from actions-server") get_actions_contingency_lists_response = requests.get(constant.GET_CONTINGENCY_LISTS) get_actions_contingency_lists_json = get_actions_contingency_lists_response.json() @@ -38,16 +43,16 @@ def delete_orphan_contingency_lists(script_mode): print("Done") - ### GET ORPHANS CONTINGENCY LISTS - CONTINGENCY LISTS IN ACTIONS SERVER WHICH ARE NOT KNOWN IN DIRECTORY SERVER + # GET ORPHANS CONTINGENCY LISTS - CONTINGENCY LISTS IN ACTIONS SERVER WHICH ARE NOT KNOWN IN DIRECTORY SERVER print("Computing orphan contingency lists") orphan_contingency_lists = [] for element_uuid in all_contingency_lists_uuid: - if(element_uuid not in existing_contingency_lists_uuid): + if element_uuid not in existing_contingency_lists_uuid: orphan_contingency_lists.append(element_uuid) print("Done") - ### DELETING OPRHANS + # DELETING OPRHANS print("Deleting the following orphan contingency lists : ") for orphan_cl in orphan_contingency_lists: print(" - ", orphan_cl) @@ -56,4 +61,4 @@ def delete_orphan_contingency_lists(script_mode): print("Done") - print("\n\n") \ No newline at end of file + print("\n\n") diff --git a/functions/clean_orphan_elements/clean_orphan_filters.py b/functions/clean_orphan_elements/clean_orphan_filters.py index 86fb169..e72d7df 100644 --- a/functions/clean_orphan_elements/clean_orphan_filters.py +++ b/functions/clean_orphan_elements/clean_orphan_filters.py @@ -3,24 +3,28 @@ from script_mode import ScriptMode + def get_directory_element_uuid(element): return element["elementUuid"] + def delete_filters(filter_uuids, script_mode): - if(script_mode == ScriptMode.TEST): + if script_mode == ScriptMode.TEST: for orphan_f in filter_uuids: print("DELETE " + constant.DELETE_FILTERS + "/" + orphan_f) else: for orphan_f in filter_uuids: requests.delete(constant.DELETE_FILTERS + "/" + orphan_f) + def get_element_id(element): return element["id"] + def delete_orphan_filters(script_mode): # DELETING ORPHAN FILTERS IN FILTER SERVER print("/// Orphan filters deletion ///") - ### GET EXISTING FILTERS FROM DIRECTORY SERVER + # GET EXISTING FILTERS FROM DIRECTORY SERVER print("Getting existing filters from directory-server") get_directory_filters_response = requests.get(constant.GET_DIRECTORY_ELEMENTS, params={"elementType": "FILTER"}) get_directory_filters_response_json = get_directory_filters_response.json() @@ -29,7 +33,7 @@ def delete_orphan_filters(script_mode): print("Done") - ### GET CONTINGENCY LISTS FROM ACTIONS SERVER + # GET CONTINGENCY LISTS FROM ACTIONS SERVER print("Getting all filters from filter-server") get_actions_filters_response = requests.get(constant.GET_FILTERS) get_actions_filters_json = get_actions_filters_response.json() @@ -38,16 +42,16 @@ def delete_orphan_filters(script_mode): print("Done") - ### GET ORPHANS CONTINGENCY LISTS - CONTINGENCY LISTS IN ACTIONS SERVER WHICH ARE NOT KNOWN IN DIRECTORY SERVER + # GET ORPHANS CONTINGENCY LISTS - CONTINGENCY LISTS IN ACTIONS SERVER WHICH ARE NOT KNOWN IN DIRECTORY SERVER print("Computing orphan filters") orphan_filters = [] for element_uuid in all_filters_uuid: - if(element_uuid not in existing_filters_uuid): + if element_uuid not in existing_filters_uuid: orphan_filters.append(element_uuid) print("Done") - ### DELETING OPRHANS + # DELETING OPRHANS print("Deleting the following orphan filters : ") for orphan_cl in orphan_filters: print(" - ", orphan_cl) @@ -56,4 +60,4 @@ def delete_orphan_filters(script_mode): print("Done") - print("\n\n") \ No newline at end of file + print("\n\n") diff --git a/functions/clean_orphan_elements/clean_orphan_networks.py b/functions/clean_orphan_elements/clean_orphan_networks.py index b919d9f..ac7ccb1 100644 --- a/functions/clean_orphan_elements/clean_orphan_networks.py +++ b/functions/clean_orphan_elements/clean_orphan_networks.py @@ -12,19 +12,19 @@ def get_network_uuid_from_network(network): return network["uuid"] - def delete_networks(network_uuids, script_mode): - if(script_mode == ScriptMode.TEST): + if script_mode == ScriptMode.TEST: for orphan_n in network_uuids: print("DELETE " + constant.DELETE_NETWORKS + "/" + orphan_n) else: for orphan_n in network_uuids: requests.delete(constant.DELETE_NETWORKS + "/" + orphan_n) + def delete_orphan_network(script_mode): # DELETING ORPHAN NETWORKS IN NETWORK STORE SERVER print("/// Orphan networks deletion ///") - ### GET EXISTING NETWORKS AMONG EXISTING STUDIES + # GET EXISTING NETWORKS AMONG EXISTING STUDIES print("Getting existing networks from existing studies") get_studies_response = requests.get(constant.GET_STUDIES) @@ -33,7 +33,7 @@ def delete_orphan_network(script_mode): existing_networks_uuid = list(get_studies_response_json_network_uuid) print("Done") - ### GET NETWORKS SAVED IN NETWORK STORE SERVER + # GET NETWORKS SAVED IN NETWORK STORE SERVER print("Getting networks from network store server") get_networks_response = requests.get(constant.GET_NETWORKS) @@ -42,15 +42,15 @@ def delete_orphan_network(script_mode): all_networks_uuid = list(get_networks_response_json_uuid) print("Done") - ### GET ORPHANS NETWORKS - NETWORKS IN NETWORK STORE SERVER WHICH ARE NOT KNOWN IN STUDY SERVER + # GET ORPHANS NETWORKS - NETWORKS IN NETWORK STORE SERVER WHICH ARE NOT KNOWN IN STUDY SERVER print("Computing orphan networks") orphan_networks = [] for network_uuid in all_networks_uuid: - if(network_uuid not in existing_networks_uuid): + if network_uuid not in existing_networks_uuid: orphan_networks.append(network_uuid) print("Done") - ### DELETING OPRHANS + # DELETING OPRHANS print("Deleting the following orphan networks : ") for orphan_n in orphan_networks: print(" - ", orphan_n) @@ -59,4 +59,4 @@ def delete_orphan_network(script_mode): print("Done") - print("\n\n") \ No newline at end of file + print("\n\n") diff --git a/script_mode.py b/script_mode.py index 79c327d..12e5a65 100644 --- a/script_mode.py +++ b/script_mode.py @@ -1,5 +1,6 @@ from enum import Enum + class ScriptMode(Enum): TEST = "test" - EXEC = "exec" \ No newline at end of file + EXEC = "exec" From fbbf36172d4aa4294cfe9cd10d7918d45ea5d251 Mon Sep 17 00:00:00 2001 From: Slimane AMAR Date: Wed, 2 Aug 2023 17:34:23 +0200 Subject: [PATCH 4/4] Add dry-run mode --- clean_orphans_elements.py | 16 +++++++--------- .../clean_orphan_contingency_lists.py | 9 ++++----- .../clean_orphan_filters.py | 10 ++++------ .../clean_orphan_networks.py | 10 ++++------ script_mode.py | 6 ------ 5 files changed, 19 insertions(+), 32 deletions(-) delete mode 100644 script_mode.py diff --git a/clean_orphans_elements.py b/clean_orphans_elements.py index b3af79c..c3e070b 100644 --- a/clean_orphans_elements.py +++ b/clean_orphans_elements.py @@ -3,26 +3,24 @@ from functions.clean_orphan_elements.clean_orphan_contingency_lists import delete_orphan_contingency_lists from functions.clean_orphan_elements.clean_orphan_filters import delete_orphan_filters from functions.clean_orphan_elements.clean_orphan_networks import delete_orphan_network -from script_mode import ScriptMode parser = argparse.ArgumentParser(description='Send requests to the gridsuite services to remove orphan elements', ) -parser.add_argument("--mode", help="test mode (default) will not execute any modification request", - choices=[ScriptMode.TEST.value, ScriptMode.EXEC.value], nargs='?', type=str, const=ScriptMode.TEST, - default=ScriptMode.TEST) +parser.add_argument("--dry-run", help="test mode (default) will not execute any modification request", + action="store_true") args = parser.parse_args() -script_mode = args.mode +dry_run = args.dry_run -if script_mode == ScriptMode.TEST: +if dry_run: print("Orphans deletion script will run without deleting anything (test mode)") else: print("Orphans deletion script (exec mode)") print("\n\n") -delete_orphan_network(script_mode) +delete_orphan_network(dry_run) -delete_orphan_contingency_lists(script_mode) +delete_orphan_contingency_lists(dry_run) -delete_orphan_filters(script_mode) +delete_orphan_filters(dry_run) diff --git a/functions/clean_orphan_elements/clean_orphan_contingency_lists.py b/functions/clean_orphan_elements/clean_orphan_contingency_lists.py index 0818825..53ef948 100644 --- a/functions/clean_orphan_elements/clean_orphan_contingency_lists.py +++ b/functions/clean_orphan_elements/clean_orphan_contingency_lists.py @@ -1,6 +1,5 @@ import requests import constant -from script_mode import ScriptMode def get_directory_element_uuid(element): @@ -11,8 +10,8 @@ def get_actions_element_uuid(element): return element["id"] -def delete_contingency_lists(contingency_list_uuids, script_mode): - if script_mode == ScriptMode.TEST: +def delete_contingency_lists(contingency_list_uuids, dry_run): + if dry_run: for orphan_cl in contingency_list_uuids: print("DELETE " + constant.DELETE_CONTINGENCY_LISTS + "/" + orphan_cl) else: @@ -20,7 +19,7 @@ def delete_contingency_lists(contingency_list_uuids, script_mode): requests.delete(constant.DELETE_CONTINGENCY_LISTS + "/" + orphan_cl) -def delete_orphan_contingency_lists(script_mode): +def delete_orphan_contingency_lists(dry_run): # DELETING ORPHAN ACTIONS IN ACTIONS SERVER print("/// Orphan actions deletion ///") # GET EXISTING ACTIONS FROM DIRECTORY SERVER @@ -57,7 +56,7 @@ def delete_orphan_contingency_lists(script_mode): for orphan_cl in orphan_contingency_lists: print(" - ", orphan_cl) - delete_contingency_lists(orphan_contingency_lists, script_mode) + delete_contingency_lists(orphan_contingency_lists, dry_run) print("Done") diff --git a/functions/clean_orphan_elements/clean_orphan_filters.py b/functions/clean_orphan_elements/clean_orphan_filters.py index e72d7df..1c120b4 100644 --- a/functions/clean_orphan_elements/clean_orphan_filters.py +++ b/functions/clean_orphan_elements/clean_orphan_filters.py @@ -1,15 +1,13 @@ import constant import requests -from script_mode import ScriptMode - def get_directory_element_uuid(element): return element["elementUuid"] -def delete_filters(filter_uuids, script_mode): - if script_mode == ScriptMode.TEST: +def delete_filters(filter_uuids, dry_run): + if dry_run: for orphan_f in filter_uuids: print("DELETE " + constant.DELETE_FILTERS + "/" + orphan_f) else: @@ -21,7 +19,7 @@ def get_element_id(element): return element["id"] -def delete_orphan_filters(script_mode): +def delete_orphan_filters(dry_run): # DELETING ORPHAN FILTERS IN FILTER SERVER print("/// Orphan filters deletion ///") # GET EXISTING FILTERS FROM DIRECTORY SERVER @@ -56,7 +54,7 @@ def delete_orphan_filters(script_mode): for orphan_cl in orphan_filters: print(" - ", orphan_cl) - delete_filters(orphan_filters, script_mode) + delete_filters(orphan_filters, dry_run) print("Done") diff --git a/functions/clean_orphan_elements/clean_orphan_networks.py b/functions/clean_orphan_elements/clean_orphan_networks.py index ac7ccb1..0aa8ffd 100644 --- a/functions/clean_orphan_elements/clean_orphan_networks.py +++ b/functions/clean_orphan_elements/clean_orphan_networks.py @@ -1,7 +1,5 @@ import requests - import constant -from script_mode import ScriptMode def get_network_uuid_from_study(study): @@ -12,8 +10,8 @@ def get_network_uuid_from_network(network): return network["uuid"] -def delete_networks(network_uuids, script_mode): - if script_mode == ScriptMode.TEST: +def delete_networks(network_uuids, dry_run): + if dry_run: for orphan_n in network_uuids: print("DELETE " + constant.DELETE_NETWORKS + "/" + orphan_n) else: @@ -21,7 +19,7 @@ def delete_networks(network_uuids, script_mode): requests.delete(constant.DELETE_NETWORKS + "/" + orphan_n) -def delete_orphan_network(script_mode): +def delete_orphan_network(dry_run): # DELETING ORPHAN NETWORKS IN NETWORK STORE SERVER print("/// Orphan networks deletion ///") # GET EXISTING NETWORKS AMONG EXISTING STUDIES @@ -55,7 +53,7 @@ def delete_orphan_network(script_mode): for orphan_n in orphan_networks: print(" - ", orphan_n) - delete_networks(orphan_networks, script_mode) + delete_networks(orphan_networks, dry_run) print("Done") diff --git a/script_mode.py b/script_mode.py deleted file mode 100644 index 12e5a65..0000000 --- a/script_mode.py +++ /dev/null @@ -1,6 +0,0 @@ -from enum import Enum - - -class ScriptMode(Enum): - TEST = "test" - EXEC = "exec"