From d8a68d9a557164e50c14b07f21e5683e90a4347a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Sch=C3=B6chlin?= Date: Thu, 12 Jun 2025 16:15:11 +0200 Subject: [PATCH 1/5] improve clouds yaml generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marc Schöchlin --- src/openstack_workload_generator/__main__.py | 21 +++++++++++++++---- .../entities/helpers.py | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/openstack_workload_generator/__main__.py b/src/openstack_workload_generator/__main__.py index 3c73163..910aede 100644 --- a/src/openstack_workload_generator/__main__.py +++ b/src/openstack_workload_generator/__main__.py @@ -172,16 +172,18 @@ def establish_connection(): for workload_domain in workload_domains.values(): for workload_project in workload_domain.get_projects(args.create_projects): + if args.generate_clouds_yaml: + clouds_yaml_data[ + f"{workload_domain.domain_name}-{workload_project.project_name}" + ] = workload_project.get_clouds_yaml_data() + if args.create_machines: workload_project.get_and_create_machines( args.create_machines, args.wait_for_machines ) if args.ansible_inventory: workload_project.dump_inventory_hosts(args.ansible_inventory) - if args.generate_clouds_yaml: - clouds_yaml_data[ - f"{workload_domain.domain_name}-{workload_project.project_name}" - ] = workload_project.get_clouds_yaml_data() + elif args.delete_machines: for machine_obj in workload_project.get_machines( args.delete_machines @@ -192,9 +194,15 @@ def establish_connection(): LOGGER.info(f"Creating a clouds yaml : {args.generate_clouds_yaml}") clouds_yaml_data_new = {"clouds": clouds_yaml_data} + initial_entries = 0 + generated_entries = 0 + total_entries = 0 + if os.path.exists(args.generate_clouds_yaml): with open(args.generate_clouds_yaml, "r") as file: existing_data = yaml.safe_load(file) + + initial_entries = len(existing_data.get("clouds", [])) backup_file = f"{args.generate_clouds_yaml}_{iso_timestamp()}" logging.warning( f"File {args.generate_clouds_yaml}, making an backup to {backup_file} and adding the new values" @@ -203,9 +211,12 @@ def establish_connection(): args.generate_clouds_yaml, f"{args.generate_clouds_yaml}_{iso_timestamp()}", ) + + generated_entries = len(clouds_yaml_data_new.get("clouds", [])) clouds_yaml_data_new = deep_merge_dict( existing_data, clouds_yaml_data_new ) + total_entries = len(clouds_yaml_data_new.get("clouds", [])) with open(args.generate_clouds_yaml, "w") as file: yaml.dump( @@ -214,6 +225,8 @@ def establish_connection(): default_flow_style=False, explicit_start=True, ) + LOGGER.info(f"Generated {generated_entries} entries, number of entries in " + f"{args.generate_clouds_yaml} is now {total_entries} (old {initial_entries} entries)") sys.exit(0) elif args.delete_projects: conn = establish_connection() diff --git a/src/openstack_workload_generator/entities/helpers.py b/src/openstack_workload_generator/entities/helpers.py index d9866ca..ee36086 100644 --- a/src/openstack_workload_generator/entities/helpers.py +++ b/src/openstack_workload_generator/entities/helpers.py @@ -246,7 +246,7 @@ def setup_logging(log_level: str) -> Tuple[logging.Logger, str]: ) logger = logging.getLogger() log_file = "STDOUT" - logging.basicConfig(format=log_format_string, level=log_level) + logging.basicConfig(format=log_format_string, level=log_level.upper()) coloredlogs.DEFAULT_FIELD_STYLES["levelname"] = {"bold": True, "color": ""} coloredlogs.install(fmt=log_format_string, level=log_level.upper()) From f44cb27ee7ff98a26684ee2f075cd368477192db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Sch=C3=B6chlin?= Date: Thu, 12 Jun 2025 16:20:47 +0200 Subject: [PATCH 2/5] Improve cloud yaml generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marc Schöchlin --- src/openstack_workload_generator/__main__.py | 96 ++++++++++---------- 1 file changed, 49 insertions(+), 47 deletions(-) diff --git a/src/openstack_workload_generator/__main__.py b/src/openstack_workload_generator/__main__.py index 910aede..ba7857c 100644 --- a/src/openstack_workload_generator/__main__.py +++ b/src/openstack_workload_generator/__main__.py @@ -22,6 +22,52 @@ ) +def establish_connection(): + if args.clouds_yaml is None: + config = loader.OpenStackConfig() + else: + LOGGER.info(f"Loading connection configuration from {args.clouds_yaml}") + config = loader.OpenStackConfig(config_files=[args.clouds_yaml]) + cloud_config = config.get_one(args.os_cloud) + return Connection(config=cloud_config) + + +def generated_clouds_yaml(): + LOGGER.info(f"Creating a clouds yaml : {args.generate_clouds_yaml}") + clouds_yaml_data_new = {"clouds": clouds_yaml_data} + initial_entries = 0 + generated_entries = 0 + total_entries = 0 + if os.path.exists(args.generate_clouds_yaml): + with open(args.generate_clouds_yaml, "r") as file: + existing_data = yaml.safe_load(file) + + initial_entries = len(existing_data.get("clouds", [])) + backup_file = f"{args.generate_clouds_yaml}_{iso_timestamp()}" + logging.warning( + f"File {args.generate_clouds_yaml}, making an backup to {backup_file} and adding the new values" + ) + shutil.copy2( + args.generate_clouds_yaml, + f"{args.generate_clouds_yaml}_{iso_timestamp()}", + ) + + generated_entries = len(clouds_yaml_data_new.get("clouds", [])) + clouds_yaml_data_new = deep_merge_dict(existing_data, clouds_yaml_data_new) + total_entries = len(clouds_yaml_data_new.get("clouds", [])) + with open(args.generate_clouds_yaml, "w") as file: + yaml.dump( + clouds_yaml_data_new, + file, + default_flow_style=False, + explicit_start=True, + ) + LOGGER.info( + f"Generated {generated_entries} entries, number of entries in " + f"{args.generate_clouds_yaml} is now {total_entries} (old {initial_entries} entries)" + ) + + LOGGER = logging.getLogger() parser = argparse.ArgumentParser(prog="Create workloads on openstack installations") @@ -141,22 +187,12 @@ setup_logging(args.log_level) - -def establish_connection(): - if args.clouds_yaml is None: - config = loader.OpenStackConfig() - else: - LOGGER.info(f"Loading connection configuration from {args.clouds_yaml}") - config = loader.OpenStackConfig(config_files=[args.clouds_yaml]) - cloud_config = config.get_one(args.os_cloud) - return Connection(config=cloud_config) - - time_start = time.time() Config.load_config(args.config) Config.show_effective_config() + if args.create_domains: conn = establish_connection() workload_domains: dict[str, WorkloadGeneratorDomain] = dict() @@ -191,42 +227,8 @@ def establish_connection(): machine_obj.delete_machine() if args.generate_clouds_yaml: - LOGGER.info(f"Creating a clouds yaml : {args.generate_clouds_yaml}") - clouds_yaml_data_new = {"clouds": clouds_yaml_data} - - initial_entries = 0 - generated_entries = 0 - total_entries = 0 - - if os.path.exists(args.generate_clouds_yaml): - with open(args.generate_clouds_yaml, "r") as file: - existing_data = yaml.safe_load(file) - - initial_entries = len(existing_data.get("clouds", [])) - backup_file = f"{args.generate_clouds_yaml}_{iso_timestamp()}" - logging.warning( - f"File {args.generate_clouds_yaml}, making an backup to {backup_file} and adding the new values" - ) - shutil.copy2( - args.generate_clouds_yaml, - f"{args.generate_clouds_yaml}_{iso_timestamp()}", - ) - - generated_entries = len(clouds_yaml_data_new.get("clouds", [])) - clouds_yaml_data_new = deep_merge_dict( - existing_data, clouds_yaml_data_new - ) - total_entries = len(clouds_yaml_data_new.get("clouds", [])) - - with open(args.generate_clouds_yaml, "w") as file: - yaml.dump( - clouds_yaml_data_new, - file, - default_flow_style=False, - explicit_start=True, - ) - LOGGER.info(f"Generated {generated_entries} entries, number of entries in " - f"{args.generate_clouds_yaml} is now {total_entries} (old {initial_entries} entries)") + generated_clouds_yaml() + sys.exit(0) elif args.delete_projects: conn = establish_connection() From 6dc535d15088c8c8cb3e787fb422e40ffe36cf66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Sch=C3=B6chlin?= Date: Thu, 12 Jun 2025 16:27:18 +0200 Subject: [PATCH 3/5] improve gitignore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marc Schöchlin --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7090e23..6c7cd59 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ venv __pycache__ clouds*.yaml *clouds.yaml - +*.yaml_????-??-??_??-??-?? From 22bc5551e11e63b184df0928da06b3a1f57f52a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Sch=C3=B6chlin?= Date: Mon, 16 Jun 2025 11:41:43 +0200 Subject: [PATCH 4/5] Set quotas on every execution and bugfix the lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marc Schöchlin --- src/openstack_workload_generator/entities/domain.py | 1 + .../entities/helpers.py | 13 ++++++++++--- .../entities/project.py | 4 +--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/openstack_workload_generator/entities/domain.py b/src/openstack_workload_generator/entities/domain.py index 5ed3488..a194b4c 100644 --- a/src/openstack_workload_generator/entities/domain.py +++ b/src/openstack_workload_generator/entities/domain.py @@ -97,6 +97,7 @@ def create_and_get_projects(self, create_projects: list[str]): for project_name in create_projects: if project_name in self.workload_projects: + self.workload_projects[project_name].adapt_quota() continue project = WorkloadGeneratorProject( self.conn, project_name, self.obj, self.workload_user diff --git a/src/openstack_workload_generator/entities/helpers.py b/src/openstack_workload_generator/entities/helpers.py index ee36086..ba0c4e9 100644 --- a/src/openstack_workload_generator/entities/helpers.py +++ b/src/openstack_workload_generator/entities/helpers.py @@ -62,9 +62,14 @@ def get(key: str, regex: str = ".+", multi_line: bool = False) -> str: @staticmethod def load_config(config_file: str): - potential_profile_file = str( + + profile_path = str( os.path.realpath(os.path.dirname(os.path.realpath(__file__))) - + f"/../../../profiles/{config_file}" + + "/../../../profiles/" + ) + potential_profile_file = str( + Path(os.getenv("OPENSTACK_WORKLOAD_MANAGER_PROFILES", profile_path)) + / Path(config_file) ) if os.getenv("OPENSTACK_WORKLOAD_MANAGER_PROFILES", None): @@ -192,7 +197,9 @@ def configured_quota_names(quota_category: str) -> list[str]: @staticmethod def quota(quota_name: str, quota_category: str, default_value: int) -> int: if quota_category in Config._config: - value = Config._config.get(quota_name, default_value) + value = Config._config[quota_category].get( # type: ignore + quota_name, default_value + ) if isinstance(value, int): return value else: diff --git a/src/openstack_workload_generator/entities/project.py b/src/openstack_workload_generator/entities/project.py index decf5d3..7aaaa97 100644 --- a/src/openstack_workload_generator/entities/project.py +++ b/src/openstack_workload_generator/entities/project.py @@ -169,8 +169,6 @@ def _set_quota(self, quota_category: str): else: raise RuntimeError(f"Not implemented: {quota_category}") - # service_obj = getattr(self._admin_conn, api_area) - # current_quota = service_obj.get_quota_set(self.obj.id) LOGGER.debug(f"current quotas for {quota_category} : {current_quota}") new_quota = {} @@ -193,7 +191,7 @@ def _set_quota(self, quota_category: str): ) new_quota[key_name] = new_value - if len(new_quota): + if len(new_quota.keys()) > 0: set_quota_method = getattr(self._admin_conn, f"set_{api_area}_quotas") set_quota_method(self.obj.id, **new_quota) LOGGER.info( From f3a69eee5e18817b6fbf438e14ef62cc839fe15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Sch=C3=B6chlin?= Date: Mon, 16 Jun 2025 12:21:25 +0200 Subject: [PATCH 5/5] Add facility to ask passwords MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - add documentation Signed-off-by: Marc Schöchlin --- README.md | 64 ++++++++++++------- .../entities/helpers.py | 7 +- 2 files changed, 48 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 98d28ad..0dba608 100644 --- a/README.md +++ b/README.md @@ -26,35 +26,26 @@ basis for later automation. # Usage ``` -$ ./openstack_workload_generator --help -usage: Create workloads on openstack installations [-h] [--log_level loglevel] [--os_cloud OS_CLOUD] - [--ansible_inventory [ANSIBLE_INVENTORY]] - [--clouds_yaml [CLOUDS_YAML]] [--wait_for_machines] - [--generate_clouds_yaml [GENERATE_CLOUDS_YAML]] - [--config CONFIG] - (--create_domains DOMAINNAME [DOMAINNAME ...] | - --delete_domains DOMAINNAME [DOMAINNAME ...]) - [--create_projects PROJECTNAME [PROJECTNAME ...] | - --delete_projects PROJECTNAME [PROJECTNAME ...]] - [--create_machines SERVERNAME [SERVERNAME ...] | - --delete_machines SERVERNAME [SERVERNAME ...]] +$ openstack_workload_generator --help +usage: Create workloads on openstack installations [-h] [--log_level loglevel] [--os_cloud OS_CLOUD] [--ansible_inventory [ANSIBLE_INVENTORY]] + [--clouds_yaml [CLOUDS_YAML]] [--wait_for_machines] [--generate_clouds_yaml [GENERATE_CLOUDS_YAML]] + [--config CONFIG] (--create_domains DOMAINNAME [DOMAINNAME ...] | --delete_domains DOMAINNAME [DOMAINNAME ...]) + [--create_projects PROJECTNAME [PROJECTNAME ...] | --delete_projects PROJECTNAME [PROJECTNAME ...]] + [--create_machines SERVERNAME [SERVERNAME ...] | --delete_machines SERVERNAME [SERVERNAME ...]] options: -h, --help show this help message and exit --log_level loglevel The loglevel - --os_cloud OS_CLOUD The openstack config to use, defaults to the value of the OS_CLOUD environment variable or - "admin" if the variable is not set + --os_cloud OS_CLOUD The openstack config to use, defaults to the value of the OS_CLOUD environment variable or "admin" if the variable is not set --ansible_inventory [ANSIBLE_INVENTORY] - Dump the created servers as an ansible inventory to the specified directory, adds a ssh - proxy jump for the hosts without a floating ip + Dump the created servers as an ansible inventory to the specified directory, adds a ssh proxy jump for the hosts without a floating ip --clouds_yaml [CLOUDS_YAML] Use a specific clouds.yaml file - --wait_for_machines Wait for every machine to be created (normally the provisioning only waits for machines - which use floating ips) + --wait_for_machines Wait for every machine to be created (normally the provisioning only waits for machines which use floating ips) --generate_clouds_yaml [GENERATE_CLOUDS_YAML] Generate a openstack clouds.yaml file - --config CONFIG The config file for environment creation, define a path to the yaml file or a subpath in - the profiles folder + --config CONFIG The config file for environment creation, define a path to the yaml file or a subpath in the profiles folder of the tool (you can overload + the search path by setting the OPENSTACK_WORKLOAD_MANAGER_PROFILES environment variable) --create_domains DOMAINNAME [DOMAINNAME ...] A list of domains to be created --delete_domains DOMAINNAME [DOMAINNAME ...] @@ -62,14 +53,43 @@ options: --create_projects PROJECTNAME [PROJECTNAME ...] A list of projects to be created in the created domains --delete_projects PROJECTNAME [PROJECTNAME ...] - A list of projects to be deleted in the created domains, all child elements are - recursively deleted + A list of projects to be deleted in the created domains, all child elements are recursively deleted --create_machines SERVERNAME [SERVERNAME ...] A list of vms to be created in the created domains --delete_machines SERVERNAME [SERVERNAME ...] A list of vms to be deleted in the created projects ``` +# Configuration + +The following cnfigurations: + +* `admin_domain_password` + * the password for the domain users which are created (User `_admin`) + * If you add "ASK_PASSWORD" as a value, the password will be asked in an interactive way +* `admin_vm_password`: + * the password for the operating system user (the username depends on the type of image you are using) + * If you add "ASK_PASSWORD" as a value, the password will be asked in an interactive way +* `vm_flavor`: + * the name of the flavor used to create virtual machines + * see `openstack flavor list` +* `vm_image`: + * the name of the image used to create virtual machines + * see `openstack image list` +* `vm_volume_size_gb`: + * the size of the persistent root volume +* `project_ipv4_subnet`: + * the network cidr of the internal network +* `*_quotas`: + * the quotas for the created projects + * execute the tool with `--log_level DEBUG` to see the configurable values +* `public_network`: + * The name of the public network which is used for floating ips +* `admin_vm_ssh_key`: + * A multiline string which ssh public keys + +``` + # Testing Scenarios ## Example usage: A minimal scenario diff --git a/src/openstack_workload_generator/entities/helpers.py b/src/openstack_workload_generator/entities/helpers.py index ba0c4e9..5ded413 100644 --- a/src/openstack_workload_generator/entities/helpers.py +++ b/src/openstack_workload_generator/entities/helpers.py @@ -1,3 +1,4 @@ +import getpass import inspect import logging import os @@ -140,7 +141,9 @@ def get_number_of_floating_ips_per_project() -> int: @staticmethod def get_admin_vm_password() -> str: - return Config.get("admin_vm_password") + if Config.get("admin_vm_password").upper() == "ASK_PASSWORD": + Config._config["admin_vm_password"] = getpass.getpass("Enter the wanted admin_vm_password: ") + return Config.get("admin_vm_password", regex=r".{5,}") @staticmethod def get_vm_flavor() -> str: @@ -176,6 +179,8 @@ def get_admin_vm_ssh_key() -> str: @staticmethod def get_admin_domain_password() -> str: + if Config.get("admin_domain_password").upper() == "ASK_PASSWORD": + Config._config["admin_domain_password"] = getpass.getpass("Enter the wanted admin_domain_password: ") return Config.get("admin_domain_password", regex=r".{5,}") @staticmethod