From f6a497cf1333bf03745d520b1410c54a4a85855c Mon Sep 17 00:00:00 2001 From: Alan King Date: Tue, 5 May 2026 17:28:33 -0400 Subject: [PATCH] [irods/irods#7717] Move TLS configuration to setup In iRODS 5.1.0 and later, TLS can be configured at setup. This commit changes the time of TLS configuration to setup rather than after the zone is set up (which is how it had been done historically). --- irods_testing_environment/irods_setup.py | 129 ++++++++++++++++++++--- irods_testing_environment/services.py | 28 ++++- stand_it_up.py | 41 ++++--- 3 files changed, 161 insertions(+), 37 deletions(-) diff --git a/irods_testing_environment/irods_setup.py b/irods_testing_environment/irods_setup.py index 7ffe572..581df12 100644 --- a/irods_testing_environment/irods_setup.py +++ b/irods_testing_environment/irods_setup.py @@ -1,11 +1,11 @@ -# grown-up modules import concurrent.futures +import itertools import json import logging import os +import pathlib -# local modules -from . import context, database_setup, execute, irods_config, odbc_setup +from . import archive, context, database_setup, execute, irods_config, odbc_setup class zone_info(object): @@ -242,6 +242,14 @@ def setup(self, self.do_unattended_install = kwargs.get('do_unattended_install', False) + self.use_tls = kwargs.get('use_tls', False) + self.certificate_chain_file = str(pathlib.Path(context.irods_config()) / "chain.pem") + self.certificate_key_file = str(pathlib.Path(context.irods_config()) / "server.key") + self.dh_params_file = str(pathlib.Path(context.irods_config()) / "dhparams.pem") + self.ca_certificate_file = str(pathlib.Path(context.irods_config()) / "server.crt") + self.ca_certificate_path = "" # This is optional but should be defined. + self.verify_server = "cert" + return self @@ -290,6 +298,22 @@ def build_input_for_catalog_consumer(self): input_args.insert(4, str(self.provides_local_storage)) input_args.insert(5, str(self.resource_name)) input_args.insert(6, str(self.vault_directory)) + + # Insert entries for TLS prompts (added in 5.1.0). + if self.irods_version >= (5, 0, 90): + insert_index = itertools.count(7) + if self.use_tls: + # Prompt for generating self-signed certificate. Testing environment takes care of this, so decline. + input_args.insert(next(insert_index), "no") + # tls_server + input_args.insert(next(insert_index), self.certificate_chain_file) + input_args.insert(next(insert_index), self.certificate_key_file) + input_args.insert(next(insert_index), self.dh_params_file) + # tls_client + input_args.insert(next(insert_index), self.ca_certificate_file) + input_args.insert(next(insert_index), self.ca_certificate_path) + input_args.insert(next(insert_index), str(2 if self.verify_server == "cert" else 1)) + input_args.insert(next(insert_index), "") # confirmation # Handle the difference between 4.2 servers and 4.3 servers. elif self.irods_version >= (4, 3, 0): input_args.insert(3, str(self.provides_local_storage)) @@ -364,6 +388,22 @@ def build_input_for_catalog_provider(self): input_args.insert(12, str(self.provides_local_storage)) input_args.insert(13, str(self.resource_name)) input_args.insert(14, str(self.vault_directory)) + + # Insert entries for TLS prompts (added in 5.1.0). + if self.irods_version >= (5, 0, 90): + insert_index = itertools.count(15) + if self.use_tls: + # Prompt for generating self-signed certificate. Testing environment takes care of this, so decline. + input_args.insert(next(insert_index), "no") + # tls_server + input_args.insert(next(insert_index), self.certificate_chain_file) + input_args.insert(next(insert_index), self.certificate_key_file) + input_args.insert(next(insert_index), self.dh_params_file) + # tls_client + input_args.insert(next(insert_index), self.ca_certificate_file) + input_args.insert(next(insert_index), self.ca_certificate_path) + input_args.insert(next(insert_index), str(2 if self.verify_server == "cert" else 1)) + input_args.insert(next(insert_index), "") # confirmation # Handle the difference between 4.2 servers and 4.3 servers. elif self.irods_version >= (4, 3, 0): input_args.insert(11, str(self.provides_local_storage)) @@ -570,6 +610,29 @@ def build_unattended_install_input_for_catalog_consumer(self): "server_control_plane_timeout_milliseconds": 10000 }) + else: + if self.use_tls: + json_input["server_config"]["client_server_policy"] = "CS_NEG_REQUIRE" + json_input["server_config"]["tls_server"] = { + "certificate_chain_file": self.certificate_chain_file, + "certificate_key_file": self.certificate_key_file, + "dh_params_file": self.dh_params_file, + } + json_input["server_config"]["tls_client"] = {"verify_server": self.verify_server} + if self.ca_certificate_file: + json_input["server_config"]["tls_client"]["ca_certificate_file"] = self.ca_certificate_file + json_input["service_account_environment"]["irods_ssl_ca_certificate_file"] = ( + self.ca_certificate_file + ) + if self.ca_certificate_path: + json_input["server_config"]["tls_client"]["ca_certificate_path"] = self.ca_certificate_path + json_input["service_account_environment"]["irods_ssl_ca_certificate_path"] = ( + self.ca_certificate_path + ) + + json_input["service_account_environment"]["irods_client_server_policy"] = "CS_NEG_REQUIRE" + json_input["service_account_environment"]["irods_ssl_verify_server"] = self.verify_server + return json.dumps(json_input, sort_keys=True, indent=4) @@ -788,6 +851,28 @@ def build_unattended_install_input_for_catalog_provider(self): "server_control_plane_timeout_milliseconds": 10000 }) + else: + if self.use_tls: + json_input["server_config"]["client_server_policy"] = "CS_NEG_REQUIRE" + json_input["server_config"]["tls_server"] = { + "certificate_chain_file": self.certificate_chain_file, + "certificate_key_file": self.certificate_key_file, + "dh_params_file": self.dh_params_file, + } + json_input["server_config"]["tls_client"] = {"verify_server": self.verify_server} + if self.ca_certificate_file: + json_input["server_config"]["tls_client"]["ca_certificate_file"] = self.ca_certificate_file + json_input["service_account_environment"]["irods_ssl_ca_certificate_file"] = ( + self.ca_certificate_file + ) + if self.ca_certificate_path: + json_input["server_config"]["tls_client"]["ca_certificate_path"] = self.ca_certificate_path + json_input["service_account_environment"]["irods_ssl_ca_certificate_path"] = ( + self.ca_certificate_path + ) + json_input["service_account_environment"]["irods_client_server_policy"] = "CS_NEG_REQUIRE" + json_input["service_account_environment"]["irods_ssl_verify_server"] = self.verify_server + return json.dumps(json_input, sort_keys=True, indent=4) def build(self): @@ -911,6 +996,24 @@ def setup_irods_server(container, setup_input, **kwargs): from . import container_info from . import irods_config + if kwargs.get("use_tls", False): + config_path = pathlib.Path(context.irods_config()) + key_file = config_path / 'server.key' + dhparams_file = config_path / 'dhparams.pem' + chain_file = config_path / 'chain.pem' + cert_file = config_path / 'server.crt' + + # Chain file and cert file use the same source for self-signed certs. + archive.copy_files_in_container( + container, + [ + (kwargs.get("path_to_key_file_on_host"), key_file), + (kwargs.get("path_to_cert_file_on_host"), chain_file), + (kwargs.get("path_to_cert_file_on_host"), cert_file), + (kwargs.get("path_to_dhparams_file_on_host"), dhparams_file), + ], + ) + try: if stop_irods(container) != 0: logging.debug(f'[{container.name}] failed to stop iRODS server before setup') @@ -939,8 +1042,12 @@ def setup_irods_server(container, setup_input, **kwargs): run_setup_script = 'bash -c \'{} {} --json_configuration_file /input\''.format( container_info.python(container), path_to_setup_script) else: - run_setup_script = 'bash -c \'{} {} < /input\''.format( - container_info.python(container), path_to_setup_script) + args = [] + if irods_config.get_irods_version(container) >= (5, 0, 90): + if kwargs.get("use_tls", False): + args.append("--tls") + args = " ".join(args) + run_setup_script = f'bash -c \'{container_info.python(container)} {path_to_setup_script} {args} < /input\'' ec = execute.execute_command(container, run_setup_script) if ec != 0: raise RuntimeError('failed to set up iRODS server [{}]'.format(container.name)) @@ -1003,9 +1110,7 @@ def setup_irods_catalog_provider(ctx, logging.warning('setting up iRODS catalog provider [{}]'.format(csp_container.name)) - setup_irods_server(csp_container, - setup_input, - do_unattended_install=kwargs.get('do_unattended_install', False)) + setup_irods_server(csp_container, setup_input, **kwargs) def setup_irods_catalog_consumer(ctx, @@ -1052,9 +1157,7 @@ def setup_irods_catalog_consumer(ctx, logging.warning('setting up iRODS catalog consumer [{}]'.format(csc_container.name)) - setup_irods_server(csc_container, - setup_input, - do_unattended_install=kwargs.get('do_unattended_install', False)) + setup_irods_server(csc_container, setup_input, **kwargs) def setup_irods_catalog_consumers(ctx, @@ -1072,8 +1175,6 @@ def setup_irods_catalog_consumers(ctx, consumer service name in the Compose project will be targeted. If an empty list is provided, nothing happens. """ - import concurrent.futures - catalog_consumer_containers = ctx.compose_project.containers( service_names=[context.irods_catalog_consumer_service()]) @@ -1170,8 +1271,6 @@ def setup_irods_zones(ctx, zone_info_list, odbc_driver=None, **kwargs): - import concurrent.futures - rc = 0 with concurrent.futures.ThreadPoolExecutor() as executor: diff --git a/irods_testing_environment/services.py b/irods_testing_environment/services.py index a948ca4..d278534 100644 --- a/irods_testing_environment/services.py +++ b/irods_testing_environment/services.py @@ -1,12 +1,13 @@ -# grown-up modules +"""Utility functions for managing Docker Compose services and setting up iRODS topologies.""" + import logging import os +import pathlib -# local modules -from . import context -from . import irods_setup +from . import context, irods_setup, tls_setup from .install import install + def create_topologies(ctx, zone_count, externals_directory=None, @@ -51,7 +52,24 @@ def create_topologies(ctx, # This should generate a list of identical zone infos zone_info_list = irods_setup.get_info_for_zones(ctx, zone_names, consumer_count) - irods_setup.setup_irods_zones(ctx, zone_info_list, odbc_driver=odbc_driver, **kwargs) + if kwargs.get("use_tls", False): + # The testing environment is using a self-signed certificate, so the certificate, key, and dhparams should be + # generated ONCE and copied to each server. + key, key_file = tls_setup.generate_tls_certificate_key() + cert_file = tls_setup.generate_tls_self_signed_certificate(key) + dhparams_file = tls_setup.generate_tls_dh_params() + kwargs["path_to_key_file_on_host"] = key_file + kwargs["path_to_cert_file_on_host"] = cert_file + kwargs["path_to_dhparams_file_on_host"] = dhparams_file + + try: + irods_setup.setup_irods_zones(ctx, zone_info_list, odbc_driver=odbc_driver, **kwargs) + + finally: + if kwargs.get("use_tls", False): + pathlib.Path(key_file).unlink() + pathlib.Path(cert_file).unlink() + pathlib.Path(dhparams_file).unlink() def create_topology(ctx, diff --git a/stand_it_up.py b/stand_it_up.py index 53931a2..3078412 100644 --- a/stand_it_up.py +++ b/stand_it_up.py @@ -1,13 +1,10 @@ -# grown-up modules -import compose.cli.command -import docker import logging import os -# local modules -from irods_testing_environment import context -from irods_testing_environment import services -from irods_testing_environment import tls_setup +import docker + +import compose.cli.command +from irods_testing_environment import context, irods_config, services, tls_setup if __name__ == "__main__": import argparse @@ -63,14 +60,24 @@ # Bring up the services logging.debug('bringing up project [{}]'.format(ctx.compose_project.name)) - services.create_topology(ctx, - externals_directory=args.irods_externals_package_directory, - package_directory=args.package_directory, - package_version=args.package_version, - odbc_driver=args.odbc_driver, - consumer_count=args.consumer_count, - install_packages=args.install_packages, - do_unattended_install=args.do_unattended_install) - - if args.use_tls: + services.create_topology( + ctx, + externals_directory=args.irods_externals_package_directory, + package_directory=args.package_directory, + package_version=args.package_version, + odbc_driver=args.odbc_driver, + consumer_count=args.consumer_count, + install_packages=args.install_packages, + do_unattended_install=args.do_unattended_install, + use_tls=args.use_tls, + ) + + containers = [ + ctx.docker_client.containers.get( + context.container_name(ctx.compose_project.name, context.irods_catalog_provider_service()) + ) + ] + + # TLS configuration happens in setup as of 5.1.0, so only do this for prior versions when requested. + if args.use_tls and irods_config.get_irods_version(containers[0]) < (5, 0, 90): tls_setup.configure_tls_in_zone(ctx.docker_client, ctx.compose_project)