[#295] Add ability to upgrade packages on running iRODS Zones

README.md

@@ -221,6 +221,38 @@ By default, Docker Compose uses the directory housing the target Compose file as
 
 For more information about Compose project names, read this: [https://docs.docker.com/compose/how-tos/project-name](https://docs.docker.com/compose/how-tos/project-name/).
 
+## Upgrading packages before running tests
+
+The `--upgrade-package-version` and `--upgrade-package-directory` options allow the user to specify packages to install after initially installing iRODS packages and setting up the iRODS Zone. This is helpful for ensuring that upgrade logic for a particular release is behaving as expected, at least for a default-configured iRODS Zone. Package upgrades only occur if one of these two options are specified.
+
+> [!NOTE]
+> Remember: Downgrading iRODS servers is not supported.
+
+For instance, if you want to run core tests after upgrading an iRODS 4.3.5 Zone to an iRODS 5.0.2 Zone, you can run the following:
+```bash
+python3 run_core_tests.py \
+    --project-dir projects/ubuntu-24.04/ubuntu-24.04-postgres-16/ \
+    --irods-package-version 4.3.5 \
+    --upgrade-package-version 5.0.2
+```
+
+If you want to upgrade to locally built packages instead, you can run the following:
+```bash
+python3 run_core_tests.py \
+    --project-dir projects/ubuntu-24.04/ubuntu-24.04-postgres-16/ \
+    --irods-package-version 4.3.5 \
+    --upgrade-package-directory /path/to/irods/package/directory
+```
+
+You can use these options in conjunction with `--skip-setup` to upgrade packages and then run tests on an already running iRODS server, like this:
+```bash
+# --upgrade-package-version also works in this case
+python3 run_core_tests.py \
+    --project-dir projects/ubuntu-24.04/ubuntu-24.04-postgres-16/ \
+    --upgrade-package-directory /path/to/irods/package/directory \
+    --skip-setup
+```
+
 ## View results with `xunit-viewer`
 
 An `xunit-viewer` (https://github.com/lukejpreston/xunit-viewer) Dockerfile was added so that the JUnit XML reports can be viewed a little more easily.

cli.py

@@ -146,19 +146,29 @@ def add_irods_test_args(parser):
     )
 
 
-def add_database_config_args(parser):
-    '''Add argparse options related to setting up and configuring iRODS.
+def add_irods_setup_args(parser):
+    """
+    Add argparse options related to setting up and configuring iRODS.
 
-    Arguments:
-    parser -- argparse.ArgumentParser to augment
-    '''
+    Args:
+        parser: argparse.ArgumentParser to augment
+    """
     parser.add_argument('--odbc-driver-path',
                         metavar='PATH_TO_ODBC_DRIVER_ARCHIVE',
                         dest='odbc_driver',
                         help=textwrap.dedent('''\
                             Path to the ODBC driver archive file on the local machine. \
                             If not provided, the driver will be downloaded.'''))
 
+    parser.add_argument(
+        '--use-tls',
+        dest='use_tls',
+        action='store_true',
+        help=textwrap.dedent('''\
+                            Indicates that TLS should be configured and enabled in the test Zone.'''),
+    )
+
+
 def add_common_args(parser):
     '''Add argparse options common to irods_testing_environment scripts.
 

federate.py

@@ -27,8 +27,8 @@
 
     cli.add_common_args(parser)
     cli.add_compose_args(parser)
-    cli.add_database_config_args(parser)
     cli.add_irods_package_args(parser)
+    cli.add_irods_setup_args(parser)
 
     parser.add_argument('--consumers-per-zone',
                         metavar='IRODS_CATALOG_CONSUMER_INSTANCES_PER_ZONE',
@@ -54,12 +54,6 @@
                         action='store_false', dest='do_setup',
                         help='If indicated, the Zones will not be set up, only federated.')
 
-    parser.add_argument('--use-tls',
-                        dest='use_tls', action='store_true',
-                        help=textwrap.dedent('''\
-                            Indicates that TLS should be configured and enabled in each Zone.\
-                            '''))
-
     parser.add_argument('--use-unattended-install',
                         action='store_true', dest='do_unattended_install',
                         help='''\

irods_testing_environment/irods_config.py

@@ -39,38 +39,77 @@ def get_irods_zone_name(container):
     return irods_zone[container.name]
 
 
+def _get_irods_version_from_file(container):
+    """
+    Get the version of iRODS running on container from the iRODS version file.
+
+    Args:
+        container: container where version is being checked
+
+    Returns:
+        iRODS version tuple in the form (major, minor, patch).
+    """
+    return tuple(int(i) for i in get_irods_version_info(container, 'irods_version').split('.'))
+
+
+def update_cached_irods_version(container):
+    """
+    Update the cached version tuple of iRODS running on container.
+
+    Args:
+        container: container in which file is found
+    """
+    irods_version[container.name] = _get_irods_version_from_file(container)
+
+
 def get_irods_version(container):
-    """Return the version of iRODS running on `container` as a tuple (major, minor, patch).
+    """
+    Return the version of iRODS running on container.
 
-    Arguments:
-    container -- container in which file is found
+    Args:
+        container: container in which file is found
+
+    Returns:
+        iRODS version tuple in the form (major, minor, patch).
     """
     global irods_version
 
     # If we have the iRODS version cached for this container, return that.
     if container.name in irods_version:
         return irods_version[container.name]
 
-    irods_version[container.name] = tuple(
-        int(i) for i in get_irods_version_info(container, 'irods_version').split('.')
-    )
+    update_cached_irods_version(container)
 
     return irods_version[container.name]
 
 
+def update_cached_irods_commit_id(container):
+    """
+    Update cached commit ID of the build of iRODS running on container.
+
+    Args:
+        container: container in which version file is found
+    """
+    irods_commit_id[container.name] = get_irods_version_info(container, 'commit_id')
+
+
 def get_irods_commit_id(container):
-    """Return the commit ID of the build of iRODS running on `container`.
+    """
+    Get the commit ID of iRODS running on container.
 
-    Arguments:
-    container -- container in which file is found
+    Args:
+        container: container in which version file is found
+
+    Returns:
+        Commit ID (sha) for the build of iRODS running in container.
     """
     global irods_commit_id
 
     # If we have the iRODS commit ID cached for this container, return that.
     if container.name in irods_commit_id:
         return irods_commit_id[container.name]
 
-    irods_commit_id[container.name] = get_irods_version_info(container, 'commit_id')
+    update_cached_irods_commit_id(container)
 
     return irods_commit_id[container.name]
 

irods_testing_environment/irods_setup.py

@@ -1,16 +1,12 @@
 # grown-up modules
-import compose
-import docker
+import concurrent.futures
 import json
 import logging
 import os
 
 # local modules
-from . import context
-from . import database_setup
-from . import odbc_setup
-from . import execute
-from . import irods_config
+from . import context, database_setup, execute, irods_config, odbc_setup
+
 
 class zone_info(object):
     """Class to hold information about an iRODS Zone and the containers running the servers."""
@@ -1262,3 +1258,128 @@ def get_info_for_zones(ctx, zone_names, consumer_service_instances_per_zone=0):
         )
 
     return zone_info_list
+
+
+def upgrade_irods_zone(ctx, provider_service_instance=1, consumer_service_instances=None):
+    """
+    Run the iRODS upgrade script in the Zone indicated by the catalog service provider instance.
+
+    Args:
+        ctx: context object which contains information about the Docker environment
+        provider_service_instance: service instance for the iRODS CSP container for this Zone
+        consumer_service_instances: service instances for the iRODS Catalog Service Consumer containers for this Zone
+                                    (if None is provided, all running iRODS Catalog Service Consumer service instances
+                                    are determined to be part of this Zone, per the irods_setup interfaces. list()
+                                    indicates that no iRODS Catalog Service Consumers are in this zone.
+
+    Raises:
+        RuntimeError: if an error occurs while running the upgrade script on any container in the Zone
+    """
+
+    def upgrade_irods(container):
+        # iRODS 5 introduced the upgrade script concept. Skip for anything before then.
+        if irods_config.server_version_is_irods_5(container):
+            ec = execute.execute_command(
+                container, 'python3 scripts/upgrade_irods.py', user='irods', workdir=context.irods_home()
+            )
+            if ec != 0:
+                raise RuntimeError(f'[{container.name}]: failed to run upgrade script')
+
+        if restart_irods(container) != 0:
+            raise RuntimeError(f'[{container.name}]: failed to start iRODS server after upgrade')
+
+        # After upgrade, need to invalidate the version and commit ID kept in cache in irods_config.
+        irods_config.update_cached_irods_version(container)
+        irods_config.update_cached_irods_commit_id(container)
+
+    rc = 0
+
+    # Upgrade catalog service provider first.
+    csp_container = ctx.docker_client.containers.get(
+        context.irods_catalog_provider_container(ctx.compose_project.name, provider_service_instance)
+    )
+    upgrade_irods(csp_container)
+
+    # Get information about the catalog service consumers, if any.
+    catalog_consumer_containers = ctx.compose_project.containers(
+        service_names=[context.irods_catalog_consumer_service()]
+    )
+
+    if consumer_service_instances:
+        if len(consumer_service_instances) == 0:
+            logging.warning('empty list of iRODS catalog service consumers to set up')  # noqa: LOG015
+            return
+
+        consumer_service_instances = [
+            context.service_instance(c.name)
+            for c in catalog_consumer_containers
+            if context.service_instance(c.name) in consumer_service_instances
+        ]
+    else:
+        consumer_service_instances = [context.service_instance(c.name) for c in catalog_consumer_containers]
+
+    # Upgrade all the catalog consumers at once.
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures_to_catalog_consumer_instances = {
+            executor.submit(
+                upgrade_irods,
+                ctx.docker_client.containers.get(
+                    context.irods_catalog_consumer_container(ctx.compose_project.name, instance)
+                ),
+            ): instance
+            for instance in consumer_service_instances
+        }
+
+        logging.debug(futures_to_catalog_consumer_instances)  # noqa: LOG015
+
+        for f in concurrent.futures.as_completed(futures_to_catalog_consumer_instances):
+            i = futures_to_catalog_consumer_instances[f]
+            container_name = context.irods_catalog_consumer_container(ctx.compose_project.name, i + 1)
+            try:
+                f.result()
+                logging.debug('[%s]: upgrade completed successfully', container_name)  # noqa: LOG015
+
+            except Exception:
+                logging.exception("[%s]: Exception occurred while upgrading packages", container_name)  # noqa: LOG015
+                rc = 1
+
+    if rc != 0:
+        raise RuntimeError(f'failed to upgrade packages one or more catalog service consumers, ec=[{rc}]')
+
+
+def upgrade_irods_zones(ctx, zone_info_list):
+    """
+    Run the iRODS upgrade script in specified Zones.
+
+    Args:
+        ctx: context object which contains information about the Docker environment
+        zone_info_list: list of iRODS Zone information for the Zones on which packages should be upgraded
+
+    Raises:
+        RuntimeError: if an error occurs while upgrading the iRODS packages on any container
+    """
+    rc = 0
+
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures_to_zone_infos = {
+            executor.submit(
+                upgrade_irods_zone,
+                ctx,
+                provider_service_instance=z.provider_service_instance,
+                consumer_service_instances=z.consumer_service_instances,
+            ): z
+            for z in zone_info_list
+        }
+
+        for f in concurrent.futures.as_completed(futures_to_zone_infos):
+            zone = futures_to_zone_infos[f]
+            try:
+                f.result()
+                logging.debug('[%s]: iRODS Zone upgrade completed successfully', zone)  # noqa: LOG015
+
+            except Exception:
+                logging.exception("Exception occurred while upgrading packages in Zone [%s]", zone)  # noqa: LOG015
+                rc = 1
+
+    if rc != 0:
+        raise RuntimeError(f'failed to upgrade packages on one or more iRODS Zones, ec=[{rc}]')

irods_testing_environment/services.py

@@ -119,3 +119,33 @@ def clone_repository_to_container(container,
                                             [os.path.abspath(repo_path)], repo_name))
 
     return repo_path
+
+
+def upgrade_irods_packages(
+    ctx, zone_count, package_directory=None, package_version=None, zone_name='tempZone', consumer_count=0
+):
+    """
+    Upgrade existing iRODS packages on the specified, identically-named iRODS Zones.
+
+    This is a convenience function for upgrading iRODS packages on multiple iRODS Zones.
+
+    Args:
+        ctx: context object which holds the Docker client and Compose project information
+        zone_count: number of identical zones to scale up to
+        package_directory: path to directory in which iRODS packages are housed
+        package_version: version tag for official iRODS packages to download and install
+        zone_name: the Zone name shared by all the iRODS Zones
+        consumer_count: number of iRODS Catalog Service Consumers to create and set up for each Zone
+    """
+    install.make_installer(ctx.platform_name()).install_irods_packages(
+        ctx,
+        package_directory=package_directory,
+        package_version=package_version,
+    )
+
+    zone_names = [zone_name for i in range(zone_count)]
+
+    # This should generate a list of identical zone infos
+    zone_info_list = irods_setup.get_info_for_zones(ctx, zone_names, consumer_count)
+
+    irods_setup.upgrade_irods_zones(ctx, zone_info_list)