zuul: new variable to easily populate TEMPEST_PLUGINS

TEMPEST_PLUGINS contains the list of the tempest plugins installed
alongside tempest by lib/tempest.
If TEMPEST_PLUGINS is not explicitly set, the new tempest_plugins
variable is used to fill it by combining its items with
the base devstack path.

Change-Id: I9f1fa2755e16871ff9d6ba33fdeaf3023eedf8d4

Author: tosky

doc/source/zuul_ci_jobs_migration.rst

@@ -102,7 +102,6 @@ job.parent.
         tox_envlist: 'all'
         devstack_localrc:
           KURYR_K8S_API_PORT: 8080
-          TEMPEST_PLUGINS: '/opt/stack/kuryr-tempest-plugin'
         devstack_services:
           kubernetes-api: true
           kubernetes-controller-manager: true
@@ -114,6 +113,8 @@ job.parent.
           kuryr-kubernetes: https://git.openstack.org/openstack/kuryr
           devstack-plugin-container: https://git.openstack.org/openstack/devstack-plugin-container
           neutron-lbaas: https://git.openstack.org/openstack/neutron-lbaas
+        tempest_plugins:
+          - kuryr-tempest-plugin
         (...)
 
 Job variables

roles/write-devstack-local-conf/README.rst

@@ -88,3 +88,12 @@ Write the local.conf file for use by devstack
    If a plugin declares a dependency on another plugin (via
    ``plugin_requires`` in the plugin's settings file), this role will
    automatically emit ``enable_plugin`` lines in the correct order.
+
+.. zuul:rolevar:: tempest_plugins
+   :type: list
+
+   A list of tempest plugins which are installed alongside tempest.
+
+   The list of values will be combined with the base devstack directory
+   and used to populate the ``TEMPEST_PLUGINS`` variable. If the variable
+   already exists, its value is *not* changed.

roles/write-devstack-local-conf/library/devstack_local_conf.py

@@ -207,13 +207,15 @@ def getPlugins(self):
 class LocalConf(object):
 
     def __init__(self, localrc, localconf, base_services, services, plugins,
-                 base_dir, projects, project):
+                 base_dir, projects, project, tempest_plugins):
         self.localrc = []
+        self.warnings = []
         self.meta_sections = {}
         self.plugin_deps = {}
         self.base_dir = base_dir
         self.projects = projects
         self.project = project
+        self.tempest_plugins = tempest_plugins
         if services or base_services:
             self.handle_services(base_services, services or {})
         self.handle_localrc(localrc)
@@ -246,12 +248,15 @@ def handle_services(self, base_services, services):
 
     def handle_localrc(self, localrc):
         lfg = False
+        tp = False
         if localrc:
             vg = VarGraph(localrc)
             for k, v in vg.getVars():
                 self.localrc.append('{}={}'.format(k, v))
                 if k == 'LIBS_FROM_GIT':
                     lfg = True
+                elif k == 'TEMPEST_PLUGINS':
+                    tp = True
 
         if not lfg and (self.projects or self.project):
             required_projects = []
@@ -266,6 +271,19 @@ def handle_localrc(self, localrc):
                 self.localrc.append('LIBS_FROM_GIT={}'.format(
                     ','.join(required_projects)))
 
+        if self.tempest_plugins:
+            if not tp:
+                tp_dirs = []
+                for tempest_plugin in self.tempest_plugins:
+                    tp_dirs.append(os.path.join(self.base_dir, tempest_plugin))
+                self.localrc.append('TEMPEST_PLUGINS="{}"'.format(
+                        ' '.join(tp_dirs)))
+            else:
+                self.warnings.append('TEMPEST_PLUGINS already defined ({}),'
+                                     'requested value {} ignored'.format(
+                                         tp, self.tempest_plugins))
+
+
     def handle_localconf(self, localconf):
         for phase, phase_data in localconf.items():
             for fn, fn_data in phase_data.items():
@@ -300,6 +318,7 @@ def main():
             path=dict(type='str'),
             projects=dict(type='dict'),
             project=dict(type='dict'),
+            tempest_plugins=dict(type='list'),
         )
     )
 
@@ -311,10 +330,11 @@ def main():
                    p.get('plugins'),
                    p.get('base_dir'),
                    p.get('projects'),
-                   p.get('project'))
+                   p.get('project'),
+                   p.get('tempest_plugins'))
     lc.write(p['path'])
 
-    module.exit_json()
+    module.exit_json(warnings=lc.warnings)
 
 
 try:

roles/write-devstack-local-conf/library/test.py

@@ -23,6 +23,20 @@
 from collections import OrderedDict
 
 class TestDevstackLocalConf(unittest.TestCase):
+
+    @staticmethod
+    def _init_localconf(p):
+        lc = LocalConf(p.get('localrc'),
+                       p.get('local_conf'),
+                       p.get('base_services'),
+                       p.get('services'),
+                       p.get('plugins'),
+                       p.get('base_dir'),
+                       p.get('projects'),
+                       p.get('project'),
+                       p.get('tempest_plugins'))
+        return lc
+
     def setUp(self):
         self.tmpdir = tempfile.mkdtemp()
 
@@ -51,14 +65,7 @@ def test_plugins(self):
                  plugins=plugins,
                  base_dir='./test',
                  path=os.path.join(self.tmpdir, 'test.local.conf'))
-        lc = LocalConf(p.get('localrc'),
-                       p.get('local_conf'),
-                       p.get('base_services'),
-                       p.get('services'),
-                       p.get('plugins'),
-                       p.get('base_dir'),
-                       p.get('projects'),
-                       p.get('project'))
+        lc = self._init_localconf(p)
         lc.write(p['path'])
 
         plugins = []
@@ -104,14 +111,7 @@ def test_plugin_deps(self):
                  plugins=plugins,
                  base_dir=self.tmpdir,
                  path=os.path.join(self.tmpdir, 'test.local.conf'))
-        lc = LocalConf(p.get('localrc'),
-                       p.get('local_conf'),
-                       p.get('base_services'),
-                       p.get('services'),
-                       p.get('plugins'),
-                       p.get('base_dir'),
-                       p.get('projects'),
-                       p.get('project'))
+        lc = self._init_localconf(p)
         lc.write(p['path'])
 
         plugins = []
@@ -145,14 +145,7 @@ def test_libs_from_git(self):
                  path=os.path.join(self.tmpdir, 'test.local.conf'),
                  projects=projects,
                  project=project)
-        lc = LocalConf(p.get('localrc'),
-                       p.get('local_conf'),
-                       p.get('base_services'),
-                       p.get('services'),
-                       p.get('plugins'),
-                       p.get('base_dir'),
-                       p.get('projects'),
-                       p.get('project'))
+        lc = self._init_localconf(p)
         lc.write(p['path'])
 
         lfg = None
@@ -184,14 +177,7 @@ def test_overridelibs_from_git(self):
                  base_dir='./test',
                  path=os.path.join(self.tmpdir, 'test.local.conf'),
                  projects=projects)
-        lc = LocalConf(p.get('localrc'),
-                       p.get('local_conf'),
-                       p.get('base_services'),
-                       p.get('services'),
-                       p.get('plugins'),
-                       p.get('base_dir'),
-                       p.get('projects'),
-                       p.get('project'))
+        lc = self._init_localconf(p)
         lc.write(p['path'])
 
         lfg = None
@@ -238,14 +224,50 @@ def test_plugin_circular_deps(self):
                  base_dir=self.tmpdir,
                  path=os.path.join(self.tmpdir, 'test.local.conf'))
         with self.assertRaises(Exception):
-            lc = LocalConf(p.get('localrc'),
-                           p.get('local_conf'),
-                           p.get('base_services'),
-                           p.get('services'),
-                           p.get('plugins'),
-                           p.get('base_dir'))
+            lc = self._init_localconf(p)
             lc.write(p['path'])
 
+    def _find_tempest_plugins_value(self, file_path):
+        tp = None
+        with open(file_path) as f:
+            for line in f:
+                if line.startswith('TEMPEST_PLUGINS'):
+                    found = line.strip().split('=')[1]
+                    self.assertIsNone(tp,
+                        "TEMPEST_PLUGIN ({}) found again ({})".format(
+                            tp, found))
+                    tp = found
+        return tp
+
+    def test_tempest_plugins(self):
+        "Test that TEMPEST_PLUGINS is correctly populated."
+        p = dict(base_services=[],
+                 base_dir='./test',
+                 path=os.path.join(self.tmpdir, 'test.local.conf'),
+                 tempest_plugins=['heat-tempest-plugin', 'sahara-tests'])
+        lc = self._init_localconf(p)
+        lc.write(p['path'])
+
+        tp = self._find_tempest_plugins_value(p['path'])
+        self.assertEqual('"./test/heat-tempest-plugin ./test/sahara-tests"', tp)
+        self.assertEqual(len(lc.warnings), 0)
+
+    def test_tempest_plugins_not_overridden(self):
+        """Test that the existing value of TEMPEST_PLUGINS is not overridden
+        by the user-provided value, but a warning is emitted."""
+        localrc = {'TEMPEST_PLUGINS': 'someplugin'}
+        p = dict(localrc=localrc,
+                 base_services=[],
+                 base_dir='./test',
+                 path=os.path.join(self.tmpdir, 'test.local.conf'),
+                 tempest_plugins=['heat-tempest-plugin', 'sahara-tests'])
+        lc = self._init_localconf(p)
+        lc.write(p['path'])
+
+        tp = self._find_tempest_plugins_value(p['path'])
+        self.assertEqual('someplugin', tp)
+        self.assertEqual(len(lc.warnings), 1)
+
 
 if __name__ == '__main__':
     unittest.main()

roles/write-devstack-local-conf/tasks/main.yaml

@@ -10,4 +10,5 @@
     local_conf: "{{ devstack_local_conf|default(omit) }}"
     base_dir: "{{ devstack_base_dir|default(omit) }}"
     projects: "{{ zuul.projects }}"
-    project: "{{ zuul.project }}"
+    project: "{{ zuul.project }}"
+    tempest_plugins: "{{ tempest_plugins|default(omit) }}"