Add runbook docker driver

Docker is now the default driver, subprocess is used to communicate with
the docker itself. runbook_runner image is used as a base image for all
the runs and each run spawns a new image, that is later removed.

Author: teferi

runbook/drivers/base.py

@@ -18,7 +18,7 @@ class Driver(object):
 
     interpreters = {
         "bash": "/bin/bash",
-        "python": "/bin/python",
+        "python": "/usr/bin/python",
     }
 
     @classmethod

runbook/drivers/docker.py

@@ -0,0 +1,120 @@
+# Copyright 2016: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+import os
+import shutil
+import subprocess
+import tempfile
+import uuid
+
+from runbook.drivers import base
+
+LOG = logging.getLogger("runner.docker")
+
+
+RUNNER_DOCKERFILE = """
+FROM ubuntu:14.04
+RUN apt-get update
+RUN apt-get install python -y
+"""
+
+RUN_DOCKERFILE = """
+FROM runbook_runner
+COPY . /runbook
+WORKDIR /runbook
+ENTRYPOINT ["{interpreter}"]
+CMD ["runbook"]
+"""
+
+
+class Driver(base.Driver):
+
+    @classmethod
+    def initialise(cls):
+        dirname = tempfile.mkdtemp()
+        with open(os.path.join(dirname, 'Dockerfile'), "w") as f:
+            f.write(RUNNER_DOCKERFILE)
+        output = subprocess.check_output(
+            ["docker", "build", "-t", "runbook_runner", dirname],
+            stderr=subprocess.STDOUT,)
+        LOG.info("Built 'runbook_runner' image, {}".format(output))
+
+        shutil.rmtree(dirname)
+
+    @classmethod
+    def run(cls, runbook, parameters):
+        LOG.info("Running runbook '{}' with parameters '{}'".format(
+            runbook, parameters))
+
+        interpreter = cls.interpreters.get(runbook.get("type"))
+        if not interpreter:
+            return {
+                "return_code": -1,
+                "output": "Don't know how to run '{}' type runbook".format(
+                    runbook.get("type")),
+            }
+
+        dirname = tempfile.mkdtemp()
+
+        with open(os.path.join(dirname, 'runbook'), "w") as f:
+            f.write(runbook["runbook"])
+
+        with open(os.path.join(dirname, 'Dockerfile'), "w") as f:
+            f.write(RUN_DOCKERFILE.format(
+                interpreter=interpreter))
+
+        run_name = "run-{}".format(uuid.uuid4().hex)
+        output = subprocess.check_output(
+            ["docker", "build", "-t", run_name, dirname],
+            stderr=subprocess.STDOUT,)
+        LOG.info("Built 'runbook_runner' image, {}".format(output))
+
+        returncode = 0
+        try:
+            output = subprocess.check_output(
+                ["docker", "run", "--name", run_name, run_name],
+                stderr=subprocess.STDOUT,
+            )
+        except subprocess.CalledProcessError as e:
+            output = e.output
+            returncode = e.returncode
+
+        # cleanup
+        try:
+            subprocess.check_output(
+                ["docker", "rm", run_name],
+                stderr=subprocess.STDOUT,
+            )
+            LOG.info("Removed container {}".format(run_name))
+        except subprocess.CalledProcessError as e:
+            LOG.warning("Could not remove container {}".format(
+                run_name, e.output))
+
+        try:
+            subprocess.check_output(
+                ["docker", "rmi", run_name],
+                stderr=subprocess.STDOUT,
+            )
+            LOG.info("Removed image {}".format(run_name))
+        except subprocess.CalledProcessError as e:
+            LOG.warning("Could not remove image {}: {}".format(
+                run_name, e.output))
+
+        shutil.rmtree(dirname, ignore_errors=True)
+        return {
+            "return_code": returncode,
+            "output": output,
+        }

runbook/drivers/shell.py

@@ -21,6 +21,8 @@
 
 from runbook.drivers import base
 
+LOG = logging.getLogger("runner.shell")
+
 
 class Driver(base.Driver):
 
@@ -30,7 +32,7 @@ def initialise(cls):
 
     @classmethod
     def run(cls, runbook, parameters):
-        logging.info("Running runbook '{}' with parameters '{}'".format(
+        LOG.info("Running runbook '{}' with parameters '{}'".format(
             runbook, parameters))
         f = tempfile.NamedTemporaryFile()
         f.write(runbook["runbook"])

runbook/runner.py

@@ -32,6 +32,7 @@
 API_TYPE = "runner"
 CONF = config.get_config(API_TYPE)
 POOL = eventlet.GreenPool()
+DRIVER = None
 
 LOG = logging.getLogger("runner")
 LOG.setLevel(logging.INFO)
@@ -97,7 +98,6 @@ def handle_hit(hit, index_name, driver):
     LOG.info("Finished run '{}': '{}'".format(hit['_id'], run_result))
 
     end_status = "finished" if run_result.get("return_code") == 0 else "failed"
-    # end_status = "scheduled"  # FIXME
 
     now = datetime.datetime.now()
     try:
@@ -122,15 +122,7 @@ def handle_hit(hit, index_name, driver):
 
 
 def job():
-    driver_name = CONF.get("driver", "shell")
-    try:
-        driver_module = importlib.import_module(
-            "runbook.drivers." + driver_name)
-    except ImportError:
-        LOG.critical("No driver named '{}'".format(driver_name))
-        return
-
-    driver = driver_module.Driver
+    driver = DRIVER
 
     es = storage.get_elasticsearch(API_TYPE)
     for region in CONF["regions"]:
@@ -146,6 +138,18 @@ def job():
 
 
 def main():
+    driver_name = CONF.get("driver", "docker")
+    try:
+        driver_module = importlib.import_module(
+            "runbook.drivers." + driver_name)
+    except ImportError:
+        LOG.critical("No driver named '{}'".format(driver_name))
+        return
+
+    driver = driver_module.Driver
+    driver.initialise()
+    global DRIVER
+    DRIVER = driver
 
     run_every_seconds = CONF.get("run_every_seconds", 30)
     schedule.every(run_every_seconds).seconds.do(job)