Add reader API for runbooks

requirements.txt

@@ -3,3 +3,4 @@ elasticsearch==5.0.1
 flask-helpers==0.1
 gunicorn==19.6.0
 jsonschema>=2.0.0,!=2.5.0,<3.0.0                       # MIT
+six==1.10.0

runbook/api/utils.py

@@ -0,0 +1,79 @@
+# Copyright 2016: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import functools
+
+import flask
+import six
+
+from runbook import config
+
+
+def check_regions(api_type):
+    def check_decorator(func):
+        regions = set(config.get_config(api_type)["regions"])
+
+        @functools.wraps(func)
+        def checker(region=None, *args, **kwargs):
+
+            if region is not None and region not in regions:
+                return flask.jsonify(
+                    {"error": "Region {} Not Found".format(region)}), 404
+            return func(region, *args, **kwargs)
+
+        return checker
+    return check_decorator
+
+
+def convert_run(hit):
+    body = {k: v for k, v in hit["_source"].items()}
+
+    if "_index" in hit:
+        body["region_id"] = hit["_index"][len("ms_runbooks_"):]
+
+    runbook = None
+    if 'inner_hits' in hit:
+        inner_hit = hit['inner_hits']['parent']['hits']['hits']
+        if inner_hit:
+            runbook = convert_runbook(inner_hit[0])
+
+    body["runbook"] = runbook
+
+    body["id"] = hit["_id"]
+    return body
+
+
+def convert_runbook(hit):
+    body = {k: v for k, v in hit["_source"].items()}
+    body.setdefault("tags", [])
+
+    # convert any single tag to list of tags
+    if isinstance(body["tags"], six.string_types):
+        body["tags"] = [body["tags"]]
+
+    if "_index" in hit:
+        body["region_id"] = hit["_index"][len("ms_runbooks_"):]
+
+    latest_run = None
+    if 'inner_hits' in hit:
+        inner_hit = hit['inner_hits']['latest']['hits']['hits']
+        if inner_hit:
+            latest_run = convert_run(inner_hit[0])
+
+    body["latest_run"] = latest_run
+
+    body["id"] = hit["_id"]
+    return body

runbook/api/v1/reader/run.py

@@ -0,0 +1,97 @@
+# Copyright 2016: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+
+import flask
+
+from runbook.api import utils
+from runbook import storage
+
+API_TYPE = "reader"
+bp = flask.Blueprint("runs", __name__)
+
+
+def get_blueprints():
+    return [["", bp]]
+
+
+INNER_QUERY_WITH_PARENT = {
+    "has_parent": {
+        "inner_hits": {
+            "name": "parent",
+            "size": 1,
+        },
+        "type": "runbook",
+        "query": {
+            "match_all": {}
+        }
+    }
+}
+
+RUNS_QUERY = {
+    "query": {
+        "bool": {
+            "should": INNER_QUERY_WITH_PARENT,
+        }
+    }
+}
+
+
+@bp.route("/region/<region>/runbook_runs", methods=["GET"])
+@bp.route("/runbook_runs", methods=["GET"])
+@utils.check_regions(API_TYPE)
+def handle_runs(region=None):
+    es = storage.get_elasticsearch(API_TYPE)
+
+    if region is None:
+        region = "*"
+    index_name = "ms_runbooks_{}".format(region)
+
+    query = copy.deepcopy(RUNS_QUERY)
+
+    runbook_id = flask.request.args.get('runbook_id', '')
+    if runbook_id:
+        query["query"]["bool"]["should"]["has_parent"]["query"] = {
+            "ids": {
+                "values": runbook_id,
+                "type": "runbook",
+            }
+        }
+
+    result = es.search(index=index_name,
+                       doc_type="run",
+                       body=query)
+    hit_list = [utils.convert_run(hit) for hit in result['hits']['hits']]
+    return flask.jsonify(hit_list)
+
+
+@bp.route("/region/<region>/runbook_runs/<run_id>", methods=["GET"])
+@utils.check_regions(API_TYPE)
+def handle_single_run(region, run_id):
+    es = storage.get_elasticsearch(API_TYPE)
+    index_name = "ms_runbooks_{}".format(region)
+
+    query = copy.deepcopy(RUNS_QUERY)
+    query["query"]["bool"]["must"] = {"ids": {"values": [run_id]}}
+    result = es.search(index=index_name,
+                       doc_type="run",
+                       body=query)
+
+    hit_list = [utils.convert_run(hit) for hit in result['hits']['hits']]
+    if not hit_list:
+        flask.abort(404)
+
+    return flask.jsonify(hit_list[0])

runbook/api/v1/reader/runbook_.py

@@ -0,0 +1,121 @@
+# Copyright 2016: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+
+import flask
+
+from runbook.api import utils
+from runbook import storage
+
+API_TYPE = "reader"
+bp = flask.Blueprint("runbooks", __name__)
+
+
+def get_blueprints():
+    return [["", bp]]
+
+
+INNER_QUERY_NO_RUNS = {
+    "bool": {
+        "must_not": [
+            {
+                "has_child": {
+                    "type": "run",
+                    "query": {
+                        "match_all": {}
+                    }
+                }
+            }
+        ]
+    }
+}
+
+INNER_QUERY_WITH_RUNS = {
+    "has_child": {
+        "inner_hits": {
+            "name": "latest",
+            "size": 1,
+            "sort": [{"created_at": {"order": "desc"}}]
+        },
+        "type": "run",
+        "query": {
+            "match_all": {}
+        }
+    }
+}
+
+RUNBOOK_QUERY = {
+    "query": {
+        "bool": {
+            "should": [
+                INNER_QUERY_NO_RUNS,
+                INNER_QUERY_WITH_RUNS
+            ],
+        }
+    }
+}
+
+
+@bp.route("/runbooks", methods=["GET"])
+@bp.route("/region/<region>/runbooks", methods=["GET"])
+@utils.check_regions(API_TYPE)
+def handle_runbooks(region=None):
+    es = storage.get_elasticsearch(API_TYPE)
+
+    if region is None:
+        region = "*"
+    index_name = "ms_runbooks_{}".format(region)
+
+    tags = flask.request.args.get('tags', '').split(',')
+
+    query = copy.deepcopy(RUNBOOK_QUERY)
+
+    # exclude deleted runbooks
+    query["query"]["bool"]["must_not"] = {"term": {"deleted": True}}
+
+    # filter by tags, combining multiple 'term' queries effectively AND's
+    # supplied tags
+    tag_terms = []
+    for tag in tags:
+        if tag:
+            tag_terms.append({"term": {"tags": tag}})
+    if tag_terms:
+        query["query"]["bool"]["must"] = tag_terms
+
+    result = es.search(index=index_name,
+                       doc_type="runbook",
+                       body=query)
+    hit_list = [utils.convert_runbook(hit) for hit in result['hits']['hits']]
+    return flask.jsonify(hit_list)
+
+
+@bp.route("/region/<region>/runbooks/<book_id>", methods=["GET"])
+@utils.check_regions(API_TYPE)
+def handle_single_runbook(region, book_id):
+    es = storage.get_elasticsearch(API_TYPE)
+    index_name = "ms_runbooks_{}".format(region)
+
+    query = copy.deepcopy(RUNBOOK_QUERY)
+    query["query"]["bool"]["must"] = {"ids": {"values": [book_id]}}
+    result = es.search(index=index_name,
+                       doc_type="runbook",
+                       body=query)
+
+    hit_list = [utils.convert_runbook(hit) for hit in result['hits']['hits']]
+    if not hit_list:
+        flask.abort(404)
+
+    return flask.jsonify(hit_list[0])

runbook/main_reader.py

@@ -18,7 +18,8 @@
 import flask
 from flask_helpers import routing
 
-from runbook.api.v1 import runbook as runbook_api
+from runbook.api.v1.reader import run as run_api
+from runbook.api.v1.reader import runbook_ as runbook_api
 from runbook import config
 
 
@@ -46,7 +47,7 @@ def handle_500(error):
     return flask.jsonify({"error": "Internal Server Error"}), 500
 
 
-for bp in [runbook_api]:
+for bp in [runbook_api, run_api]:
     for url_prefix, blueprint in bp.get_blueprints():
         app.register_blueprint(blueprint, url_prefix="/api/v1%s" % url_prefix)
 

tests/unit/api/v1/base.py

@@ -0,0 +1,63 @@
+# Copyright 2016: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import importlib
+import json
+
+import mock
+import testtools
+
+
+TEST_CONFIG = {
+    "flask": {
+        "PORT": 5000,
+        "HOST": "0.0.0.0",
+        "DEBUG": True
+    },
+    "backend": {
+        "type": "elastic",
+        "connection": [{"host": "127.0.0.1", "port": 9200}]
+    },
+    "regions": [
+        "region_one",
+        "region_two"
+    ]
+}
+
+
+class APITestCase(testtools.TestCase):
+    api_type = None
+
+    def setUp(self):
+        super(APITestCase, self).setUp()
+        self.addCleanup(mock.patch.stopall)
+
+        # NOTE(kzaitsev): mock all get_config for all the tests
+        self.patcher = mock.patch('runbook.config.get_config')
+        self.get_config = self.patcher.start()
+        self.get_config.return_value = TEST_CONFIG
+
+        if self.api_type not in ["reader", "writer"]:
+            raise RuntimeError("Unknown api_type '{}'".format(self.api_type))
+
+        main = importlib.import_module("runbook.main_{}".format(self.api_type))
+        self.client = main.app.test_client()
+        self.app = main.app
+
+    def test_not_found(self):
+        resp = self.client.get('/404')
+        self.assertEqual({"error": "Not Found"},
+                         json.loads(resp.data.decode()))
+        self.assertEqual(404, resp.status_code)