Add runbook config and storage management

Configs are split in two: reader and writer for ro/wo api services
respectively. main-reader/main-writer entry point files are added.

Author: teferi

etc/sample_config.json

@@ -0,0 +1,18 @@
+{
+    "flask": {
+        "PORT": 5000,
+        "HOST": "0.0.0.0",
+        "DEBUG": true
+    },
+    "backend": {
+        "type": "elastic",
+        "connection": [{"host": "127.0.0.1", "port": 9200}]
+    },
+    "regions": [
+        "region_one",
+        "region_two"
+    ],
+    "logging": {
+        "level": "INFO"
+    }
+}

requirements.txt

@@ -1,4 +1,5 @@
-gunicorn==19.6.0
 Flask==0.11.1
-
-flask-helpers
+elasticsearch==5.0.1
+flask-helpers==0.1
+gunicorn==19.6.0
+jsonschema>=2.0.0,!=2.5.0,<3.0.0                       # MIT

runbook/config.py

@@ -16,30 +16,119 @@
 import json
 import logging
 import os
+import sys
 
+import jsonschema
 
-config = None
 
+CONF = {
+    "reader": None,
+    "writer": None
+}
 
-def get_config():
+DEFAULT_CONF = {
+    "reader": {
+        "flask": {
+            "HOST": "0.0.0.0",
+            "PORT": 5000,
+            "DEBUG": False
+        },
+        "backend": {
+            "type": "elastic",
+            "connection": [{"host": "127.0.0.1", "port": 9200}]
+        },
+        "regions": [],
+    },
+    "writer": {
+        "flask": {
+            "HOST": "0.0.0.0",
+            "PORT": 5001,
+            "DEBUG": False
+        },
+        "backend": {
+            "type": "elastic",
+            "connection": [{"host": "127.0.0.1", "port": 9200}]
+        },
+        "regions": [],
+    }
+}
+
+
+CONF_SCHEMA = {
+    "type": "object",
+    "$schema": "http://json-schema.org/draft-04/schema",
+    "properties": {
+        "flask": {
+            "type": "object",
+            "properties": {
+                "PORT": {"type": "integer"},
+                "HOST": {"type": "string"},
+                "DEBUG": {"type": "boolean"}
+            },
+        },
+        "backend": {
+            "type": "object",
+            "properties": {
+                "type": {"type": "string"},
+                "connection": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "host": {"type": "string"},
+                            "port": {"type": "integer"}
+                        },
+                        "required": ["host"]
+                    },
+                    "minItems": 1
+                },
+            },
+            "required": ["type", "connection"]
+        },
+        "regions": {
+            "type": "array",
+            "items": {
+                "type": "string",
+            },
+        },
+        "logging": {
+            "type": "object",
+            "properties": {
+                "level": {"type": "string"}
+            }
+        }
+    },
+    "additionalProperties": False
+}
+
+
+def get_config(api_type=None):
     """Return cached configuration.
 
     :returns: application config
     :rtype: dict
     """
-    global config
-    if not config:
-        path = os.environ.get("RUNBOOK_CONF", "/etc/runbook/config.json")
+    if api_type not in ["reader", "writer"]:
+        raise RuntimeError("Unknown api type '{}'".format(api_type))
+
+    global CONF
+    if not CONF[api_type]:
+        path = os.environ.get(
+            "RUNBOOK_{}_CONF".format(api_type.upper()),
+            "/etc/runbook/{}-config.json".format(api_type))
         try:
-            config = json.load(open(path))
+            CONF[api_type] = json.load(open(path))
             logging.info("Config is '%s'" % path)
         except IOError as e:
             logging.warning("Config at '%s': %s" % (path, e))
-            config = {
-                "flask": {
-                    "HOST": "0.0.0.0",
-                    "PORT": 5000,
-                    "DEBUG": False
-                }
-            }
-    return config
+            CONF[api_type] = DEFAULT_CONF[api_type]
+    try:
+        jsonschema.validate(CONF[api_type], CONF_SCHEMA)
+    except jsonschema.ValidationError as e:
+        logging.error(e.message)
+        sys.exit(1)
+    except jsonschema.SchemaError as e:
+        logging.error(e)
+        sys.exit(1)
+    else:
+        return CONF[api_type]

runbook/main.py runbook/main-reader.pyrunbook/main.py renamed to runbook/main-reader.py

@@ -13,6 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import logging
 
 import flask
 from flask_helpers import routing
@@ -21,8 +22,11 @@
 from runbook import config
 
 
+CONF = config.get_config("reader")
+APP_CONF = CONF.get("flask", {})
+
 app = flask.Flask(__name__, static_folder=None)
-app.config.update(config.get_config()["flask"])
+app.config.update(APP_CONF)
 
 
 @app.route("/api/")
@@ -32,9 +36,16 @@ def versions():
 
 @app.errorhandler(404)
 def not_found(error):
+    logging.error(error)
     return flask.jsonify({"error": "Not Found"}), 404
 
 
+@app.errorhandler(500)
+def handle_500(error):
+    logging.error(error)
+    return flask.jsonify({"error": "Internal Server Error"}), 500
+
+
 for bp in [runbook_api]:
     for url_prefix, blueprint in bp.get_blueprints():
         app.register_blueprint(blueprint, url_prefix="/api/v1%s" % url_prefix)
@@ -44,7 +55,8 @@ def not_found(error):
 
 
 def main():
-    app.run()
+    app.run(host=APP_CONF.get("HOST", "0.0.0.0"),
+            port=APP_CONF.get("PORT", "5000"))
 
 
 if __name__ == "__main__":

runbook/main-writer.py

@@ -0,0 +1,63 @@
+# Copyright 2016: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+
+import flask
+from flask_helpers import routing
+
+from runbook.api.v1 import runbook as runbook_api
+from runbook import config
+
+
+CONF = config.get_config("writer")
+APP_CONF = CONF.get("flask", {})
+
+app = flask.Flask(__name__, static_folder=None)
+app.config.update(APP_CONF)
+
+
+@app.route("/api/")
+def versions():
+    return flask.jsonify({"versions": ["1.0"]})
+
+
+@app.errorhandler(404)
+def not_found(error):
+    logging.error(error)
+    return flask.jsonify({"error": "Not Found"}), 404
+
+
+@app.errorhandler(500)
+def handle_500(error):
+    logging.error(error)
+    return flask.jsonify({"error": "Internal Server Error"}), 500
+
+
+for bp in [runbook_api]:
+    for url_prefix, blueprint in bp.get_blueprints():
+        app.register_blueprint(blueprint, url_prefix="/api/v1%s" % url_prefix)
+
+
+app = routing.add_routing_map(app, html_uri=None, json_uri="/")
+
+
+def main():
+    app.run(host=APP_CONF.get("HOST", "0.0.0.0"),
+            port=APP_CONF.get("PORT", "5001"))
+
+
+if __name__ == "__main__":
+    main()

runbook/storage.py

@@ -0,0 +1,84 @@
+# Copyright 2016: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+import logging
+
+import elasticsearch
+
+from runbook import config
+
+
+LOG = logging.getLogger("storage")
+
+ES_MAPPINGS = {
+    "mappings": {
+        "runbook": {
+            "_all": {"enabled": False},
+            "properties": {
+                "runbook": {"type": "binary"},
+                "description": {"type": "text"},
+                "name": {"type": "keyword"},
+                "type": {"type": "keyword"},
+            }
+        },
+        "run": {
+            "_all": {"enabled": False},
+            "properties": {
+                "date": {"type": "date"},
+                "user": {"type": "string"},
+                "output": {"type": "binary"},
+                "return_code": {"type": "integer"},
+                "status": {"type": "keyword"},
+            }
+        }
+    }
+}
+
+ES_CLIENT = None
+
+
+def get_elasticsearch(api_type):
+    """Configures or returns already configured ES client."""
+    global ES_CLIENT
+    if not ES_CLIENT:
+        nodes = config.get_config(api_type)["backend"]["connection"]
+        ES_CLIENT = elasticsearch.Elasticsearch(nodes)
+    return ES_CLIENT
+
+
+def ensure_index(index, api_type):
+    """Esures index exists in es."""
+    es = get_elasticsearch(api_type)
+
+    try:
+        if not es.indices.exists(index):
+            mapping = json.dumps(ES_MAPPINGS)
+            LOG.info("Creating Elasticsearch index: {}".format(index))
+            es.indices.create(index, body=mapping)
+    except elasticsearch.exceptions.ElasticsearchException:
+        LOG.exception("Was unable to get or create index: {}".format(index))
+        raise
+
+
+def configure_regions(api_type="writer"):
+    """Ensure there are indices for every region we have configured."""
+
+    for region in config.get_config(api_type)["regions"]:
+        ensure_index("ms_{}_{}".format("runbooks", region), api_type)
+
+
+if __name__ == "__main__":
+    configure_regions()