From 3126644d98bb0c680c370d51125cb98801cb83d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Mon, 2 Apr 2018 01:31:04 +0200 Subject: [PATCH 01/10] started working on an import API endpoint --- aw_server/api.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/aw_server/api.py b/aw_server/api.py index 997eb9b0..dfc4293e 100644 --- a/aw_server/api.py +++ b/aw_server/api.py @@ -69,11 +69,25 @@ def export_bucket(self, bucket_id: str) -> Dict[str, Any]: del event["id"] return bucket - def create_bucket(self, bucket_id: str, event_type: str, client: str, hostname: str) -> bool: + def import_bucket(self, bucket_data: Dict[str, Any]): + bucket_id = bucket_data["id"] + self.db.create_bucket( + bucket_id, + type=bucket_data["event_type"], + client=bucket_data["client"], + hostname=bucket_data["hostname"], + created=bucket_data["created"] + ) + self.create_events(bucket_id, bucket_data["events"]) + + def create_bucket(self, bucket_id: str, event_type: str, client: str, + hostname: str, created: Optional[datetime] = None)-> bool: """ Create bucket. Returns True if successful, otherwise false if a bucket with the given ID already existed. """ + if created is None: + created = datetime.now() if bucket_id in self.db.buckets(): return False self.db.create_bucket( @@ -81,7 +95,7 @@ def create_bucket(self, bucket_id: str, event_type: str, client: str, hostname: type=event_type, client=client, hostname=hostname, - created=datetime.now() + created=created ) return True From ba4f83306d4ad5cb50388b096a47e397b7b944f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Thu, 30 Aug 2018 13:34:29 +0200 Subject: [PATCH 02/10] implemented (untested) import REST endpoint, fixed API spec --- aw_server/api.py | 8 +++++++- aw_server/rest.py | 36 +++++++++++++++++++++++------------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/aw_server/api.py b/aw_server/api.py index a140f585..cc0a91c5 100644 --- a/aw_server/api.py +++ b/aw_server/api.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Any, Optional +from typing import Dict, List, Any, Optional, Union from datetime import datetime from socket import gethostname import functools @@ -75,6 +75,8 @@ def export_bucket(self, bucket_id: str) -> Dict[str, Any]: def import_bucket(self, bucket_data: Dict[str, Any]): bucket_id = bucket_data["id"] + logger.info(f"Importing bucket {bucket_id}") + # TODO: Check that bucket doesn't already exist self.db.create_bucket( bucket_id, type=bucket_data["event_type"], @@ -84,6 +86,10 @@ def import_bucket(self, bucket_data: Dict[str, Any]): ) self.create_events(bucket_id, bucket_data["events"]) + def import_all(self, data: Dict[str, Any]): + for bucket in data["buckets"]: + self.import_bucket(bucket) + def create_bucket(self, bucket_id: str, event_type: str, client: str, hostname: str, created: Optional[datetime] = None)-> bool: """ diff --git a/aw_server/rest.py b/aw_server/rest.py index ff925a73..a71cef98 100644 --- a/aw_server/rest.py +++ b/aw_server/rest.py @@ -26,6 +26,7 @@ blueprint = Blueprint('api', __name__, url_prefix='/api') api = Api(blueprint, doc='/') + # TODO: Clean up JSONEncoder code? class CustomJSONEncoder(json.JSONEncoder): def __init__(self, *args, **kwargs): @@ -40,6 +41,8 @@ def default(self, obj, *args, **kwargs): except TypeError: pass return json.JSONEncoder.default(self, obj) + + app.json_encoder = CustomJSONEncoder app.register_blueprint(blueprint) @@ -54,26 +57,20 @@ def format(self, value): return json.loads(value) -# Loads event schema from JSONSchema in aw_core +# Loads event and bucket schema from JSONSchema in aw_core event = api.schema_model('Event', schema.get_json_schema("event")) +bucket = api.schema_model('Bucket', schema.get_json_schema("bucket")) +export_full = api.schema_model('Export', schema.get_json_schema("export")) # TODO: Construct all the models from JSONSchema? # A downside to contructing from JSONSchema: flask-restplus does not have marshalling support + info = api.model('Info', { 'hostname': fields.String(), 'version': fields.String(), 'testing': fields.Boolean(), }) -bucket = api.model('Bucket', { - 'id': fields.String(required=True, description='The buckets unique id'), - 'name': fields.String(required=False, description='The buckets readable and renameable name'), - 'type': fields.String(required=True, description='The buckets event type'), - 'client': fields.String(required=True, description='The name of the watcher client'), - 'hostname': fields.String(required=True, description='The hostname of the client that the bucket belongs to'), - 'created': fields.DateTime(required=True, description='The creation datetime of the bucket'), -}) - create_bucket = api.model('CreateBucket', { 'client': fields.String(required=True), 'type': fields.String(required=True), @@ -81,10 +78,11 @@ def format(self, value): }) query = api.model('Query', { - 'timeperiods': fields.List(fields.String, required=True, description='List of periods to query'), + 'timeperiods': fields.List(fields.String, required=True, description='List of periods to query'), 'query': fields.List(fields.String, required=True, description='String list of query statements'), }) + def copy_doc(api_method): """Decorator that copies another functions docstring to the decorated function. Used to copy the docstrings in ServerAPI over to the flask-restplus Resources. @@ -117,7 +115,7 @@ def get(self) -> Dict[str, Dict]: @api.route("/0/buckets/") class BucketResource(Resource): - @api.marshal_with(bucket) + @api.doc(model=bucket) @copy_doc(ServerAPI.get_bucket_metadata) def get(self, bucket_id): return app.api.get_bucket_metadata(bucket_id) @@ -254,18 +252,30 @@ def post(self): return {"type": type(qe).__name__, "message": str(qe)}, 400 -# EXPORTING +# EXPORT AND IMPORT + @api.route("/0/export") class ExportAllResource(Resource): + @api.doc(model=export_full) @copy_doc(ServerAPI.export_all) def get(self): return app.api.export_all(), 200 +@api.route("/0/import") +class ImportAllResource(Resource): + @api.expect(export_full) + @copy_doc(ServerAPI.import_all) + def post(self): + data = request.get_json() + return app.api.import_all(data), 200 + + # TODO: Perhaps we don't need this, could be done with a query argument to /0/export instead @api.route("/0/buckets//export") class BucketExportResource(Resource): + @api.doc(model=bucket) @copy_doc(ServerAPI.export_bucket) def get(self, bucket_id): return app.api.export_bucket(bucket_id) From d1198a9a70ef26782c0c9434176c622a1927c67e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Mon, 10 Sep 2018 13:05:06 +0200 Subject: [PATCH 03/10] fixed unsupported use of f-string --- aw_server/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aw_server/api.py b/aw_server/api.py index cc0a91c5..df840d9a 100644 --- a/aw_server/api.py +++ b/aw_server/api.py @@ -33,7 +33,7 @@ class ServerAPI: def __init__(self, db, testing) -> None: self.db = db self.testing = testing - self.last_event = {} #type: dict + self.last_event = {} # type: dict def get_info(self) -> Dict[str, Dict]: """Get server info""" @@ -75,7 +75,7 @@ def export_bucket(self, bucket_id: str) -> Dict[str, Any]: def import_bucket(self, bucket_data: Dict[str, Any]): bucket_id = bucket_data["id"] - logger.info(f"Importing bucket {bucket_id}") + logger.info("Importing bucket {}".format(bucket_id)) # TODO: Check that bucket doesn't already exist self.db.create_bucket( bucket_id, From f348488a2403c36c256ce29f1158f77de4ebcae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Mon, 10 Sep 2018 20:39:19 +0200 Subject: [PATCH 04/10] fixed bug in import --- aw_server/api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/aw_server/api.py b/aw_server/api.py index df840d9a..e70bc3a5 100644 --- a/aw_server/api.py +++ b/aw_server/api.py @@ -82,9 +82,11 @@ def import_bucket(self, bucket_data: Dict[str, Any]): type=bucket_data["event_type"], client=bucket_data["client"], hostname=bucket_data["hostname"], - created=bucket_data["created"] + created=(bucket_data["created"] + if isinstance(bucket_data["created"], datetime) + else iso8601.parse_date(bucket_data["created"])), ) - self.create_events(bucket_id, bucket_data["events"]) + self.create_events(bucket_id, [Event(**e) if isinstance(e, dict) else e for e in bucket_data["events"]]) def import_all(self, data: Dict[str, Any]): for bucket in data["buckets"]: From f98ed55415776974d101b1cdfdcaa8233a1be7d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bj=C3=A4reholt?= Date: Tue, 26 Feb 2019 21:37:20 +0100 Subject: [PATCH 05/10] Made import/export data consistent as defined in aw-core schema --- aw_server/api.py | 23 ++++++++++++----------- aw_server/rest.py | 29 +++++++++++++++-------------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/aw_server/api.py b/aw_server/api.py index e70bc3a5..bcfdbf69 100644 --- a/aw_server/api.py +++ b/aw_server/api.py @@ -73,13 +73,21 @@ def export_bucket(self, bucket_id: str) -> Dict[str, Any]: del event["id"] return bucket - def import_bucket(self, bucket_data: Dict[str, Any]): + def export_all(self) -> Dict[str, dict]: + """Exports all buckets and their events to a format consistent across versions""" + buckets = self.get_buckets() + exported_buckets = [] + for bid in buckets.keys(): + exported_buckets.append(self.export_bucket(bid)) + return exported_buckets + + def import_bucket(self, bucket_data: Any): bucket_id = bucket_data["id"] logger.info("Importing bucket {}".format(bucket_id)) # TODO: Check that bucket doesn't already exist self.db.create_bucket( bucket_id, - type=bucket_data["event_type"], + type=bucket_data["type"], client=bucket_data["client"], hostname=bucket_data["hostname"], created=(bucket_data["created"] @@ -88,8 +96,8 @@ def import_bucket(self, bucket_data: Dict[str, Any]): ) self.create_events(bucket_id, [Event(**e) if isinstance(e, dict) else e for e in bucket_data["events"]]) - def import_all(self, data: Dict[str, Any]): - for bucket in data["buckets"]: + def import_all(self, buckets: List[Any]): + for bucket in buckets: self.import_bucket(bucket) def create_bucket(self, bucket_id: str, event_type: str, client: str, @@ -229,10 +237,3 @@ def get_log(self): for line in log_file.readlines()[::-1]: payload.append(json.loads(line)) return payload, 200 - - def export_all(self) -> Dict[str, dict]: - """Exports all buckets and their events to a format consistent across versions""" - buckets = self.get_buckets() - for bid in buckets.keys(): - buckets[bid] = self.export_bucket(bid) - return buckets diff --git a/aw_server/rest.py b/aw_server/rest.py index a71cef98..287055e1 100644 --- a/aw_server/rest.py +++ b/aw_server/rest.py @@ -60,7 +60,7 @@ def format(self, value): # Loads event and bucket schema from JSONSchema in aw_core event = api.schema_model('Event', schema.get_json_schema("event")) bucket = api.schema_model('Bucket', schema.get_json_schema("bucket")) -export_full = api.schema_model('Export', schema.get_json_schema("export")) +buckets_export = api.schema_model('Export', schema.get_json_schema("export")) # TODO: Construct all the models from JSONSchema? # A downside to contructing from JSONSchema: flask-restplus does not have marshalling support @@ -257,28 +257,29 @@ def post(self): @api.route("/0/export") class ExportAllResource(Resource): - @api.doc(model=export_full) + @api.doc(model=buckets_export) @copy_doc(ServerAPI.export_all) def get(self): - return app.api.export_all(), 200 - - -@api.route("/0/import") -class ImportAllResource(Resource): - @api.expect(export_full) - @copy_doc(ServerAPI.import_all) - def post(self): - data = request.get_json() - return app.api.import_all(data), 200 + return {"buckets": app.api.export_all()}, 200 # TODO: Perhaps we don't need this, could be done with a query argument to /0/export instead @api.route("/0/buckets//export") class BucketExportResource(Resource): - @api.doc(model=bucket) + @api.doc(model=buckets_export) @copy_doc(ServerAPI.export_bucket) def get(self, bucket_id): - return app.api.export_bucket(bucket_id) + return {"buckets": [app.api.export_bucket(bucket_id)]}, 200 + + +@api.route("/0/import") +class ImportAllResource(Resource): + @api.expect(buckets_export) + @copy_doc(ServerAPI.import_all) + def post(self): + data = request.get_json() + buckets = data["buckets"] + return app.api.import_all(buckets), 200 # LOGGING From d7bd78105d1e13f27eb7f68f1440ab05549daab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bj=C3=A4reholt?= Date: Tue, 26 Feb 2019 21:46:24 +0100 Subject: [PATCH 06/10] Fix mypy typecheck --- aw_server/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aw_server/api.py b/aw_server/api.py index bcfdbf69..1ad5d200 100644 --- a/aw_server/api.py +++ b/aw_server/api.py @@ -73,7 +73,7 @@ def export_bucket(self, bucket_id: str) -> Dict[str, Any]: del event["id"] return bucket - def export_all(self) -> Dict[str, dict]: + def export_all(self) -> List[Any]: """Exports all buckets and their events to a format consistent across versions""" buckets = self.get_buckets() exported_buckets = [] From 09519eefa102c7ab06465f8a0d8ae29e9ac39977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bj=C3=A4reholt?= Date: Wed, 27 Feb 2019 20:05:59 +0100 Subject: [PATCH 07/10] Made exports/import consistents, no longer following aw-core schema --- aw_server/api.py | 6 +++--- aw_server/rest.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/aw_server/api.py b/aw_server/api.py index 1ad5d200..692cacaa 100644 --- a/aw_server/api.py +++ b/aw_server/api.py @@ -76,9 +76,9 @@ def export_bucket(self, bucket_id: str) -> Dict[str, Any]: def export_all(self) -> List[Any]: """Exports all buckets and their events to a format consistent across versions""" buckets = self.get_buckets() - exported_buckets = [] + exported_buckets = {} for bid in buckets.keys(): - exported_buckets.append(self.export_bucket(bid)) + exported_buckets[bid] = self.export_bucket(bid) return exported_buckets def import_bucket(self, bucket_data: Any): @@ -97,7 +97,7 @@ def import_bucket(self, bucket_data: Any): self.create_events(bucket_id, [Event(**e) if isinstance(e, dict) else e for e in bucket_data["events"]]) def import_all(self, buckets: List[Any]): - for bucket in buckets: + for bid, bucket in buckets.items(): self.import_bucket(bucket) def create_bucket(self, bucket_id: str, event_type: str, client: str, diff --git a/aw_server/rest.py b/aw_server/rest.py index 287055e1..dae6cbc1 100644 --- a/aw_server/rest.py +++ b/aw_server/rest.py @@ -269,7 +269,8 @@ class BucketExportResource(Resource): @api.doc(model=buckets_export) @copy_doc(ServerAPI.export_bucket) def get(self, bucket_id): - return {"buckets": [app.api.export_bucket(bucket_id)]}, 200 + bucket_export = app.api.export_bucket(bucket_id) + return {"buckets": {bucket_export["id"]: bucket_export}}, 200 @api.route("/0/import") From a42dc6a14e9b50117a5f7f0313b632f0332c09cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bj=C3=A4reholt?= Date: Wed, 27 Feb 2019 20:07:22 +0100 Subject: [PATCH 08/10] Fixed mypy issues --- aw_server/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aw_server/api.py b/aw_server/api.py index 692cacaa..2711be8f 100644 --- a/aw_server/api.py +++ b/aw_server/api.py @@ -73,7 +73,7 @@ def export_bucket(self, bucket_id: str) -> Dict[str, Any]: del event["id"] return bucket - def export_all(self) -> List[Any]: + def export_all(self) -> Dict[str, Any]: """Exports all buckets and their events to a format consistent across versions""" buckets = self.get_buckets() exported_buckets = {} @@ -96,7 +96,7 @@ def import_bucket(self, bucket_data: Any): ) self.create_events(bucket_id, [Event(**e) if isinstance(e, dict) else e for e in bucket_data["events"]]) - def import_all(self, buckets: List[Any]): + def import_all(self, buckets: Dict[str, Any]): for bid, bucket in buckets.items(): self.import_bucket(bucket) From e35fda282af220a48e521e325546caf580b35dbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bj=C3=A4reholt?= Date: Wed, 27 Feb 2019 21:32:10 +0100 Subject: [PATCH 09/10] Allow bucket import from browser forms --- aw_server/rest.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/aw_server/rest.py b/aw_server/rest.py index dae6cbc1..31febe17 100644 --- a/aw_server/rest.py +++ b/aw_server/rest.py @@ -278,9 +278,18 @@ class ImportAllResource(Resource): @api.expect(buckets_export) @copy_doc(ServerAPI.import_all) def post(self): - data = request.get_json() - buckets = data["buckets"] - return app.api.import_all(buckets), 200 + # If import comes from a form in th web-ui + if len(request.files) > 0: + # web-ui form only allows one file, but technically it's possible to + # upload multiple files at the same time + for filename, f in request.files.items(): + buckets = json.loads(f.stream.read())["buckets"] + app.api.import_all(buckets) + # Normal import from body + else: + buckets = request.get_json()["buckets"] + app.api.import_all(buckets) + return None, 200 # LOGGING From daa9e0b672eb0999874463360cd10c81163623af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bj=C3=A4reholt?= Date: Thu, 28 Feb 2019 18:44:59 +0100 Subject: [PATCH 10/10] Add HTTP header to export so it's downloaded as file from browsers --- aw_server/rest.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/aw_server/rest.py b/aw_server/rest.py index 31febe17..f558c819 100644 --- a/aw_server/rest.py +++ b/aw_server/rest.py @@ -2,7 +2,7 @@ import traceback import json -from flask import request, Blueprint, jsonify +from flask import request, Blueprint, jsonify, make_response from flask_restplus import Api, Resource, fields import iso8601 from datetime import datetime, timedelta @@ -260,7 +260,12 @@ class ExportAllResource(Resource): @api.doc(model=buckets_export) @copy_doc(ServerAPI.export_all) def get(self): - return {"buckets": app.api.export_all()}, 200 + buckets_export = app.api.export_all() + payload = {"buckets": buckets_export} + response = make_response(json.dumps(payload)) + filename = "aw-buckets-export.json" + response.headers["Content-Disposition"] = "attachment; filename={}".format(filename) + return response # TODO: Perhaps we don't need this, could be done with a query argument to /0/export instead @@ -270,7 +275,11 @@ class BucketExportResource(Resource): @copy_doc(ServerAPI.export_bucket) def get(self, bucket_id): bucket_export = app.api.export_bucket(bucket_id) - return {"buckets": {bucket_export["id"]: bucket_export}}, 200 + payload = {"buckets": {bucket_export["id"]: bucket_export}} + response = make_response(json.dumps(payload)) + filename = "aw-bucket-export_{}.json".format(bucket_export["id"]) + response.headers["Content-Disposition"] = "attachment; filename={}".format(filename) + return response @api.route("/0/import")