diff --git a/docs/source/audit_logs/commands.rst b/docs/source/audit_logs/commands.rst new file mode 100644 index 00000000..2bc34951 --- /dev/null +++ b/docs/source/audit_logs/commands.rst @@ -0,0 +1,16 @@ +Commands +======== + +If you've registered the ``audit_logs`` app in your ``piccolo_conf.py`` file +(see the :ref:`migrations docs `), it gives you access to a +custom command. + +clean +----- + +If you run the following on the command line, it will delete all logs +from the database. + +.. code-block:: bash + + piccolo audit_logs clean diff --git a/docs/source/audit_logs/tables.rst b/docs/source/audit_logs/tables.rst new file mode 100644 index 00000000..84259d1d --- /dev/null +++ b/docs/source/audit_logs/tables.rst @@ -0,0 +1,63 @@ +Tables +====== + +``audit_logs`` is a ``Piccolo`` app that records changes made by users to database tables. +We store the audit logs in :class:`AuditLog `. + +------------------------------------------------------------------------------- + +.. _AuditLogMigrations: + +Migrations +---------- + +We recommend creating the ``audit_logs`` tables using migrations. + +You can add ``piccolo_api.audit_logs.piccolo_app`` to the ``apps`` arguments +of the :class:`AppRegistry ` in ``piccolo_conf.py``. + +.. code-block:: bash + + APP_REGISTRY = AppRegistry( + apps=[ + ... + "piccolo_api.audit_logs.piccolo_app", + ... + ] + ) + +To learn more about Piccolo apps, see the `Piccolo docs `_. + +To run the migrations and create the table, run: + +.. code-block:: bash + + piccolo migrations forwards audit_logs + +------------------------------------------------------------------------------- + +Creating them manually +---------------------- + +If you prefer not to use migrations, and want to create them manually, you can +do this instead: + +.. code-block:: python + + from piccolo_api.audit_logs.tables import AuditLog + from piccolo.tables import create_db_tables_sync + + create_db_tables_sync(AuditLog, if_not_exists=True) + +------------------------------------------------------------------------------- + +Source +------ + +AuditLog +~~~~~~~~ + +.. currentmodule:: piccolo_api.audit_logs.tables + +.. autoclass:: AuditLog + :members: diff --git a/docs/source/index.rst b/docs/source/index.rst index bd4538f1..46fe6b64 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -40,6 +40,13 @@ ASGI app, covering authentication, security, and more. ./change_password/index ./advanced_auth/index +.. toctree:: + :caption: Audit logs + :maxdepth: 1 + + ./audit_logs/tables.rst + ./audit_logs/commands.rst + .. toctree:: :caption: Piccolo Admin :maxdepth: 1 diff --git a/piccolo_api/audit_logs/__init__.py b/piccolo_api/audit_logs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/piccolo_api/audit_logs/commands.py b/piccolo_api/audit_logs/commands.py new file mode 100644 index 00000000..22a0ac90 --- /dev/null +++ b/piccolo_api/audit_logs/commands.py @@ -0,0 +1,10 @@ +from .tables import AuditLog + + +async def clean(): + """ + Removes all audit logs. + """ + print("Removing audit logs ...") + await AuditLog.delete(force=True).run() + print("Successfully removed audit logs") diff --git a/piccolo_api/audit_logs/piccolo_app.py b/piccolo_api/audit_logs/piccolo_app.py new file mode 100644 index 00000000..8de05477 --- /dev/null +++ b/piccolo_api/audit_logs/piccolo_app.py @@ -0,0 +1,24 @@ +""" +Import all of the Tables subclasses in your app here, and register them with +the APP_CONFIG. +""" + +import os + +from piccolo.conf.apps import AppConfig + +from .commands import clean +from .tables import AuditLog + +CURRENT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) + + +APP_CONFIG = AppConfig( + app_name="audit_logs", + migrations_folder_path=os.path.join( + CURRENT_DIRECTORY, "piccolo_migrations" + ), + table_classes=[AuditLog], + migration_dependencies=[], + commands=[clean], +) diff --git a/piccolo_api/audit_logs/piccolo_migrations/2022-06-28T18-57-05-840638.py b/piccolo_api/audit_logs/piccolo_migrations/2022-06-28T18-57-05-840638.py new file mode 100644 index 00000000..49d37a6b --- /dev/null +++ b/piccolo_api/audit_logs/piccolo_migrations/2022-06-28T18-57-05-840638.py @@ -0,0 +1,150 @@ +from enum import Enum + +from piccolo.apps.migrations.auto.migration_manager import MigrationManager +from piccolo.columns.column_types import JSON, Text, Timestamp, Varchar +from piccolo.columns.defaults.timestamp import TimestampNow +from piccolo.columns.indexes import IndexMethod + +ID = "2022-06-28T18:57:05:840638" +VERSION = "0.80.0" +DESCRIPTION = "" + + +async def forwards(): + manager = MigrationManager( + migration_id=ID, app_name="audit_logs", description=DESCRIPTION + ) + + manager.add_table("AuditLog", tablename="audit_log") + + manager.add_column( + table_class_name="AuditLog", + tablename="audit_log", + column_name="action_time", + db_column_name="action_time", + column_class_name="Timestamp", + column_class=Timestamp, + params={ + "default": TimestampNow(), + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + "db_column_name": None, + "secret": False, + }, + ) + + manager.add_column( + table_class_name="AuditLog", + tablename="audit_log", + column_name="action_type", + db_column_name="action_type", + column_class_name="Varchar", + column_class=Varchar, + params={ + "length": 255, + "default": "", + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": Enum( + "ActionType", + { + "creating": "creating", + "updating": "updating", + "deleting": "deleting", + }, + ), + "db_column_name": None, + "secret": False, + }, + ) + + manager.add_column( + table_class_name="AuditLog", + tablename="audit_log", + column_name="action_user", + db_column_name="action_user", + column_class_name="Varchar", + column_class=Varchar, + params={ + "length": 255, + "default": "", + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + "db_column_name": None, + "secret": False, + }, + ) + + manager.add_column( + table_class_name="AuditLog", + tablename="audit_log", + column_name="table_name", + db_column_name="table_name", + column_class_name="Varchar", + column_class=Varchar, + params={ + "length": 255, + "default": "", + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + "db_column_name": None, + "secret": False, + }, + ) + + manager.add_column( + table_class_name="AuditLog", + tablename="audit_log", + column_name="change_message", + db_column_name="change_message", + column_class_name="Text", + column_class=Text, + params={ + "default": "", + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + "db_column_name": None, + "secret": False, + }, + ) + + manager.add_column( + table_class_name="AuditLog", + tablename="audit_log", + column_name="changes_in_row", + db_column_name="changes_in_row", + column_class_name="JSON", + column_class=JSON, + params={ + "default": "{}", + "null": False, + "primary_key": False, + "unique": False, + "index": False, + "index_method": IndexMethod.btree, + "choices": None, + "db_column_name": None, + "secret": False, + }, + ) + + return manager diff --git a/piccolo_api/audit_logs/tables.py b/piccolo_api/audit_logs/tables.py new file mode 100644 index 00000000..071faf73 --- /dev/null +++ b/piccolo_api/audit_logs/tables.py @@ -0,0 +1,124 @@ +import typing as t +import uuid +from enum import Enum + +from piccolo.apps.user.tables import BaseUser +from piccolo.columns import JSON, Text, Timestamp, Varchar +from piccolo.table import Table + + +class AuditLog(Table): + class ActionType(str, Enum): + """An enumeration of AuditLog table actions type.""" + + creating = "creating" + updating = "updating" + deleting = "deleting" + + action_time = Timestamp() + action_type = Varchar(choices=ActionType) + action_user = Varchar() + table_name = Varchar() + change_message = Text() + changes_in_row = JSON() + + @classmethod + async def record_save_action( + cls, + table: t.Type[Table], + user_id: int, + new_row_id=t.Union[str, uuid.UUID, int], + ): + """ + A method for tracking creating record actions. + + :param table: + A table for which we monitor activities. + :param user_id: + The ``primary key`` of authenticated user. + :param new_row_id: + The ``primary key`` of the newly created record. + """ + result = cls( + action_type=cls.ActionType.creating, + action_user=cls.get_user_username(user_id), + table_name=table._meta.tablename.title(), + change_message=f"User {cls.get_user_username(user_id)} " + f"create row {new_row_id} in {table._meta.tablename.title()} " + f"table", + ) + await result.save().run() + + @classmethod + async def record_patch_action( + cls, + table: t.Type[Table], + row_id: t.Union[str, uuid.UUID, int], + user_id: int, + changes_in_row: t.Dict[str, t.Any], + ): + """ + A method for tracking updating record actions. + + :param table: + A table for which we monitor activities. + :param row_id: + The ``primary key`` of the table for which we + monitor activities. + :param user_id: + The ``primary key`` of authenticated user. + :param changes_in_row: + JSON with all changed columns in the existing row. + """ + result = cls( + action_type=cls.ActionType.updating, + action_user=cls.get_user_username(user_id), + table_name=table._meta.tablename.title(), + change_message=f"User {cls.get_user_username(user_id)} update row " + f"{row_id} in {table._meta.tablename.title()} table", + changes_in_row=changes_in_row, + ) + await result.save().run() + + @classmethod + async def record_delete_action( + cls, + table: t.Type[Table], + row_id: t.Union[str, uuid.UUID, int], + user_id: int, + ): + """ + A method for tracking deletion record actions. + + :param table: + A table for which we monitor activities. + :param row_id: + The ``primary key`` of the table for which we + monitor activities. + :param user_id: + The ``primary key`` of authenticated user. + """ + result = cls( + action_type=cls.ActionType.deleting, + action_user=cls.get_user_username(user_id), + table_name=table._meta.tablename.title(), + change_message=f"User {cls.get_user_username(user_id)} delete row " + f"{row_id} in {table._meta.tablename.title()} table", + ) + await result.save().run() + + @classmethod + def get_user_username(cls, user_id: int) -> str: + """ + Returns the username of authenticated user. + + :param user_id: + The ``primary key`` of authenticated user. + """ + user = ( + BaseUser.select(BaseUser.username) + .where(BaseUser._meta.primary_key == user_id) + .first() + .run_sync() + ) + return user["username"] diff --git a/piccolo_api/crud/endpoints.py b/piccolo_api/crud/endpoints.py index f4fe89bd..08e17231 100644 --- a/piccolo_api/crud/endpoints.py +++ b/piccolo_api/crud/endpoints.py @@ -27,6 +27,7 @@ from starlette.responses import JSONResponse, Response from starlette.routing import Route, Router +from piccolo_api.audit_logs.tables import AuditLog from piccolo_api.crud.hooks import ( Hook, HookType, @@ -148,6 +149,7 @@ def __init__( schema_extra: t.Optional[t.Dict[str, t.Any]] = None, max_joins: int = 0, hooks: t.Optional[t.List[Hook]] = None, + audit_log_table: t.Optional[t.Type[AuditLog]] = None, ) -> None: """ :param table: @@ -217,6 +219,7 @@ def __init__( } else: self._hook_map = None # type: ignore + self.audit_log_table = audit_log_table schema_extra = schema_extra if isinstance(schema_extra, dict) else {} self.visible_fields_options = get_visible_fields_options( @@ -334,6 +337,22 @@ def pydantic_model_plural( rows=(t.List[base_model], None), ) + def get_single_row( + self, + table: t.Type[Table], + row_id: t.Union[str, uuid.UUID, int], + ) -> t.Dict[str, t.Any]: + """ + Return a single row. + """ + row = ( + self.table.select(exclude_secrets=self.exclude_secrets) + .where(self.table._meta.primary_key == row_id) + .first() + .run_sync() + ) + return row + @apply_validators async def get_schema(self, request: Request) -> JSONResponse: """ @@ -813,6 +832,16 @@ async def post_single( hooks=self._hook_map, hook_type=HookType.pre_save, row=row ) response = await row.save().run() + new_row_id = list(response[0].values()) + try: + if self.audit_log_table: + await self.audit_log_table.record_save_action( + self.table, + user_id=request.user.user_id, + new_row_id=new_row_id[0], + ) + except Exception as exception: + logger.log(msg=f"{exception}", level=logging.WARNING) json = dump_json(response) # Returns the id of the inserted row. return CustomJSONResponse(json, status_code=201) @@ -1057,15 +1086,24 @@ async def patch_single( ) try: + old_row = self.get_single_row(cls, row_id) await cls.update(values).where( cls._meta.primary_key == row_id ).run() - new_row = ( - await cls.select(exclude_secrets=self.exclude_secrets) - .where(cls._meta.primary_key == row_id) - .first() - .run() - ) + new_row = self.get_single_row(cls, row_id) + changes_in_row = { + k: v for k, v in new_row.items() - old_row.items() + } + try: + if self.audit_log_table: + await self.audit_log_table.record_patch_action( + cls, + row_id=row_id, + user_id=request.user.user_id, + changes_in_row=changes_in_row, + ) + except Exception as exception: + logger.log(msg=f"{exception}", level=logging.WARNING) return CustomJSONResponse(self.pydantic_model(**new_row).json()) except ValueError: return Response("Unable to save the resource.", status_code=500) @@ -1089,6 +1127,13 @@ async def delete_single( await self.table.delete().where( self.table._meta.primary_key == row_id ).run() + try: + if self.audit_log_table: + await self.audit_log_table.record_delete_action( + self.table, row_id=row_id, user_id=request.user.user_id + ) + except Exception as exception: + logger.log(msg=f"{exception}", level=logging.WARNING) return Response(status_code=204) except ValueError: return Response("Unable to delete the resource.", status_code=500) diff --git a/tests/audit_logs/test_audit_logs.py b/tests/audit_logs/test_audit_logs.py new file mode 100644 index 00000000..25cbe255 --- /dev/null +++ b/tests/audit_logs/test_audit_logs.py @@ -0,0 +1,186 @@ +from unittest import TestCase + +from piccolo.apps.user.tables import BaseUser +from piccolo.columns import Integer, Varchar +from piccolo.table import Table +from piccolo.utils.sync import run_sync +from starlette.testclient import TestClient + +from piccolo_api.audit_logs.commands import clean +from piccolo_api.audit_logs.tables import AuditLog +from piccolo_api.crud.endpoints import PiccoloCRUD + + +class Movie(Table): + name = Varchar(length=100, required=True) + rating = Integer() + + +class TestSaveAuditLogs(TestCase): + def setUp(self): + BaseUser.create_table(if_not_exists=True).run_sync() + AuditLog.create_table(if_not_exists=True).run_sync() + Movie.create_table(if_not_exists=True).run_sync() + + def tearDown(self): + BaseUser.alter().drop_table().run_sync() + AuditLog.alter().drop_table().run_sync() + Movie.alter().drop_table().run_sync() + + def test_save_audit_logs(self): + """ + Make sure a AuditLog post_save_action works. + """ + user = run_sync( + BaseUser.create_user(username="admin", password="admin123") + ) + client = TestClient( + PiccoloCRUD(table=Movie, read_only=False, audit_log_table=AuditLog) + ) + + json = {"name": "Star Wars", "rating": 93} + + response = client.post("/", json=json) + run_sync( + AuditLog.record_save_action( + Movie, user_id=user.id, new_row_id=response.json()[0]["id"] + ) + ) + self.assertEqual(response.status_code, 201) + + audit_log = AuditLog.select(AuditLog.action_type).first().run_sync() + self.assertEqual(audit_log["action_type"], "creating") + self.assertEqual(len(audit_log), 1) + + +class TestPatchAuditLogs(TestCase): + def setUp(self): + BaseUser.create_table(if_not_exists=True).run_sync() + AuditLog.create_table(if_not_exists=True).run_sync() + Movie.create_table(if_not_exists=True).run_sync() + + def tearDown(self): + BaseUser.alter().drop_table().run_sync() + AuditLog.alter().drop_table().run_sync() + Movie.alter().drop_table().run_sync() + + def test_patch_audit_logs(self): + """ + Make sure a AuditLog post_patch_action works. + """ + user = run_sync( + BaseUser.create_user(username="admin", password="admin123") + ) + + client = TestClient( + PiccoloCRUD(table=Movie, read_only=False, audit_log_table=AuditLog) + ) + + rating = 93 + movie = Movie(name="Star Wars", rating=rating) + movie.save().run_sync() + + new_name = "Star Wars: A New Hope" + + response = client.patch(f"/{movie.id}/", json={"name": new_name}) + run_sync( + AuditLog.record_patch_action( + Movie, + row_id=movie.id, + user_id=user.id, + changes_in_row={"name": new_name}, + ) + ) + self.assertEqual(response.status_code, 200) + + audit_log = AuditLog.select(AuditLog.action_type).first().run_sync() + self.assertEqual(audit_log["action_type"], "updating") + self.assertEqual(len(audit_log), 1) + + +class TestDeleteAuditLogs(TestCase): + def setUp(self): + BaseUser.create_table(if_not_exists=True).run_sync() + AuditLog.create_table(if_not_exists=True).run_sync() + Movie.create_table(if_not_exists=True).run_sync() + + def tearDown(self): + BaseUser.alter().drop_table().run_sync() + AuditLog.alter().drop_table().run_sync() + Movie.alter().drop_table().run_sync() + + def test_delete_audit_logs(self): + """ + Make sure a AuditLog post_delete_action works. + """ + user = run_sync( + BaseUser.create_user(username="admin", password="admin123") + ) + client = TestClient( + PiccoloCRUD(table=Movie, read_only=False, audit_log_table=AuditLog) + ) + + movie = Movie(name="Star Wars", rating=93) + movie.save().run_sync() + + response = client.delete(f"/{movie.id}/") + run_sync( + AuditLog.record_delete_action( + Movie, row_id=movie.id, user_id=user.id + ) + ) + self.assertTrue(response.status_code == 204) + + audit_log = AuditLog.select(AuditLog.action_type).first().run_sync() + self.assertEqual(audit_log["action_type"], "deleting") + self.assertEqual(len(audit_log), 1) + + +class TestCleanAuditLogs(TestCase): + def setUp(self): + BaseUser.create_table(if_not_exists=True).run_sync() + AuditLog.create_table(if_not_exists=True).run_sync() + Movie.create_table(if_not_exists=True).run_sync() + + def tearDown(self): + BaseUser.alter().drop_table().run_sync() + AuditLog.alter().drop_table().run_sync() + Movie.alter().drop_table().run_sync() + + def test_clean_audit_logs(self): + """ + Make sure a AuditLog clean() method works. + """ + user = run_sync( + BaseUser.create_user(username="admin", password="admin123") + ) + client = TestClient(PiccoloCRUD(table=Movie, read_only=False)) + + json = {"name": "Star Wars", "rating": 93} + + response = client.post("/", json=json) + + run_sync( + AuditLog.record_save_action( + Movie, user_id=user.id, new_row_id=response.json()[0]["id"] + ) + ) + self.assertEqual(response.status_code, 201) + + movie = Movie.select().first().run_sync() + + response = client.delete(f"/{movie['id']}/") + run_sync( + AuditLog.record_delete_action( + Movie, row_id=movie["id"], user_id=user.id + ) + ) + self.assertTrue(response.status_code == 204) + + audit_log = AuditLog.select().run_sync() + self.assertEqual(len(audit_log), 2) + + run_sync(clean()) + + audit_log = AuditLog.select().run_sync() + self.assertEqual(len(audit_log), 0)