From 8056793266e81c9f60dafe1f12e04f310feb4c75 Mon Sep 17 00:00:00 2001 From: leksyib Date: Thu, 20 Jun 2019 16:07:23 +0100 Subject: [PATCH] CON-72-story(notify-admin-when-device-is-not-seen-for-a-while) - create device is offline notification - periodically check the device last seen - add too states for a device: online or offline [Finishes CON-72] --- .circleci/config.yml | 12 +++---- .codeclimate.yml | 4 ++- .coveragerc | 2 -- .gitignore | 2 -- admin_notifications/__init__.py | 0 admin_notifications/helpers/__init__.py | 0 .../helpers/create_notification.py | 32 ++++++++++++++++++ .../helpers/device_last_seen.py | 33 +++++++++++++++++++ .../helpers/notification_templates.py | 7 ++++ admin_notifications/helpers/queue_manager.py | 8 +++++ admin_notifications/models.py | 18 ++++++++++ admin_notifications/socket_handler.py | 17 ++++++++++ alembic/env.py | 2 +- ...41_add_activity_column_to_devices_table.py | 33 +++++++++++++++++++ api/devices/models.py | 5 +-- api/devices/schema.py | 2 +- celerybeat.pid | 1 + cworker.py | 19 +++++++++-- docker/dev/start_redis.sh | 3 +- docker/prod/start_redis.sh | 3 +- fixtures/devices/devices_fixtures.py | 6 ++-- helpers/email/email_setup.py | 2 +- manage.py | 9 +++-- requirements.txt | 1 + tests/test_admin_notification/__init__.py | 0 .../test_device_not_seen.py | 16 +++++++++ utilities/utility.py | 12 +++++++ 27 files changed, 222 insertions(+), 27 deletions(-) create mode 100644 admin_notifications/__init__.py create mode 100644 admin_notifications/helpers/__init__.py create mode 100644 admin_notifications/helpers/create_notification.py create mode 100644 admin_notifications/helpers/device_last_seen.py create mode 100644 admin_notifications/helpers/notification_templates.py create mode 100644 admin_notifications/helpers/queue_manager.py create mode 100644 admin_notifications/models.py create mode 100644 admin_notifications/socket_handler.py create mode 100644 alembic/versions/79ef610dbd41_add_activity_column_to_devices_table.py create mode 100644 celerybeat.pid create mode 100644 tests/test_admin_notification/__init__.py create mode 100644 tests/test_admin_notification/test_device_not_seen.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 2a0dadcb8..85b7da577 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,9 +36,9 @@ gcloud_setup: &gcloud_setup run: name: setup gcloud command: | - # install + # install sudo curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz - sudo mkdir -p /usr/local/gcloud + sudo mkdir -p /usr/local/gcloud sudo tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz sudo /usr/local/gcloud/google-cloud-sdk/install.sh --quiet echo PATH=$PATH:/usr/local/gcloud/google-cloud-sdk/bin >> ~/.bashrc @@ -190,8 +190,8 @@ jobs: command: | ./cc-test-reporter before-build . venv/bin/activate - coverage combine parallel-coverage/ - coverage xml + coverage combine parallel-coverage/ + coverage xml -i coverage report ./cc-test-reporter format-coverage -o ./.coverage -t coverage.py ./cc-test-reporter upload-coverage -i .coverage @@ -304,13 +304,13 @@ jobs: command: | if [ "$CIRCLE_BRANCH" == master ] || [ "$CIRCLE_BRANCH" == develop ]; then touch google-service-key.json - echo $GOOGLE_CREDENTIALS_STAGING | base64 --decode >> google-service-key.json + echo $GOOGLE_CREDENTIALS_STAGING | base64 --decode >> google-service-key.json gcloud auth activate-service-account --key-file google-service-key.json gcloud --quiet config set project ${GOOGLE_PROJECT_ID_STAGING} gcloud --quiet config set compute/zone ${GOOGLE_COMPUTE_ZONE} else touch google-service-key.json - echo $GOOGLE_CREDENTIALS_SANDBOX | base64 --decode >> google-service-key.json + echo $GOOGLE_CREDENTIALS_SANDBOX | base64 --decode >> google-service-key.json gcloud auth activate-service-account --key-file google-service-key.json gcloud --quiet config set project ${GOOGLE_PROJECT_ID_SANDBOX} gcloud --quiet config set compute/zone ${GOOGLE_COMPUTE_ZONE} diff --git a/.codeclimate.yml b/.codeclimate.yml index d0ded27fc..42f34455e 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -2,4 +2,6 @@ version: "2" exclude_patterns: - "helpers/auth/authentication.py" - "helpers/calendar/events.py" - - "alembic/" \ No newline at end of file + - "**/alembic/" + - "**/*__init__.py" + - "**/tests/" diff --git a/.coveragerc b/.coveragerc index 3d0cca121..a52e224fe 100644 --- a/.coveragerc +++ b/.coveragerc @@ -13,5 +13,3 @@ omit = [html] directory=html_coverage_report - - \ No newline at end of file diff --git a/.gitignore b/.gitignore index dbf0b9abc..adcf114e1 100644 --- a/.gitignore +++ b/.gitignore @@ -42,8 +42,6 @@ html_coverage_report/ .tox/ .coverage .coverage.* -.coveragerc -setup.cfg .cache nosetests.xml coverage.xml diff --git a/admin_notifications/__init__.py b/admin_notifications/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/admin_notifications/helpers/__init__.py b/admin_notifications/helpers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/admin_notifications/helpers/create_notification.py b/admin_notifications/helpers/create_notification.py new file mode 100644 index 000000000..7c73379ec --- /dev/null +++ b/admin_notifications/helpers/create_notification.py @@ -0,0 +1,32 @@ +from admin_notifications.models import AdminNotification +from api.location.models import Location +from datetime import datetime + + +def update_notification(notification_id): + notification = AdminNotification.query.filter_by(id=notification_id).first() + notification.date_received = datetime.now() + notification.save() + + +def create_notification(title, message, location_id): + """ + Create notifications in the database and emit them to the client + """ + from manage import socketio + location = Location.query.filter_by(id=location_id).first() + location_name = location.name + notification = AdminNotification( + title=title, + message=message, + location_id=location_id, + status="unread" + ) + notification.save() + new_notification = {"title": title, "message": message} + return socketio.emit( + f"notifications-{location_name}", + {'notification': new_notification}, + broadcast=True, + callback=update_notification(notification.id) + ) diff --git a/admin_notifications/helpers/device_last_seen.py b/admin_notifications/helpers/device_last_seen.py new file mode 100644 index 000000000..43a4377b6 --- /dev/null +++ b/admin_notifications/helpers/device_last_seen.py @@ -0,0 +1,33 @@ +from datetime import datetime +from api.devices.models import Devices as DevicesModel +from utilities.utility import update_entity_fields +from admin_notifications.helpers.create_notification import create_notification +from admin_notifications.helpers.notification_templates import device_offline_notification # noqa 501 +import celery + + +@celery.task(name='check-device-last-seen') +def notify_when_device_is_offline(): + """Asynchronous method that checks whether a device's last seen is greater\ + than 24hours, turns them to offline and subsequently notify's + """ + query = DevicesModel.query + online_devices = query.filter(DevicesModel.activity == "online").all() + for device in online_devices: + device_last_seen = device.last_seen + current_time = datetime.now() + duration_offline = current_time - device_last_seen + + if duration_offline.days > 1: + update_entity_fields(device, activity="offline") + device.save() + + room_name = device.room.name + room_id = device.room.id + notification_payload = device_offline_notification( + room_name, room_id) + create_notification(title=notification_payload['title'], + message=notification_payload['message'], + location_id=device.room.location_id) + + return online_devices diff --git a/admin_notifications/helpers/notification_templates.py b/admin_notifications/helpers/notification_templates.py new file mode 100644 index 000000000..0feb12985 --- /dev/null +++ b/admin_notifications/helpers/notification_templates.py @@ -0,0 +1,7 @@ + +def device_offline_notification(room_name, room_id): + """Notification message when device has been offline for a while""" + return { + "title": "Device is offline", + "message": f"A device in {room_name} roomid:{room_id} is offline." + } diff --git a/admin_notifications/helpers/queue_manager.py b/admin_notifications/helpers/queue_manager.py new file mode 100644 index 000000000..b0a98aa53 --- /dev/null +++ b/admin_notifications/helpers/queue_manager.py @@ -0,0 +1,8 @@ +from datetime import timedelta +"""Celery beat schedule that checks a device's last seen every 24 hours""" +beat_schedule = { + 'run-check-device-last-seen-hourly': { + 'task': 'check-device-last-seen', + 'schedule': timedelta(hours=1) + } +} diff --git a/admin_notifications/models.py b/admin_notifications/models.py new file mode 100644 index 000000000..1cf151dfd --- /dev/null +++ b/admin_notifications/models.py @@ -0,0 +1,18 @@ +from sqlalchemy import (Column, String, Enum, Integer, ForeignKey) +from helpers.database import Base +from utilities.utility import Utility, StatusType + + +class AdminNotification(Base, Utility): + __tablename__ = 'admin_notifications' + + id = Column(Integer, primary_key=True) # noqa + title = Column(String, nullable=True) + message = Column(String, nullable=True) + date_received = Column(String, nullable=True) + date_read = Column(String, nullable=True) + status = Column(Enum(StatusType), default="unread") + location_id = Column( + Integer, + ForeignKey('locations.id', ondelete="CASCADE"), + nullable=True) diff --git a/admin_notifications/socket_handler.py b/admin_notifications/socket_handler.py new file mode 100644 index 000000000..ca4463674 --- /dev/null +++ b/admin_notifications/socket_handler.py @@ -0,0 +1,17 @@ +from flask_socketio import send +from admin_notifications.models import AdminNotification + + +def serialize_message(notification): + return { + "title": notification.title, + "message": notification.message, + } + + +def send_notifications(): + query = AdminNotification.query + notifications = query.filter_by(status="unread").all() + notifications = [serialize_message(notification) + for notification in notifications] + return send(notifications, broadcast=True) diff --git a/alembic/env.py b/alembic/env.py index 01ce12db5..5a0334069 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -45,7 +45,7 @@ from api.response.models import Response from api.tag.models import Tag from api.structure.models import Structure - +from admin_notifications.models import AdminNotification target_metadata = Base.metadata diff --git a/alembic/versions/79ef610dbd41_add_activity_column_to_devices_table.py b/alembic/versions/79ef610dbd41_add_activity_column_to_devices_table.py new file mode 100644 index 000000000..1cf93af7a --- /dev/null +++ b/alembic/versions/79ef610dbd41_add_activity_column_to_devices_table.py @@ -0,0 +1,33 @@ +"""add activity column to devices table + +Revision ID: 79ef610dbd41 +Revises: af8e4f84b552 +Create Date: 2019-06-28 08:05:37.542613 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision = '79ef610dbd41' +down_revision = 'af8e4f84b552' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + activitytype = postgresql.ENUM( + 'online', 'offline', name='activitytype') + activitytype.create(op.get_bind()) + op.add_column('devices', sa.Column('activity', sa.Enum( + 'online', 'offline', name='activitytype'), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('devices', 'activity') + # ### end Alembic commands ### diff --git a/api/devices/models.py b/api/devices/models.py index 083a4eaa5..6416abace 100644 --- a/api/devices/models.py +++ b/api/devices/models.py @@ -4,12 +4,12 @@ from helpers.database import Base from utilities.validations import validate_empty_fields -from utilities.utility import Utility, StateType +from utilities.utility import Utility, StateType, ActivityType class Devices(Base, Utility): __tablename__ = 'devices' - id = Column(Integer, Sequence('devices_id_seq', start=1, increment=1), primary_key=True) # noqa + id = Column(Integer, Sequence('devices_id_seq', start=1, increment=1), primary_key=True) # noqa name = Column(String, nullable=False) device_type = Column(String, nullable=False) date_added = Column(DateTime, nullable=False) @@ -18,6 +18,7 @@ class Devices(Base, Utility): room_id = Column(Integer, ForeignKey('rooms.id', ondelete="CASCADE")) room = relationship('Room') state = Column(Enum(StateType), default="active") + activity = Column(Enum(ActivityType), default="online") def __init__(self, **kwargs): validate_empty_fields(**kwargs) diff --git a/api/devices/schema.py b/api/devices/schema.py index 798dd94a7..7565f0b10 100644 --- a/api/devices/schema.py +++ b/api/devices/schema.py @@ -41,7 +41,7 @@ def mutate(self, info, **kwargs): room_location = location_join_room().filter( RoomModel.id == kwargs['room_id'], RoomModel.state == 'active' - ).first() + ).first() if not room_location: raise GraphQLError("Room not found") user = get_user_from_db() diff --git a/celerybeat.pid b/celerybeat.pid new file mode 100644 index 000000000..8f92bfdd4 --- /dev/null +++ b/celerybeat.pid @@ -0,0 +1 @@ +35 diff --git a/cworker.py b/cworker.py index ad6695d1a..bdd1d8a5b 100644 --- a/cworker.py +++ b/cworker.py @@ -1,16 +1,26 @@ import os - from celery import Celery from app import create_app - +from admin_notifications.helpers.queue_manager import beat_schedule app = create_app(os.getenv('APP_SETTINGS') or 'default') app.app_context().push() +app.config.update( + CELERY_BROKER_URL=os.getenv('CELERY_BROKER_URL'), + CELERY_RESULT_BACKEND=os.getenv('CELERY_RESULT_BACKEND'), + CELERY_ACCEPT_CONTENT=['pickle'], + CELERYBEAT_SCHEDULE=beat_schedule +) + + def make_celery(app): - celery = Celery(app.name, broker=app.config['CELERY_BROKER_URL']) + celery = Celery(app.name, broker=app.config['CELERY_BROKER_URL'], include=['admin_notifications.helpers.device_last_seen', 'admin_notifications.helpers.create_notification'], # noqa 501 + backend=app.config['CELERY_BROKER_URL']) + celery.conf.update(app.config) + celery.conf.enable_utc = False TaskBase = celery.Task class ContextTask(TaskBase): @@ -24,3 +34,6 @@ def __call__(self, *args, **kwargs): celery = make_celery(app) + +celery_scheduler = Celery(app.name, broker=app.config['CELERY_BROKER_URL']) +celery_scheduler.conf.enable_utc = False diff --git a/docker/dev/start_redis.sh b/docker/dev/start_redis.sh index 451095664..45faafc1c 100755 --- a/docker/dev/start_redis.sh +++ b/docker/dev/start_redis.sh @@ -4,4 +4,5 @@ #done cd /app export $(cat .env | xargs) -celery worker -A cworker.celery --loglevel=info +celery worker -A cworker.celery --loglevel=info & +celery -A cworker.celery beat -l info diff --git a/docker/prod/start_redis.sh b/docker/prod/start_redis.sh index 1f7378356..fe68346ce 100755 --- a/docker/prod/start_redis.sh +++ b/docker/prod/start_redis.sh @@ -1,4 +1,5 @@ #!/bin/bash cd /app export $(cat .env | xargs) -celery worker -A cworker.celery --loglevel=info +celery worker -A cworker.celery --loglevel=info & +celery -A cworker.celery beat -l info diff --git a/fixtures/devices/devices_fixtures.py b/fixtures/devices/devices_fixtures.py index 51bcc086a..5152489c9 100644 --- a/fixtures/devices/devices_fixtures.py +++ b/fixtures/devices/devices_fixtures.py @@ -49,8 +49,8 @@ "name": "Samsung" } ] - } } +} query_device = ''' { @@ -289,7 +289,7 @@ 'id': '1', 'name': 'Samsung', 'deviceType': 'External Display' - }] - } + }] } +} devices_query_response = b'{"data":{"createDevice":{"device":{"name":"Apple tablet","location":"Kampala","deviceType":"External Display"}}}}' # noqaE501 diff --git a/helpers/email/email_setup.py b/helpers/email/email_setup.py index 236499f23..82ff25ad1 100644 --- a/helpers/email/email_setup.py +++ b/helpers/email/email_setup.py @@ -30,7 +30,7 @@ def __init__( html=self.template, sender=self.sender) - @celery.task + @celery.task(name='asynchronous-email-notifications') def send_async_email(msg_dict): mail = Mail() msg = Message() diff --git a/manage.py b/manage.py index a9e7943d6..e81f7d343 100644 --- a/manage.py +++ b/manage.py @@ -3,13 +3,14 @@ import bugsnag from flask_script import Manager, Shell from bugsnag.flask import handle_exceptions +from flask_socketio import SocketIO # Configure bugnsag bugsnag.configure( - api_key=os.getenv('BUGSNAG_API_TOKEN'), - release_stage="development", - project_root="app" + api_key=os.getenv('BUGSNAG_API_TOKEN'), + release_stage="development", + project_root="app" ) # local imports @@ -18,6 +19,7 @@ app = create_app(os.getenv('APP_SETTINGS') or 'default') handle_exceptions(app) manager = Manager(app) +socketio = SocketIO(app) def make_shell_context(): @@ -30,4 +32,5 @@ def make_shell_context(): if __name__ == '__main__': + socketio.run(app, debug=True) manager.run() diff --git a/requirements.txt b/requirements.txt index e27578af6..ea392e095 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ Flask-JSON==0.3.2 Flask-Script==2.0.6 Flask-GraphQL==1.4.1 Flask-Mail==0.9.1 +Flask-SocketIO==4.1.0 google-api-python-client==1.6.7 graphene-sqlalchemy==2.0.0 graphene==2.1 diff --git a/tests/test_admin_notification/__init__.py b/tests/test_admin_notification/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_admin_notification/test_device_not_seen.py b/tests/test_admin_notification/test_device_not_seen.py new file mode 100644 index 000000000..d3d8a38fa --- /dev/null +++ b/tests/test_admin_notification/test_device_not_seen.py @@ -0,0 +1,16 @@ +from tests.base import BaseTestCase +from fixtures.token.token_fixture import ADMIN_TOKEN +from fixtures.devices.devices_fixtures import devices_query +from admin_notifications.helpers.device_last_seen import ( + notify_when_device_is_offline) + + +class TestDeviceOffline(BaseTestCase): + def test_when_device_is_offline(self): + """ + Testing for device creation + """ + headers = {"Authorization": "Bearer" + " " + ADMIN_TOKEN} + self.app_test.post(devices_query, headers=headers) + response = notify_when_device_is_offline() + assert response[0].activity.value == 'offline' diff --git a/utilities/utility.py b/utilities/utility.py index 69dca7ae2..b9abd579a 100644 --- a/utilities/utility.py +++ b/utilities/utility.py @@ -87,3 +87,15 @@ class QuestionType(enum.Enum): check = "check" text_area = "text_area" missing_items = "missing_items" + textarea = "textarea" + missingitem = "missing_items" + + +class StatusType(enum.Enum): + read = "read" + unread = "unread" + + +class ActivityType(enum.Enum): + online = "online" + offline = "offline"