Sync page fixes

monitoring/settings.py

@@ -21,7 +21,7 @@
 # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = False
+DEBUG = True
 
 try:
 
@@ -77,6 +77,7 @@
     'rest_framework',
     'monitoring.publishing',
     'monitoring.availability',
+    'monitoring.synchronisation',
     'monitoring.benchmarks',
 ]
 

monitoring/synchronisation/admin.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.contrib import admin
+
+# Register your models here.

monitoring/synchronisation/apps.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.apps import AppConfig
+
+
+class SynchronisationConfig(AppConfig):
+    name = 'monitoring.synchronisation'

monitoring/synchronisation/models.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models
+
+
+class GridSiteSync(models.Model):
+    fetched = models.DateTimeField(auto_now=True)
+    site = models.CharField(max_length=255)
+    year = models.IntegerField()
+    month = models.IntegerField()
+    site_count = models.IntegerField()
+    repository_count = models.IntegerField()
+    difference = models.IntegerField()
+
+    class Meta:
+        unique_together = ['site', 'year', 'month']
+
+
+class VSuperSummaries(models.Model):
+    Site = models.CharField(max_length=255)
+    Year = models.IntegerField()
+    Month = models.IntegerField()
+    NumberOfJobs = models.IntegerField()
+
+    class Meta:
+        managed = False
+        db_table = 'VSuperSummaries'
+        unique_together = ('Site', 'Year', 'Month')
+
+
+class VSyncRecords(models.Model):
+    Site = models.CharField(max_length=255)
+    Year = models.IntegerField()
+    Month = models.IntegerField()
+    NumberOfJobs = models.IntegerField()
+
+    class Meta:
+        managed = False
+        db_table = 'VSyncRecords'
+        unique_together = ('Site', 'Year', 'Month')

monitoring/synchronisation/serializers.py

@@ -0,0 +1,15 @@
+from rest_framework import serializers
+
+from monitoring.synchronisation.models import GridSiteSync
+
+
+class GridSiteSyncSerializer(serializers.HyperlinkedModelSerializer):
+    # Override default format with None so that Python datetime is used as
+    # ouput format. Encoding will be determined by the renderer and can be
+    # formatted by a template filter.
+    updated = serializers.DateTimeField(format=None)
+
+    class Meta:
+        model = GridSiteSync
+        fields = ('url', 'site', 'year', 'month',
+                  'site_count', 'repository_count', 'difference')

monitoring/synchronisation/tests.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.test import TestCase
+
+# Create your tests here.

monitoring/synchronisation/urls.py

@@ -0,0 +1,13 @@
+from django.urls import include, path
+
+from rest_framework import routers
+
+from monitoring.synchronisation import views
+
+router = routers.SimpleRouter()
+router.register(r'grid', views.GridSiteSyncViewSet)
+
+
+urlpatterns = [
+    path('', include(router.urls)),
+]

monitoring/synchronisation/views.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from datetime import datetime, timedelta
+
+from django.db.models import Max
+
+from rest_framework import viewsets
+from rest_framework.renderers import TemplateHTMLRenderer
+
+from monitoring.synchronisation.models import GridSiteSync, VSuperSummaries, VSyncRecords
+from monitoring.synchronisation.serializers import GridSiteSyncSerializer
+
+
+class GridSiteSyncViewSet(viewsets.ReadOnlyModelViewSet):
+    queryset = GridSiteSync.objects.all()
+    serializer_class = GridSiteSyncSerializer
+    template_name = 'gridsites.html'
+
+    def retrieve(self, request, pk=None):
+        last_fetched = GridSiteSync.objects.aggregate(Max('fetched'))['fetched__max']
+        # If there's no data then last_fetched is None.
+        if last_fetched is not None:
+            print(last_fetched.replace(tzinfo=None), datetime.today() - timedelta(hours=1, seconds=20))
+        if last_fetched is None or last_fetched.replace(tzinfo=None) < (datetime.today() - timedelta(hours=1, seconds=20)):
+            print('Out of date')
+            fetchset = VSuperSummaries.objects.using('grid').raw("SELECT Site, max(LatestEndTime) AS LatestPublish FROM VSuperSummaries WHERE Year=2019 GROUP BY 1;")
+            for f in fetchset:
+                GridSiteSync.objects.update_or_create(defaults={'updated': f.LatestPublish}, name=f.Site)
+        else:
+            print('No need to update')
+
+        response = super(GridSiteSyncViewSet, self).retrieve(request)
+        date = response.data['updated'].replace(tzinfo=None)
+
+        # Wrap data in a dict so that it can display in template.
+        if type(request.accepted_renderer) is TemplateHTMLRenderer:
+            # Single result put in list to work with same HTML template.
+            response.data = {'sites': [response.data], 'last_fetched': last_fetched}
+
+        diff = datetime.today() - date
+        if diff <= timedelta(days=7):
+            response.data['returncode'] = 0
+            response.data['stdout'] = "OK [ last published %s days ago: %s ]" % (diff.days, date.strftime("%Y-%m-%d"))
+        elif diff > timedelta(days=7):
+            response.data['returncode'] = 1
+            response.data['stdout'] = "WARNING [ last published %s days ago: %s ]" % (diff.days, date.strftime("%Y-%m-%d"))
+        else:
+            response.data['returncode'] = 3
+            response.data['stdout'] = "UNKNOWN"
+
+        return response

monitoring/urls.py

@@ -7,6 +7,7 @@
     path('', TemplateView.as_view(template_name='home.html'), name='home'),
     path('availability/', include('monitoring.availability.urls')),
     path('publishing/', include('monitoring.publishing.urls')),
+    path('synchronisation/', include('monitoring.synchronisation.urls')),
     path('benchmarks/', include('monitoring.benchmarks.urls')),
     path('api-auth/', include('rest_framework.urls', namespace='rest_framework')),
 ]