backend: Set expiration for ingested reports, fixes #258. (#468)

Author: La0

backend/code_coverage_backend/gcp.py

@@ -132,6 +132,10 @@ def ingest_report(self, report):
         assert len(overall_coverage) > 0, "No overall coverage"
         self.redis.hmset(report.key_overall, overall_coverage)
 
+        # Apply expiry for overall report
+        if report.ttl is not None:
+            self.redis.expire(report.key_overall, report.ttl)
+
         # Add the changeset to the sorted sets of known reports
         # The numeric push_id is used as a score to keep the ingested
         # changesets ordered

backend/code_coverage_backend/report.py

@@ -94,3 +94,15 @@ def key_overall(self):
         platform = self.platform or "all"
         suite = self.suite or "all"
         return f"overall:{self.repository}:{self.changeset}:{platform}:{suite}"
+
+    @property
+    def ttl(self):
+        """Time to live in seconds for the full report
+
+        Will be None (no expiry) for full report (all:all)
+        Otherwise set to 2 weeks
+        """
+        if self.suite == DEFAULT_FILTER and self.platform == DEFAULT_FILTER:
+            return
+
+        return 15 * 24 * 3600

backend/tests/test_gcp.py

@@ -65,6 +65,10 @@ def test_ingestion(mock_cache):
     mock_cache.ingest_report(report_2)
     mock_cache.ingest_report(report_10)
 
+    # Check expiry
+    assert report_1.ttl is None
+    assert mock_cache.redis.ttl(report_1.key_overall) == -1
+
     # They must be in redis and on the file system
     assert mock_cache.redis.zcard(b"reports:myrepo:all:all") == 3
     assert mock_cache.redis.zcard(b"history:myrepo") == 3
@@ -108,6 +112,33 @@ def test_ingestion(mock_cache):
     ]
 
 
+def test_expiry(mock_cache):
+    """
+    Test expiry for platform & suite reports
+    """
+    mock_cache.bucket.add_mock_blob("myrepo/rev1/all:somesuite.json.zstd", coverage=1.0)
+    report_suite = Report(
+        mock_cache.reports_dir,
+        "myrepo",
+        "rev1",
+        platform="all",
+        suite="somesuite",
+        date=1000,
+        push_id=1,
+    )
+    mock_cache.ingest_report(report_suite)
+    assert report_suite.ttl == 1296000
+    assert mock_cache.redis.ttl(report_suite.key_overall) > 0
+
+    mock_cache.bucket.add_mock_blob("myrepo/rev1/win:all.json.zstd", coverage=1.0)
+    report_platform = Report(
+        mock_cache.reports_dir, "myrepo", "rev1", platform="win", date=2000, push_id=2
+    )
+    mock_cache.ingest_report(report_platform)
+    assert report_platform.ttl == 1296000
+    assert mock_cache.redis.ttl(report_platform.key_overall) > 0
+
+
 def test_ingest_hgmo(mock_cache, mock_hgmo):
     """
     Test ingestion using a mock HGMO