feat: add automatic S3 upload for HTML reports

Author: KateOlaz

src/ps_helper/extensions/metrics_extension.py

@@ -4,6 +4,8 @@
 import math
 import datetime
 from collections import defaultdict
+from ..scripts.generate_report import generate_html_report
+from ..scripts.utils import upload_html_to_s3
 
 from scrapy import signals
 from pydantic import ValidationError
@@ -202,3 +204,30 @@ def spider_closed(self, spider, reason):
 
         spider.logger.info(f"Saved metrics: {file_path}")
         print(json.dumps(metrics, indent=2, ensure_ascii=False))
+
+        if os.getenv('PRODUCTION') == 'True':
+            try:
+                _, html_content = generate_html_report(file_path)
+
+                url = self._upload_report_to_s3(html_content, spider)
+                spider.logger.info(f"Report uploaded to S3: {url}")
+
+            except Exception as e:
+                spider.logger.error(f"Failed to generate/upload HTML report: {e}")
+
+    def _upload_report_to_s3(self, html_content, spider):
+        """Upload HTML report to S3 from memory"""
+
+        bucket_name = os.getenv('S3_BUCKET_NAME')
+        timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+        key = f"scrapy-reports/{spider.name}/{timestamp}-report.html"
+
+        url = upload_html_to_s3(
+            html_str=html_content,
+            bucket=bucket_name,
+            key=key,
+            publico=False,
+            expira_seg=3 * 24 * 3600
+        )
+
+        return url

src/ps_helper/scripts/generate_report.py

@@ -926,4 +926,4 @@ def _generate_retry_reasons_html(data):
     with open(output_path, "w", encoding="utf-8") as f:
         f.write(html_template)
 
-    return output_path
+    return output_path, html_template

src/ps_helper/scripts/utils.py

@@ -0,0 +1,50 @@
+import uuid
+import boto3
+
+
+def upload_html_to_s3(
+    html_str: str,
+    bucket: str,
+    key: str = None,
+    publico: bool = False,
+    expira_seg: int = 30 * 24 * 3600
+):
+    """
+    Uploads HTML to S3 from memory.
+
+    Args:
+        html_str: HTML content as string
+        bucket: S3 bucket name
+        key: S3 path (optional, generates UUID if not provided)
+        publico: If True, makes the file public
+        expira_seg: Expiration seconds for private URL (default 30 days)
+
+    Returns:
+        str: File URL (public or signed)
+    """
+
+    s3 = boto3.client("s3")
+    key = key or f"reports/{uuid.uuid4()}.html"
+
+    put_args = dict(
+        Bucket=bucket,
+        Key=key,
+        Body=html_str.encode("utf-8"),
+        ContentType="text/html; charset=utf-8"
+    )
+
+    if publico:
+        put_args["ACL"] = "public-read"
+
+    s3.put_object(**put_args)
+
+    if publico:
+        url = f"https://{bucket}.s3.amazonaws.com/{key}"
+    else:
+        url = s3.generate_presigned_url(
+            "get_object",
+            Params={"Bucket": bucket, "Key": key},
+            ExpiresIn=expira_seg
+        )
+
+    return url