_MultiPage.py

import math
import re
import weakref
from time import time

import dateutil.parser
import humanize

from ._util import _formatDuration


# Handles data multi-page downloads (scalardata, rawdata, archivefiles)
class _MultiPage:
    def __init__(self, parent: object):
        self.parent = weakref.ref(parent)
        self.result = None

    def getAllPages(self, service: str, url: str, filters: dict):
        """
        Requests all pages from the service, with the url and filters
        Multiple pages will be downloaded until completed
        @return: Service response with concatenated data for all pages obtained
        """
        # pop archivefiles extension
        extension = None
        if service.startswith("archivefile") and "extension" in filters:
            extension = filters["extension"]
            del filters["extension"]

        # download first page
        start = time()
        response, responseTime = self._doPageRequest(url, filters, service, extension)
        rNext = response["next"]

        if rNext is not None:
            print(
                "Data quantity is greater than the row limit and",
                "will be downloaded in multiple pages.",
            )

            pageCount = 1
            pageEstimate = self._estimatePages(response, service)
            if pageEstimate > 0:
                # Exclude the first page when calculating the time estimation
                timeEstimate = _formatDuration((pageEstimate - 1) * responseTime)
                print(
                    f"Downloading time for the first page: {humanize.naturaldelta(responseTime)}"  # noqa: E501
                )
                print(f"Estimated approx. {pageEstimate} pages in total.")
                print(
                    f"Estimated approx. {timeEstimate} to complete for the rest of the pages."  # noqa: E501
                )

            # keep downloading pages until next is None
            print("")
            while rNext is not None:
                pageCount += 1
                rowCount = self._rowCount(response, service)

                print(f"   ({rowCount} samples) Downloading page {pageCount}...")
                nextResponse, nextTime = self._doPageRequest(
                    url, rNext["parameters"], service, extension
                )
                rNext = nextResponse["next"]

                # concatenate new data obtained
                self._catenateData(response, nextResponse, service)

            totalTime = _formatDuration(time() - start)
            print(
                f"   ({self._rowCount(response, service):d} samples)"
                f" Completed in {totalTime}."
            )
            response["next"] = None

        return response

    def _doPageRequest(
        self, url: str, filters: dict, service: str, extension: str = None
    ):
        """
        Wraps the _doRequest method
        Performs additional processing of the response for certain services
        @param extension: Only provide for archivefiles filtering
        Returns a tuple (jsonResponse, duration)
        """
        if service.startswith("archivefile"):
            response, duration = self.parent()._doRequest(url, filters, getTime=True)
            response = self.parent()._filterByExtension(response, extension)
        else:
            response, duration = self.parent()._doRequest(url, filters, getTime=True)

        return response, duration

    def _catenateData(self, response: object, nextResponse: object, service: str):
        """
        Concatenates the data results from nextResponse into response
        Compatible with the row structure of different services
        """
        if service.startswith("scalardata"):
            keys = response["sensorData"][0]["data"].keys()

            for sensorData in response["sensorData"]:
                sensorCode = sensorData["sensorCode"]

                nextSensor = next(
                    ns
                    for ns in nextResponse["sensorData"]
                    if ns["sensorCode"] == sensorCode
                )
                for key in keys:
                    sensorData["data"][key] += nextSensor["data"][key]

        elif service.startswith("rawdata"):
            for key in response["data"]:
                response["data"][key] += nextResponse["data"][key]

        elif service.startswith("archivefile"):
            response["files"] += nextResponse["files"]

    def _estimatePages(self, response: object, service: str):
        """
        Estimate the number of pages the request will require.

        It is calculated from the first page's response and its duration.

        Parameters
        ----------
        responseTime : float
            Request duration in seconds.
        """
        # timespan covered by the data in the response
        pageTimespan = self._responseTimespan(response, service)
        if pageTimespan == 0:
            return 0

        # total timespan to cover in the next parameter excluding the first page
        totalBegin = dateutil.parser.parse(
            response["next"]["parameters"]["dateFrom"], ignoretz=True
        )
        totalEnd = dateutil.parser.parse(
            response["next"]["parameters"]["dateTo"], ignoretz=True
        )
        totalTimespan = totalEnd - totalBegin

        # handle cases of very small timeframes
        pageSeconds = max(pageTimespan.total_seconds(), 1)
        totalSeconds = totalTimespan.total_seconds()

        # plus one for the first page
        return math.ceil(totalSeconds / pageSeconds) + 1

    def _rowCount(self, response, service: str):
        """
        Returns the number of records in the response
        """
        if service.startswith("scalardata"):
            return len(response["sensorData"][0]["data"]["sampleTimes"])

        elif service.startswith("rawdata"):
            return len(response["data"]["times"])

        elif service.startswith("archivefile"):
            return len(response["files"])

        return 0

    def _responseTimespan(self, response, service: str):
        """
        Determines the timespan the data in the response covers
        Returns a timedelta object
        """
        # grab the first and last sample times

        if service.startswith("scalardata"):
            first = response["sensorData"][0]["data"]["sampleTimes"][0]
            last = response["sensorData"][0]["data"]["sampleTimes"][-1]

        elif service.startswith("rawdata"):
            first = response["data"]["times"][0]
            last = response["data"]["times"][-1]

        elif service.startswith("archivefile"):
            row0 = response["files"][0]
            if isinstance(row0, str):
                regExp = r"\d{8}T\d{6}\.\d{3}Z"
                reFirst = re.search(regExp, response["files"][0])
                reLast = re.search(regExp, response["files"][-1])
                first = reFirst.group()
                last = reLast.group()
                if (
                    reFirst is None
                    or reLast is None
                    or reFirst.group() == reLast.group()
                ):
                    return 0
            else:
                first = response["files"][0]["dateFrom"]
                last = response["files"][-1]["dateFrom"]

        # compute the timedelta
        dateFirst = dateutil.parser.parse(first)
        dateLast = dateutil.parser.parse(last)
        return dateLast - dateFirst