From 5b4c06ba12adac6b4460a7ce43708ec0eac21a7b Mon Sep 17 00:00:00 2001 From: ybnd Date: Thu, 28 Jan 2021 06:51:40 +0100 Subject: [PATCH 1/4] Catch unrecognized arguments to subcommands --- shapeflow/cli.py | 5 +++++ test/test_cli.py | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/shapeflow/cli.py b/shapeflow/cli.py index 205c01b9..0f3ffd56 100644 --- a/shapeflow/cli.py +++ b/shapeflow/cli.py @@ -97,6 +97,11 @@ def __init__(self, args: OptArgs = None): args = sys.argv[1:] try: self.args, self.sub_args = self._parse(args) + + # only the root Command is allowed to pass on sub_args + if len(self.sub_args) > 0 and hasattr(self, "__command__"): + raise CliError(f"unrecognized argument(s) {self.sub_args}") + self.command() except argparse.ArgumentError: raise CliError diff --git a/test/test_cli.py b/test/test_cli.py index bf7189b5..f6c52152 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -39,7 +39,6 @@ def test_valid_commands(self): def test_invalid_commands(self): self.assertRaises(shapeflow.cli.CliError, shapeflow.cli.Sf, ['sevre']) - self.assertRaises(shapeflow.cli.CliError, shapeflow.cli.Sf, ['']) self.assertRaises(shapeflow.cli.CliError, shapeflow.cli.Sf, ['dmp']) self.assertRaises(shapeflow.cli.CliError, shapeflow.cli.Sf, ['nope']) From 5edbbd3edb81d21291879bb370a1bf7d8c370df1 Mon Sep 17 00:00:00 2001 From: ybnd Date: Thu, 28 Jan 2021 06:24:39 +0100 Subject: [PATCH 2/4] Untangle shapeflow/__init__ & shapeflow/core/__init__ --- requirements.txt | 4 +- shapeflow/__init__.py | 631 +--------------------- shapeflow/api.py | 26 +- shapeflow/cli.py | 16 +- shapeflow/config.py | 37 +- shapeflow/core/__init__.py | 534 ------------------ shapeflow/core/backend.py | 302 +---------- shapeflow/core/caching.py | 46 ++ shapeflow/core/config.py | 115 +++- shapeflow/core/db.py | 7 +- shapeflow/core/dispatching.py | 277 ++++++++++ shapeflow/core/features.py | 270 +++++++++ shapeflow/core/interface.py | 10 +- shapeflow/core/logging.py | 120 ++++ shapeflow/core/streaming.py | 23 +- shapeflow/db.py | 13 +- shapeflow/main.py | 36 +- shapeflow/maths/colors.py | 4 +- shapeflow/plugins/Area_mm2.py | 3 +- shapeflow/plugins/BackgroundFilter.py | 2 +- shapeflow/plugins/HsvRangeFilter.py | 2 +- shapeflow/plugins/PerspectiveTransform.py | 2 +- shapeflow/plugins/PixelSum.py | 3 +- shapeflow/plugins/Volume_uL.py | 3 +- shapeflow/plugins/__init__.py | 8 +- shapeflow/server.py | 9 +- shapeflow/settings.py | 518 ++++++++++++++++++ shapeflow/util/__init__.py | 116 ++++ shapeflow/video.py | 14 +- test/test_config.py | 6 +- test/test_endpoints.py | 2 +- test/test_main.py | 8 +- test/test_plugins.py | 11 +- test/test_server.py | 9 +- test/test_streaming.py | 11 +- test/test_video.py | 2 +- 36 files changed, 1622 insertions(+), 1578 deletions(-) create mode 100644 shapeflow/core/caching.py create mode 100644 shapeflow/core/dispatching.py create mode 100644 shapeflow/core/features.py create mode 100644 shapeflow/core/logging.py create mode 100644 shapeflow/settings.py diff --git a/requirements.txt b/requirements.txt index f83ca749..bb6f7843 100755 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ cffi diskcache numpy OnionSVG -opencv-python==4.2.0.32 +opencv-python pandas openpyxl pyyaml @@ -16,4 +16,4 @@ unidecode pydantic SQLAlchemy requests -shortuuid \ No newline at end of file +shortuuid diff --git a/shapeflow/__init__.py b/shapeflow/__init__.py index 83b41701..74b7103d 100644 --- a/shapeflow/__init__.py +++ b/shapeflow/__init__.py @@ -1,39 +1,17 @@ -"""Common elements. - -* Application settings - -* Logging - -* Caching -""" - import os -import glob -import shutil -import pathlib -import copy -import re -import sqlite3 -import datetime import logging -from multiprocessing import cpu_count - -from typing import Dict, Any, Type from pathlib import Path -from enum import Enum - -from contextlib import contextmanager -import yaml - -from pydantic import BaseModel, Field, FilePath, DirectoryPath, validator -import diskcache +__version__: str = '0.4.3' +import pathlib -__version__: str = '0.4.3' """Library version """ +VDEBUG = 9 +logging.addLevelName(VDEBUG, "VDEBUG") + # Get root directory _user_dir = pathlib.Path.home() if os.name == 'nt': # if running on Windows @@ -49,606 +27,9 @@ Windows: ``C:\\Users\\\\AppData\\Roaming\\shapeflow`` """ -_SETTINGS_FILE = ROOTDIR / 'settings.yaml' - if not ROOTDIR.is_dir(): _path = _user_dir for _subdir in _subdirs: _path = _path / _subdir if not _path.is_dir(): - _path.mkdir() - - -class _Settings(BaseModel): - """Abstract application settings - """ - - class Config: - validate_assignment = True - - def to_dict(self) -> dict: - """ - Returns - ------- - dict - Application settings as a dict - - """ - d: dict = {} - for k,v in self.__dict__.items(): - if isinstance(v, _Settings): - d.update({ - k:v.to_dict() - }) - elif isinstance(v, Enum): - d.update({ - k:v.value - }) - elif isinstance(v, Path): - d.update({ - k:str(v) # type: ignore - }) # todo: fix ` Dict entry 0 has incompatible type "str": "str"; expected "str": "Dict[str, Any]" ` - else: - d.update({ - k:v - }) - return d - - @contextmanager - def override(self, overrides: dict): # todo: consider deprecating in favor of mocks - """Override some parameters of the settings in a context. - Settings will only be modified within this context and restored to - their previous values afterwards. - Usage:: - with settings.override({"parameter": "override value"}): - - - Parameters - ---------- - overrides: dict - A ``dict`` mapping field names to values with which to - override those fields - """ - originals: dict = {} - try: - for attribute, value in overrides.items(): - originals[attribute] = copy.deepcopy( - getattr(self, attribute) - ) - setattr(self, attribute, value) - yield - finally: - for attribute, original in originals.items(): - setattr(self, attribute, original) - - @classmethod - def _validate_filepath(cls, value): - if not isinstance(value, Path): - value = Path(value) - - if not value.exists() and not value.is_file(): - value.touch() - - return value - - @classmethod - def _validate_directorypath(cls, value): - if not isinstance(value, Path): - value = Path(value) - - if not value.exists() and not value.is_dir(): - value.mkdir() - - return value - - @classmethod - def schema(cls, by_alias: bool = True, ref_template: str = '') -> Dict[str, Any]: - """Inject title & description into ``pydantic`` schema. - - These get lost due to some `bug`_ with ``Enum``. - - .. _bug: https://github.com/samuelcolvin/pydantic/pull/1749 - """ - - schema = super().schema(by_alias) - - def _inject(class_: Type[_Settings], schema, definitions): - for field in class_.__fields__.values(): - if 'properties' in schema and field.alias in schema['properties']: - if 'title' not in schema['properties'][field.alias]: - schema['properties'][field.alias][ - 'title'] = field.field_info.title - if field.field_info.description is not None and 'description' not in schema['properties'][field.alias]: - schema['properties'][field.alias]['description'] = field.field_info.description - if issubclass(field.type_, _Settings): - # recurse into nested _Settings classes - _inject(field.type_, definitions[field.type_.__name__], definitions) - return schema - - return _inject(cls, schema, schema['definitions']) - - -class FormatSettings(_Settings): - """Formatting settings - """ - datetime_format: str = Field(default='%Y/%m/%d %H:%M:%S.%f', title="date/time format") - """Base ``datetime`` `format string `_. - Defaults to ``'%Y/%m/%d %H:%M:%S.%f'``. - - .. _dtfs: https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior - """ - datetime_format_fs: str = Field(default='%Y-%m-%d_%H-%M-%S', title="file system date/time format") - """Filesystem-safe ``datetime`` `format string `_. - Used to append date & time to file names. - Defaults to ``'%Y-%m-%d_%H-%M-%S'``. - - .. _dtfs: https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior - """ - - -VDEBUG = 9 -logging.addLevelName(VDEBUG, "VDEBUG") - - -class LoggingLevel(str, Enum): - """Logging level. - """ - critical = "critical" - """Only log critical (unrecoverable) errors - """ - error = "error" - """Only log errors - """ - warning = "warning" - """Only log warnings (or errors) - """ - info = "info" - """Log general information - """ - debug = "debug" - """Log debugging information - """ - vdebug = "vdebug" - """Log verbose debugging information - """ - - @property - def level(self) -> int: - """Return the ``int`` logging level for compatibility with built-in - ``logging`` library - """ - _levels: dict = { - LoggingLevel.critical: logging.CRITICAL, - LoggingLevel.error: logging.ERROR, - LoggingLevel.warning: logging.WARNING, - LoggingLevel.info: logging.INFO, - LoggingLevel.debug: logging.DEBUG, - LoggingLevel.vdebug: VDEBUG - } - return _levels[self] - - -class LogSettings(_Settings): - """Logging settings - """ - path: FilePath = Field(default=str(ROOTDIR / 'current.log'), title='running log file') - """The application logs to this file - """ - dir: DirectoryPath = Field(default=str(ROOTDIR / 'log'), title='log file directory') - """This is the log directory. Logs from previous runs are stored here. - """ - keep: int = Field(default=16, title="# of log files to keep") - """The applications stores a number of old logs. - - When the amount of log files in :attr:`shapeflow.LogSettings.dir` exceeds - this number, the oldest files are deleted. - """ - lvl_console: LoggingLevel = Field(default=LoggingLevel.info, title="logging level (Python console)") - """The level at which the application logs to the Python console. - - Defaults to :attr:`~shapeflow.LoggingLevel.info` to keep the console from - getting too spammy. - Set to a lower level such as :attr:`~shapeflow.LoggingLevel.debug` to show - more detailed logs in the console. - """ - lvl_file: LoggingLevel = Field(default=LoggingLevel.debug, title="logging level (file)") - """The level at which the application logs to the log file at - :attr:`~shapeflow.LogSettings.path`. - - Defaults to :attr:`shapeflow.LoggingLevel.debug`. - """ - - _validate_path = validator('path', allow_reuse=True, pre=True)(_Settings._validate_filepath) - _validate_dir = validator('dir', allow_reuse=True, pre=True)(_Settings._validate_directorypath) - - -class CacheSettings(_Settings): - """Caching settings - """ - do_cache: bool = Field(default=True, title="use the cache") - """Enables the cache. Set to ``True`` by default. - - Disabling the cache will make the application significantly slower. - """ - dir: DirectoryPath = Field(default=str(ROOTDIR / 'cache'), title="cache directory") - """Where to keep the cache - """ - size_limit_gb: float = Field(default=4.0, title="cache size limit (GB)") - """How big the cache is allowed to get - """ - resolve_frame_number: bool = Field(default=True, title="resolve to (nearest) cached frame numbers") - """Whether to resolve frame numbers to the nearest requested frame numbers. - - Increases seeking performance, but may make it a bit more 'jumpy' if a low - number of frames is requested for an analysis. - """ - block_timeout: float = Field(default=0.1, title="wait for blocked item (s)") - """How long to keep waiting for data that's actively being committed to the - cache before giving up and computing it instead. - - In the rare case that two cachable data requests are being processed at the - same time, the first request will block the cache for those specific data - and cause the second request to wait until it can grab this data from the - cache. - This timeout prevents the second request from waiting forever until the - first request finishes (for example, in case it crashes). - """ - reset_on_error: bool = Field(default=False, title="reset the cache if it can't be opened") - """Clear the cache if it can't be opened. - - In rare cases, ``diskcache`` may cache data in a way it can't read back. - To recover from such an error, the cache will be cleared completely. - The only downside to this is decreased performance for a short while. - """ - - _validate_dir = validator('dir', allow_reuse=True, pre=True)(_Settings._validate_directorypath) - - -class RenderSettings(_Settings): - """Rendering settings - """ - dir: DirectoryPath = Field(default=str(ROOTDIR / 'render'), title="render directory") - """The directory where SVG files should be rendered to - """ - keep: bool = Field(default=False, title="keep files after rendering") - """Keep rendered images after they've been used. - - Disabled by default, you may want to enable this if you want to inspect the - renders. - """ - - _validate_dir = validator('dir', allow_reuse=True, pre=True)(_Settings._validate_directorypath) - - -class DatabaseSettings(_Settings): - """Database settings. - """ - path: FilePath = Field(default=str(ROOTDIR / 'history.db'), title="database file") - """The path to the database file - """ - cleanup_interval: int = Field(default=7, title='clean-up interval (days)') - """The database can get cluttered after a while, and will be cleaned at - this interval - """ - - _validate_path = validator('path', allow_reuse=True, pre=True)(_Settings._validate_filepath) - - -class ResultSaveMode(str, Enum): - """Where (or whether) to save the results of an analysis - """ - skip = "skip" - """Don't save results at all - """ - next_to_video = "next to video file" - """Save results in the same directory as the video file that was analyzed - """ - next_to_design = "next to design file" - """Save results in the same directory as the design file that was analyzed - """ - directory = "in result directory" - """Save results in their own directory at - :attr:`shapeflow.ApplicationSettings.result_dir` - """ - - -class ApplicationSettings(_Settings): - """Application settings. - """ - save_state: bool = Field(default=True, title="save application state on exit") - """Whether to save the application state when exiting the application - """ - load_state: bool = Field(default=True, title="load application state on start") - """Whether to load the application state when starting the application - """ - state_path: FilePath = Field(default=str(ROOTDIR / 'state'), title="application state file") - """Where to save the application state - """ - recent_files: int = Field(default=16, title="# of recent files to fetch") - """The number of recent files to show in the user interface - """ - video_pattern: str = Field(default="*.mp4 *.avi *.mov *.mpv *.mkv", title="video file pattern") - """Recognized video file extensions. - Defaults to ``"*.mp4 *.avi *.mov *.mpv *.mkv"``. - """ - design_pattern: str = Field(default="*.svg", title="design file pattern") - """Recognized design file extensions. - Defaults to ``"*.svg"``. - """ - save_result_auto: ResultSaveMode = Field(default=ResultSaveMode.next_to_video, title="result save mode (auto)") - """Where or whether to save results after each run of an analysis - """ - save_result_manual: ResultSaveMode = Field(default=ResultSaveMode.next_to_video, title="result save mode (manual)") - """Where or whether to save results that are exported manually - via the user interface - """ - result_dir: DirectoryPath = Field(default=str(ROOTDIR / 'results'), title="result directory") - """The path to the result directory - """ - cancel_on_q_stop: bool = Field(default=False, title="cancel running analyzers when stopping queue") - """Whether to cancel the currently running analysis when stopping a queue. - - Defaults to ``False``, i.e. the currently running analysis will be - completed first. - """ - threads: int = Field(default=cpu_count(), title="# of threads") - f"""The number of threads the server uses. Defaults to {cpu_count()}, the - number of logical cores of your machine's CPU. - """ - - _validate_dir = validator('result_dir', allow_reuse=True, pre=True)(_Settings._validate_directorypath) - _validate_state_path = validator('state_path', allow_reuse=True, pre=True)(_Settings._validate_filepath) - - @validator('threads', pre=True, allow_reuse=True) - def _validate_threads(cls, value): - if value < 8: - return 8 # At least 8 threads to run decently - else: - return value - - -class Settings(_Settings): - """``shapeflow`` settings. - - * app: :class:`~shapeflow.ApplicationSettings` - - * log: :class:`~shapeflow.LogSettings` - - * cache: :class:`~shapeflow.CacheSettings` - - * render: :class:`~shapeflow.RenderSettings` - - * format: :class:`~shapeflow.FormatSettings` - - * db: :class:`~shapeflow.DatabaseSettings` - """ - app: ApplicationSettings = Field(default=ApplicationSettings(), title="Application") - log: LogSettings = Field(default=LogSettings(), title="Logging") - cache: CacheSettings = Field(default=CacheSettings(), title="Caching") - render: RenderSettings = Field(default=RenderSettings(), title="SVG Rendering") - format: FormatSettings = Field(default=FormatSettings(), title="Formatting") - db: DatabaseSettings = Field(default=DatabaseSettings(), title="Database") - - @classmethod - def from_dict(cls, settings: dict): - for k in cls.__fields__.keys(): - if k not in settings: - settings.update({k:{}}) - - return cls( - **{field.name:field.type_(**settings[field.name]) - for field in cls.__fields__.values()} - ) - -settings: Settings -"""This global :class:`~shapeflow.Settings` object is used throughout the - library -""" - - -def _load_settings() -> Settings: # todo: if there are unexpected fields: warn, don't crash - """Load :class:`~shapeflow.Settings` from .yaml - """ - global settings - - if _SETTINGS_FILE.is_file(): - with open(_SETTINGS_FILE, 'r') as f: - settings_yaml = yaml.safe_load(f) - - # Get settings - if settings_yaml is not None: - settings = Settings.from_dict(settings_yaml) - else: - settings = Settings() - - # Move the previous log file to ROOTDIR/log - if Path(settings.log.path).is_file(): - shutil.move( - str(settings.log.path), # todo: convert to pathlib - os.path.join( - settings.log.dir, - datetime.datetime.fromtimestamp( - os.path.getmtime(settings.log.path) - ).strftime(settings.format.datetime_format_fs) + '.log' - ) - ) - - # If more files than specified in ini.log.keep, remove the oldest - files = glob.glob(os.path.join(settings.log.dir, '*.log')) # todo: convert to pathlib - files.sort(key=lambda f: os.path.getmtime(f), reverse=True) - while len(files) > settings.log.keep: - os.remove(files.pop()) - else: - settings = Settings() - - return settings - - -def save_settings(path: str = str(_SETTINGS_FILE)): - """Save :data:`~shapeflow.settings` to .yaml - """ - with open(path, 'w+') as f: - yaml.safe_dump(settings.to_dict(), f) - - -# Instantiate global settings object -_load_settings() -save_settings() - - -def update_settings(s: dict) -> dict: - """Update the global settings object. - - .. note:: - Just doing ``settings = Settings(**new_settings_dict)`` - would prevent other modules from accessing the updated settings! - - Parameters - ---------- - s : dict - new settings to integrate into the global settings - - Returns - ------- - dict - the current global settings as a ``dict`` - """ - global settings - - for cat, cat_new in s.items(): - sub = getattr(settings, cat) - for kw, val in cat_new.items(): - setattr(sub, kw, val) - - save_settings() - return settings.to_dict() - - -class Logger(logging.Logger): - """``shapeflow`` logger. - - * Adds a verbose debug logging level :func:`~shapeflow.Logger.vdebug` - - * Strips newlines from log output to keep each log event on its own line - """ - _pattern = re.compile(r'(\n|\r|\t| [ ]+)') - - def debug(self, msg, *args, **kwargs): - """:meta private:""" - super().debug(self._remove_newlines(msg)) - - def info(self, msg, *args, **kwargs): - """:meta private:""" - super().info(self._remove_newlines(msg)) - - def warning(self, msg, *args, **kwargs): - """:meta private:""" - super().warning(self._remove_newlines(msg)) - - def error(self, msg, *args, **kwargs): - """:meta private:""" - super().error(self._remove_newlines(msg)) - - def critical(self, msg, *args, **kwargs): - """:meta private:""" - super().critical(self._remove_newlines(msg)) - - def vdebug(self, message, *args, **kwargs): - """Log message with severity 'VDEBUG'. - A slightly more verbose debug level for really dense logs. - """ - if self.isEnabledFor(VDEBUG): - self.log( - VDEBUG, self._remove_newlines(message), *args, **kwargs - ) - - def _remove_newlines(self, msg: str) -> str: - return self._pattern.sub(' ', msg) - - -# Define log handlers -_console_handler = logging.StreamHandler() -_console_handler.setLevel(settings.log.lvl_console.level) - -_file_handler = logging.FileHandler(str(settings.log.path)) -_file_handler.setLevel(settings.log.lvl_file.level) - -_formatter = logging.Formatter( - '%(asctime)s - %(levelname)s - %(name)s - %(message)s' -) -_console_handler.setFormatter(_formatter) -_file_handler.setFormatter(_formatter) - -# Handle logs from other packages -waitress = logging.getLogger("waitress") -waitress.addHandler(_console_handler) -waitress.addHandler(_file_handler) -waitress.propagate = False - - -def get_logger(name: str) -> Logger: - """Get a new :class:`~shapeflow.Logger` object - - Parameters - ---------- - name : str - name of the logger - - Returns - ------- - Logger - a fresh logging handle - """ - logger = Logger(name) - # log at the _least_ restrictive level - logger.setLevel( - min([settings.log.lvl_console.level, settings.log.lvl_file.level]) - ) - - logger.addHandler(_console_handler) - logger.addHandler(_file_handler) - - logger.vdebug(f'new logger') - return logger - - -log = get_logger(__name__) -log.info(f"v{__version__}") -log.debug(f"settings: {settings.dict()}") - - -def get_cache(retry: bool = False) -> diskcache.Cache: - """Get a new :class:`diskcache.Cache` object - In some rare cases this can fail due to a corrupt cache. - If ``settings.cache.reset_on_error`` is on, and an exception is - raised the cache directory is removed and :func:`get_cache` is - called again with ``retry`` set to ``True``. - - Parameters - ---------- - retry : bool - Whether this call is a "retry call". - Defaults to ``False``, i.e. a first call - - Returns - ------- - diskcache.Cache - a fresh cache handle - - """ - try: - return diskcache.Cache( - directory=str(settings.cache.dir), - size_limit=settings.cache.size_limit_gb * 1e9 - ) - except sqlite3.OperationalError as e: - log.error(f"could not open cache - {e.__class__.__name__}: {str(e)}") - if not retry: - if settings.cache.reset_on_error: - log.error(f"removing cache directory") - shutil.rmtree(str(settings.cache.dir)) - log.error(f"trying to open cache again...") - get_cache(retry=True) - else: - log.error(f"could not open cache on retry") - raise e + _path.mkdir() \ No newline at end of file diff --git a/shapeflow/api.py b/shapeflow/api.py index a94abdf0..aa474e36 100644 --- a/shapeflow/api.py +++ b/shapeflow/api.py @@ -1,18 +1,16 @@ from typing import Dict, Optional, List, Callable, Tuple, Type, Any import numpy as np -import shortuuid -from shapeflow.core import Dispatcher, Endpoint, stream_image, stream_json, stream_plain -from shapeflow.util.meta import bind -from shapeflow.maths.colors import HsvColor -from shapeflow.core.streaming import BaseStreamer, EventStreamer, PlainFileStreamer +from shapeflow.core.dispatching import Endpoint, Dispatcher +from shapeflow.core.streaming import BaseStreamer, EventStreamer, \ + PlainFileStreamer, Stream # todo: also specify http methods maybe? class _VideoAnalyzerDispatcher(Dispatcher): """Dispatches ``/api/va//`` """ - status = Endpoint(Callable[[], dict], stream_json) + status = Endpoint(Callable[[], dict], Stream.json) """Get the analyzer's status :func:`shapeflow.core.backend.BaseAnalyzer.status` @@ -52,7 +50,7 @@ class _VideoAnalyzerDispatcher(Dispatcher): :func:`shapeflow.core.backend.BaseAnalyzer.cancel` """ - get_config = Endpoint(Callable[[], dict], stream_json) + get_config = Endpoint(Callable[[], dict], Stream.json) """Return the analyzer's configuration :func:`shapeflow.core.backend.BaseAnalyzer.get_config` @@ -107,7 +105,7 @@ class _VideoAnalyzerDispatcher(Dispatcher): :func:`shapeflow.video.VideoAnalyzer.get_overlay_png` """ - get_frame = Endpoint(Callable[[Optional[int]], np.ndarray], stream_image) + get_frame = Endpoint(Callable[[Optional[int]], np.ndarray], Stream.image) """Return the transformed frame at the provided frame number (or the current frame number if ``None``) @@ -119,18 +117,18 @@ class _VideoAnalyzerDispatcher(Dispatcher): :func:`shapeflow.video.VideoAnalyzer.set_filter_click` """ - get_inverse_transformed_overlay = Endpoint(Callable[[], np.ndarray], stream_image) + get_inverse_transformed_overlay = Endpoint(Callable[[], np.ndarray], Stream.image) """Return the inverse transformed overlay image :func:`shapeflow.video.VideoAnalyzer.get_inverse_transformed_overlay` """ - get_inverse_overlaid_frame = Endpoint(Callable[[Optional[int]], np.ndarray], stream_image) + get_inverse_overlaid_frame = Endpoint(Callable[[Optional[int]], np.ndarray], Stream.image) """Return the inverse overlaid frame at the provided frame number (or the current frame number if ``None``) :func:`shapeflow.video.VideoAnalyzer.get_inverse_overlaid_frame` """ - get_state_frame = Endpoint(Callable[[Optional[int], Optional[int]], np.ndarray], stream_image) + get_state_frame = Endpoint(Callable[[Optional[int], Optional[int]], np.ndarray], Stream.image) """Return the state frame at the provided frame number (or the current frame number if ``None``) @@ -183,7 +181,7 @@ class _VideoAnalyzerDispatcher(Dispatcher): :func:`shapeflow.video.VideoAnalyzer.get_fps` """ - get_raw_frame = Endpoint(Callable[[Optional[int]], np.ndarray], stream_image) + get_raw_frame = Endpoint(Callable[[Optional[int]], np.ndarray], Stream.image) """Return the raw frame at the provided frame number (or the current frame number if ``None``) @@ -381,7 +379,7 @@ class ApiDispatcher(Dispatcher): :func:`shapeflow.main._Main.set_settings` """ - events = Endpoint(Callable[[], EventStreamer], stream_json) + events = Endpoint(Callable[[], EventStreamer], Stream.json) """Open an event stream :func:`shapeflow.main._Main.events` @@ -391,7 +389,7 @@ class ApiDispatcher(Dispatcher): :func:`shapeflow.main._Main.stop_events` """ - log = Endpoint(Callable[[], PlainFileStreamer], stream_plain) + log = Endpoint(Callable[[], PlainFileStreamer], Stream.plain) """Open a log stream :func:`shapeflow.main._Main.log` diff --git a/shapeflow/cli.py b/shapeflow/cli.py index 0f3ffd56..a2e5c5f4 100644 --- a/shapeflow/cli.py +++ b/shapeflow/cli.py @@ -10,14 +10,16 @@ import socket import json import requests -import re import abc from pathlib import Path import argparse -import textwrap -from typing import List, Callable, Optional, Tuple, Union +from typing import List, Callable, Optional, Tuple + +from shapeflow import __version__ +from shapeflow.settings import settings +from shapeflow.core import get_logger +from shapeflow.core.logging import RootException -from shapeflow import __version__, get_logger, settings log = get_logger(__name__) # type aliases @@ -25,7 +27,7 @@ Parsing = Callable[[OptArgs], None] -class CliError(Exception): +class CliError(RootException): pass @@ -37,7 +39,7 @@ class IterCommand(abc.ABCMeta): """ __command__: str """Command name. This is how the command is addressed from the commandline. - """ # todo: nope, doesn't work' + """ def __str__(cls): try: @@ -277,7 +279,7 @@ class Dump(Command): ) def command(self): - from shapeflow.config import schemas + from shapeflow.main import schemas if not self.args.dir.is_dir(): log.warning(f"making directory '{self.args.dir}'") diff --git a/shapeflow/config.py b/shapeflow/config.py index 60df5283..ce7f60ba 100644 --- a/shapeflow/config.py +++ b/shapeflow/config.py @@ -1,15 +1,13 @@ -from typing import Optional, Tuple, Dict, Any, Type, Union +from typing import Optional, Tuple, Dict, Any import json from pydantic import Field, validator -from shapeflow import __version__, settings +from shapeflow import __version__ from shapeflow.core.config import extend, ConfigType, \ - log, VERSION, CLASS, untag, BaseConfig -from shapeflow.core.backend import BaseAnalyzerConfig, \ - FeatureType, FeatureConfig, AnalyzerState, QueueState -from shapeflow.core import EnforcedStr + log, VERSION, CLASS, untag, BaseConfig, EnforcedStr +from shapeflow.core.features import FeatureConfig, FeatureType from shapeflow.core.interface import FilterType, TransformType, TransformConfig, \ FilterConfig, HandlerConfig from shapeflow.maths.coordinates import Roi @@ -130,6 +128,15 @@ class DesignFileHandlerConfig(BaseConfig): _limit_alpha = validator('overlay_alpha', allow_reuse=True, pre=True)(BaseConfig._float_limits) +class BaseAnalyzerConfig(BaseConfig): + """Abstract analyzer configuration. + """ + video_path: Optional[str] = Field(default=None) + design_path: Optional[str] = Field(default=None) + name: Optional[str] = Field(default=None) + description: Optional[str] = Field(default=None) + + @extend(ConfigType) class VideoAnalyzerConfig(BaseAnalyzerConfig): """Video analyzer configuration @@ -267,24 +274,6 @@ def _validate_parameters(cls, value, values): _validate_fis = validator('frame_interval_setting')(BaseConfig._resolve_enforcedstr) -def schemas() -> Dict[str, dict]: - """Get the JSON schemas of - - * :class:`shapeflow.video.VideoAnalyzerConfig` - - * :class:`shapeflow.Settings` - - * :class:`shapeflow.core.backend.AnalyzerState` - - * :class:`shapeflow.core.backend.QueueState` - """ - return { - 'config': VideoAnalyzerConfig.schema(), - 'settings': settings.schema(), - 'analyzer_state': dict(AnalyzerState.__members__), - 'queue_state': dict(QueueState.__members__), - } - def loads(config: str) -> BaseConfig: """Load a configuration object from a JSON string. diff --git a/shapeflow/core/__init__.py b/shapeflow/core/__init__.py index b7cfeb0a..e69de29b 100644 --- a/shapeflow/core/__init__.py +++ b/shapeflow/core/__init__.py @@ -1,534 +0,0 @@ -import abc -import threading -from typing import Callable, Dict, List, Tuple, Type, Optional, _GenericAlias, Any # type: ignore -import collections -from contextlib import contextmanager - -import uuid - -from shapeflow import get_logger -from shapeflow.util.meta import bind - - -log = get_logger(__name__) - - -# todo: move up to shapeflow -class RootException(Exception): - """All ``shapeflow`` exceptions should be subclasses of this one. - Automatically logs the exception class and message at the ``ERROR`` level. - """ - msg = '' - """The message to log - """ - - def __init__(self, *args): - # https://stackoverflow.com/questions/49224770/ - # if no arguments are passed set the first positional argument - # to be the default message. To do that, we have to replace the - # 'args' tuple with another one, that will only contain the message. - # (we cannot do an assignment since tuples are immutable) - if not (args): - args = (self.msg,) - - log.error(self.__class__.__name__ + ': ' + ' '.join(args)) - super(Exception, self).__init__(*args) - - -class DispatchingError(RootException): - """An error dispatching a method call or exposing an endpoint. - """ - - -class EnforcedStr(str): - """A string that is enforced to be one of several options. - Works like a dynamic ``Enum`` -- options can be added at runtime. - """ - _options: List[str] = [''] - _descriptions: Dict[str, str] = {} - _str: str - - _default: Optional[str] = None - - def __init__(self, string: str = None): - super().__init__() - if string is not None: - if string not in self.options: - if string: - log.warning(f"Illegal {self.__class__.__name__} '{string}', " - f"should be one of {self.options}. " - f"Defaulting to '{self.default}'.") - self._str = str(self.default) - else: - self._str = str(string) - else: - self._str = str(self.default) - - def __repr__(self): - return f"<{self.__class__.__name__} '{self._str}'>" - - def __str__(self): - return str(self._str) # Make SURE it's a string :( - - def __eq__(self, other): - if hasattr(other, '_str'): - return self._str == other._str - elif isinstance(other, str): - return self._str == other - else: - return False - - @property - def options(self): - """The accepted options - """ - return self._options - - @property - def descriptions(self): - """The descriptions of each option - """ - return self._descriptions - - @property - def describe(self): - """The description of the currently selected option - """ - return self.descriptions[self._str] - - @property - def default(self): - """The default option for this :class:`~shapeflow.core.EnforcedStr` - """ - if self._default is not None: - return self._default - else: - return self._options[0] - - @classmethod - def set_default(cls, value: 'EnforcedStr') -> None: - """Explicitly sets the default. - - Parameters - ---------- - value : EnforcedStr - The default value to set - """ - if isinstance(value, cls) and value in cls().options: - log.debug(f"setting default of '{cls.__name__}' to '{value}'") - cls._default = value - else: - raise ValueError( - f"cannot set default of '{cls.__name__}' to '{value}'" - ) - - def __hash__(self): # todo: why? - return hash(str(self)) - - @classmethod - def __modify_schema__(cls, field_schema): - """Modify ``pydantic`` schema to include default, descriptions and - act as an ``Enum`` - """ - # pydantic - temp = cls() - field_schema.update( - enum=temp.options, - default=temp.default, - descriptions=temp.descriptions - ) - - -class _Streaming(EnforcedStr): - _options = ['off', 'image', 'json', 'plain'] - - -stream_off = _Streaming('off') -stream_image = _Streaming('image') -stream_json = _Streaming('json') -stream_plain = _Streaming('plain') - - -class Endpoint(object): - """An endpoint for an internal method. - """ - _name: str - _registered: bool - _signature: Type[Callable] - _method: Optional[Callable] - _streaming: _Streaming - _update: Optional[Callable[['Endpoint'], None]] - - def __init__(self, signature: _GenericAlias, streaming: _Streaming = stream_off): # todo: type Callable[] correctly - try: - assert signature.__origin__ == collections.abc.Callable - assert hasattr(signature, '__args__') - except Exception: - raise TypeError('Invalid Endpoint signature') - - self._method = None - self._update = None - self._registered = False - self._signature = signature - self._streaming = streaming - - def compatible(self, method: Callable) -> bool: - """Checks whether a method is compatible with the endpoint's signature - - Parameters - ---------- - method : Callable - Any method or function - - Returns - ------- - bool - ``True`` if the method is compatible, ``False`` if it isn't. - """ - if hasattr(method, '__annotations__'): - args: List = [] - for arg in self.signature: - if arg == type(None): - arg = None - args.append(arg) - # Don't be too pedantic unannotated None-type return - return tuple(method.__annotations__.values()) == tuple(args) - else: - return False - - def expose(self): - """ Expose a method at this endpoint. - Used as a decorator:: - @endpoint.expose() - def some_method(): - pass - """ - def wrapper(method): - if self._method is not None: - log.debug( # todo: add traceback - f"Exposing '{method.__qualname__}' at endpoint '{self.name}' will override " - f"previously exposed method '{self._method.__qualname__}'." - ) # todo: keep in mind we're also marking the methods themselves - - if not self.compatible(method): - raise DispatchingError( - f"Cannot expose '{method.__qualname__}' at endpoint '{self.name}'. " - f"Incompatible signature: {method.__annotations__} vs. {self.signature}" - ) - - method._endpoint = self - self._method = method - if self._update is not None: - self._update(self) - - return method - return wrapper - - @property - def method(self) -> Optional[Callable]: - """The method exposed at this endpoint. Can be ``None`` - """ - return self._method - - @property - def signature(self) -> tuple: - """The signature of this endpoint. - """ - return self._signature.__args__ # type: ignore - - @property - def streaming(self) -> _Streaming: - """What or whether this endpoint streams. - """ - return self._streaming - - @property - def registered(self) -> bool: - """Whether this endpoint is registered. - """ - return self._registered - - @property - def name(self) -> str: - """The name of this endpoint. - Taken from its attribute name in the object where it is registered. - """ - try: - return self._name - except AttributeError: - return '' - - def register(self, name: str, callback: Callable[['Endpoint'], None]): - """Register the endpoint in some other object. - """ - self._registered = True - self._name = name - self._update = callback - - -class Dispatcher(object): # todo: these should also register specific instances & handle dispatching? - """Dispatches requests to :class:`shapeflow.core.Endpoint` objects. - """ - _endpoints: Tuple[Endpoint, ...] #type: ignore - _dispatchers: Tuple['Dispatcher', ...] - - _name: str - _parent: Optional['Dispatcher'] - _address_space: Dict[str, Optional[Callable]] - - _update: Optional[Callable[['Dispatcher'], None]] - - _instance: Optional[object] - - def __init__(self, instance: object = None): - self._update = None - if instance is not None: - self._set_instance(instance) - else: - self._address_space = {} - self._endpoints = tuple() - self._dispatchers = tuple() - - @property - def name(self) -> str: - """The name of this dispatcher. - """ - try: - return self._name - except AttributeError: - return self.__class__.__name__ - - @property - def dispatchers(self) -> Tuple['Dispatcher', ...]: - """The dispatchers nested in this dispatcher. - """ - return self._dispatchers - - @property - def endpoints(self) -> Tuple[Endpoint, ...]: - """The endpoints contained in this dispatcher. - """ - return self._endpoints - - @property - def address_space(self) -> Dict[str, Optional[Callable]]: - """The address-method mapping of this dispatcher. - """ - return self._address_space - - def _set_instance(self, instance: object): - self._instance = instance - self._address_space = {} - self._endpoints = tuple() - self._dispatchers = tuple() - - for attr, val in self.__class__.__dict__.items(): - if isinstance(val, Endpoint): # todo: also register dispatchers - self._add_endpoint(attr, val) - elif isinstance(val, Dispatcher): - self._add_dispatcher(attr, val) - - def _register(self, name: str, callback: Callable[['Dispatcher'], None]): - """Register this dispatcher within another dispatcher. - """ - self._update = callback - self._name = name - - def _add_endpoint(self, name: str, endpoint: Endpoint): - endpoint.register(name=name, callback=self._update_endpoint) - - if endpoint.method is not None and self._instance is not None: - method = bind(self._instance, endpoint.method) - else: - method = endpoint.method - - self._address_space[name] = method - self._endpoints = tuple(list(self._endpoints) + [endpoint]) - setattr(self, name, endpoint) - - if self._update is not None: - self._update(self) - - def _add_dispatcher(self, name: str, dispatcher: 'Dispatcher'): - dispatcher._register(name=name, callback=self._update_dispatcher) - - self._address_space.update({ - "/".join([name, address]): method - for address, method in dispatcher.address_space.items() - if method is not None and "__" not in address - }) - self._dispatchers = tuple(list(self._dispatchers) + [dispatcher]) - setattr(self, name, dispatcher) - - if self._update is not None: - self._update(self) - - def _update_endpoint(self, endpoint: Endpoint) -> None: - self._address_space.update({ - endpoint.name: endpoint.method - }) - if self._update is not None: - self._update(self) - - def _update_dispatcher(self, dispatcher: 'Dispatcher') -> None: - self._address_space.update({ # todo: this doesn't take into account deleted keys! - "/".join([dispatcher.name, address]): method - for address, method in dispatcher.address_space.items() - if method is not None and "__" not in address - }) - - if self._update is not None: - self._update(self) - - def dispatch(self, address: str, *args, **kwargs) -> Any: - """Dispatch a request to a method. - - Parameters - ---------- - address : str - The address to dispatch to - args - Any positional arguments to pass on to the method - kwargs - Any keyword arguments to pass on to the method - - Returns - ------- - Any - Whatever the method returns. - """ - try: - method = self.address_space[address] - - if method is not None: - # todo: consider doing some type checking here, args/kwargs vs. method._endpoint.signature - return method(*args, **kwargs) - except KeyError: - raise DispatchingError( - f"'{self.name}' can't dispatch address '{address}'." - ) - - def __getitem__(self, item): - return getattr(self, item) - - -class Described(object): - """A class with a description. - - This description is taken from the first line of the docstring if there is - one or set to the name of the class if there isn't. - """ - @classmethod - def _description(cls): - if cls.__doc__ is not None: - return cls.__doc__.split('\n')[0] - else: - return cls.__name__ - - -class Lockable(object): - """Wrapper around :class:`threading.Lock` & :class:`threading.Event` - - Defines a :class:`~shapeflow.core.Lockable.lock` context to handle locking - and unlocking along with a ``_cancel`` and ``_error`` events to communicate - with :class:`~shapeflow.core.Lockable` objects from other threads. - - Doesn't need to initialize; lock & events are created when they're needed. - """ - _lock: threading.Lock - _cancel: threading.Event - _error: threading.Event - - @property - def _ensure_lock(self) -> threading.Lock: - try: - return self._lock - except AttributeError: - self._lock = threading.Lock() - return self._lock - - @property - def _ensure_cancel(self) -> threading.Event: - try: - return self._cancel - except AttributeError: - self._cancel = threading.Event() - return self._cancel - - @property - def _ensure_error(self) -> threading.Event: - try: - return self._error - except AttributeError: - self._error = threading.Event() - return self._error - - @contextmanager - def lock(self): - """Locking context. - - If ``_lock`` event doesn't exist yet it is instantiated first. - Upon exiting the context, the :class:`threading.Lock` object - is compared to the original to ensure that no shenanigans took place. - """ - log.vdebug(f"Acquiring lock {self}...") - locked = self._ensure_lock.acquire() - original_lock = self._lock - log.vdebug(f"Acquired lock {self}") - try: - log.vdebug(f"Locking {self}") - yield locked - finally: - log.vdebug(f"Unlocking {self}") - # Make 'sure' nothing weird happened to self._lock - assert self._lock == original_lock - self._lock.release() - - def cancel(self): - """Sets the ``_cancel`` event. - If ``_cancel`` event doesn't exist yet it is instantiated first. - """ - self._ensure_cancel.set() - - def error(self): - """Sets the ``_error`` event. - If ``_error`` event doesn't exist yet it is instantiated first. - """ - self._ensure_error.set() - - @property - def canceled(self) -> bool: - """Returns ``True`` if the ``_cancel`` event is set. - If ``_cancel`` event doesn't exist yet it is instantiated first. - """ - return self._ensure_cancel.is_set() - - @property - def errored(self) -> bool: - """Returns ``True`` if the ``_error`` event is set. - If ``_error`` event doesn't exist yet it is instantiated first. - """ - return self._ensure_error.is_set() - - def clear_cancel(self): - """Clears the ``_cancel`` event. - If ``_cancel`` event doesn't exist yet it is instantiated first. - """ - return self._ensure_cancel.clear() - - def clear_error(self): - """Clears the ``_error`` event. - If ``_error`` event doesn't exist yet it is instantiated first. - """ - return self._ensure_error.clear() - - -class RootInstance(Lockable): # todo: basically deprecated - _id: str - - def _set_id(self, id: str): - self._id = id - - @property - def id(self): - return self._id \ No newline at end of file diff --git a/shapeflow/core/backend.py b/shapeflow/core/backend.py index 2d13cf63..e65a2c06 100644 --- a/shapeflow/core/backend.py +++ b/shapeflow/core/backend.py @@ -1,32 +1,25 @@ from enum import IntEnum, Enum import diskcache -import sys import abc import time import threading from contextlib import contextmanager -from typing import Any, List, Optional, Tuple, Dict, Type, Mapping +from typing import Any, List, Optional, Dict, Type, Mapping -import numpy as np import pandas as pd -from pydantic import Field - -from shapeflow import settings, get_logger, get_cache +from shapeflow.config import BaseAnalyzerConfig +from shapeflow.core.caching import get_cache +from shapeflow.core.logging import get_logger, RootException from shapeflow.api import api -from shapeflow.core import RootException, RootInstance, Described -from shapeflow.maths.colors import Color, HsvColor, as_hsv from shapeflow.util.meta import describe_function -from shapeflow.util import Timer, Timing +from shapeflow.util import Timer, Timing, Described, Lockable from shapeflow.core.db import BaseAnalysisModel -from shapeflow.core.config import Factory, BaseConfig, Instance, Configurable +from shapeflow.core.config import Factory, BaseConfig, Instance from shapeflow.core.streaming import EventStreamer -from shapeflow.core.interface import InterfaceType - - log = get_logger(__name__) @@ -224,280 +217,11 @@ def _close_cache(self): self._cache.close() self._cache = None -class FeatureConfig(BaseConfig, abc.ABC): - """Abstract :class:`~shapeflow.core.backend.Feature` parameters""" - pass - - -class Feature(abc.ABC, Configurable): # todo: should probably use Config for parameters after all :) - """A feature implements interactions between BackendElements to - produce a certain value - """ - _color: Optional[Color] - _state: Optional[np.ndarray] - - _label: str = '' # todo: keep these in the config instead? - """Label string, to be used in exported data and the user interface - """ - _unit: str = '' - """Unit string, to be used in exported data and the user interface - """ - _elements: Tuple[Instance, ...] = () - - _config: Optional[FeatureConfig] - _global_config: FeatureConfig - _config_class: Type[FeatureConfig] = FeatureConfig - - def __init__(self, elements: Tuple[Instance, ...], global_config: FeatureConfig, config: Optional[dict] = None): - self._skip = False - self._ready = False - - self._elements = elements - self._global_config = global_config - - if config is not None: - self._config = global_config.__class__(**config) - else: - self._config = None - - self._color = HsvColor(h=0,s=200,v=255) # start out as red - - def calculate(self, frame: np.ndarray, state: np.ndarray = None) \ - -> Tuple[Any, Optional[np.ndarray]]: - """Calculate the feature for the given frame - - Parameters - ---------- - frame : np.ndarray - A video frame - state : Optional[np.ndarray] - An ``np.ndarray`` for the state image. Should have the same - dimensions as the ``frame`` - - Returns - ------- - Any - The calculated feature value - Optional[np.ndarray] - If a state image was provided, return this state frame with the - feature's state frame added onto it. If not, return ``None``. - """ - """Calculate the feature for given frame - """ - if state is not None: - state = self.state(frame, state) - return self.value(frame), state - - @classmethod - def label(cls) -> str: - """The label of this feature. Used in the user interface and results. - """ - return cls._label - @classmethod - def unit(cls) -> str: - """The unit of this feature. Used in the user interface and results. - """ - return cls._unit - - @property - def skip(self) -> bool: - """Whether this feature should be skipped - """ - raise NotImplementedError - - @property - def ready(self) -> bool: - """Whether this feature is ready to be calculated - """ - raise NotImplementedError - - def set_color(self, color: Color): - self._color = color - - @property - def color(self) -> Color: - """Color of the feature in figures. - - A feature's color is handled by its :class:`~shapeflow.core.backend.FeatureSet` - as not to overlap with any other features in that set. - """ - if self._color is not None: - return self._color - else: - raise ValueError - - @abc.abstractmethod - def _guideline_color(self) -> Color: - """Returns the 'guideline color' of a feature, which is used to resolve - to the final color within a feature set. - """ - raise NotImplementedError - - @abc.abstractmethod # todo: we're dealing with frames explicitly, so maybe this should be an shapeflow.video thing... - def state(self, frame: np.ndarray, state: np.ndarray) -> np.ndarray: - """Return the feature's state image for a given frame - """ - raise NotImplementedError - - @abc.abstractmethod - def value(self, frame: np.ndarray) -> Any: - """Compute the value of the feature for a given frame - """ - raise NotImplementedError - - @property - def config(self): - """The configuration of the feature. - Default to the global configuration if no specific one is provided. - """ - if self._config is not None: - return self._config - else: - return self._global_config - - -class FeatureSet(Configurable): - """A set of :class:`~shapeflow.core.backend.Feature` instances - """ - _feature: Tuple[Feature, ...] - _colors: Tuple[Color, ...] - _config_class = FeatureConfig - - def __init__(self, features: Tuple[Feature, ...]): - self._features = features - - def resolve_colors(self) -> Tuple[Color, ...]: - """Resolve the colors of all features in this set so that none of them - overlap. - """ - guideline_colors = [ - as_hsv(f._guideline_color()) for f in self._features - ] - - min_v = 20.0 - max_v = 255.0 - tolerance = 15 - - bins: list = [] - # todo: clean up binning - for index, color in enumerate(guideline_colors): - if not bins: - bins.append([index]) - else: - in_bin = False - for bin in bins: - if abs( - float(color.h) - - np.mean([guideline_colors[i].h for i in bin]) - ) < tolerance: - bin.append(index) - in_bin = True - break - if not in_bin: - bins.append([index]) - - for bin in bins: - if len(bin) < 4: - increment = 60.0 - else: - increment = (max_v - min_v) / len(bin) - - for repetition, index in enumerate(bin): - self._features[index].set_color( - HsvColor( - h=guideline_colors[index].h, - s=220, - v=int(max_v - repetition * increment) - ) - ) - - self._colors = tuple([feature.color for feature in self._features]) - return self.colors - - @property - def colors(self) -> Tuple[Color, ...]: - """The resolved colors in this feature set - """ - return self._colors - - @property - def features(self) -> Tuple[Feature, ...]: - """The features in this feature set - """ - return self._features - - def calculate(self, frame: np.ndarray, state: Optional[np.ndarray]) -> Tuple[List[Any], Optional[np.ndarray]]: - """Calculate all features in this set for a given frame - - Parameters - ---------- - frame : np.ndarray - An image - state : Optional[np.ndarray] - An empty ``np.ndarray`` for the state image. Should have the same - dimensions as the ``frame`` - - Returns - ------- - List[Any] - The calculated feature values - Optional[np.ndarray] - If a state image was provided, return the composite state image of - this feature set. If not, return ``None``. - """ - values = [] - - for feature in self.features: - value, state = feature.calculate(frame=frame, state=state) - values.append(value) - - return values, state - - -class FeatureType(InterfaceType): - """:class:`~shapeflow.core.backend.Feature` factory - """ - _type = Feature - _mapping: Mapping[str, Type[Feature]] = {} - _config_type = FeatureConfig - - def get(self) -> Type[Feature]: - """Get the :class:`~shapeflow.core.backend.Feature` for this feature type - """ - feature = super().get() - assert issubclass(feature, Feature) - return feature - - def config_schema(self) -> dict: - """The ``pydantic`` configuration schema for - this type of :class:`~shapeflow.core.backend.Feature` - """ - return self.get().config_schema() - - @classmethod - def __modify_schema__(cls, field_schema): - """Modify ``pydantic`` schema to include units and labels. - """ - super().__modify_schema__(field_schema) - field_schema.update( - units={ k:v._unit for k,v in cls._mapping.items() }, - labels={ k:v._label for k,v in cls._mapping.items() } - ) - - -class BaseAnalyzerConfig(BaseConfig): - """Abstract analyzer configuration. - """ - video_path: Optional[str] = Field(default=None) - design_path: Optional[str] = Field(default=None) - name: Optional[str] = Field(default=None) - description: Optional[str] = Field(default=None) - - -class BaseAnalyzer(Instance, RootInstance): +class BaseAnalyzer(Instance, Lockable): """Abstract analyzer. """ + _id: str _config: BaseAnalyzerConfig _state: int @@ -538,6 +262,13 @@ def __init__(self, config: BaseAnalyzerConfig = None, eventstreamer: EventStream self._cancel = threading.Event() self._error = threading.Event() + def _set_id(self, id: str): + self._id = id + + @property + def id(self): + return self._id + def set_model(self, model: BaseAnalysisModel): self._model = model if self.config.name is None: @@ -886,6 +617,9 @@ def export(self): def description(self): return self._description + def _set_id(self, id: str): + self._id = id + class AnalyzerType(Factory): """Analyzer type factory diff --git a/shapeflow/core/caching.py b/shapeflow/core/caching.py new file mode 100644 index 00000000..a6f8ffa9 --- /dev/null +++ b/shapeflow/core/caching.py @@ -0,0 +1,46 @@ +import shutil +import sqlite3 + +import diskcache + +from shapeflow.core.logging import get_logger + + +log = get_logger(__name__) + + +def get_cache(retry: bool = False) -> diskcache.Cache: + """Get a new :class:`diskcache.Cache` object + In some rare cases this can fail due to a corrupt cache. + If ``settings.cache.reset_on_error`` is on, and an exception is + raised the cache directory is removed and :func:`get_cache` is + called again with ``retry`` set to ``True``. + + Parameters + ---------- + retry : bool + Whether this call is a "retry call". + Defaults to ``False``, i.e. a first call + + Returns + ------- + diskcache.Cache + a fresh cache handle + + """ + try: + return diskcache.Cache( + directory=str(settings.cache.dir), + size_limit=settings.cache.size_limit_gb * 1e9 + ) + except sqlite3.OperationalError as e: + log.error(f"could not open cache - {e.__class__.__name__}: {str(e)}") + if not retry: + if settings.cache.reset_on_error: + log.error(f"removing cache directory") + shutil.rmtree(str(settings.cache.dir)) + log.error(f"trying to open cache again...") + get_cache(retry=True) + else: + log.error(f"could not open cache on retry") + raise e diff --git a/shapeflow/core/config.py b/shapeflow/core/config.py index 6ffa6881..b39430d6 100644 --- a/shapeflow/core/config.py +++ b/shapeflow/core/config.py @@ -1,17 +1,13 @@ import abc import copy -import json +from typing import Optional, Union, Type, Dict, Mapping, List import numpy as np -from typing import Optional, Union, Type, Dict, Any, Mapping, List -from functools import partial - -from shapeflow import get_logger, __version__ -from shapeflow.core import EnforcedStr, Described -from shapeflow.util import ndarray2str, str2ndarray - -from pydantic import BaseModel, Field, root_validator, validator +from pydantic import BaseModel +from shapeflow import __version__ +from shapeflow.core.logging import get_logger +from shapeflow.util import ndarray2str, Described log = get_logger(__name__) @@ -28,7 +24,105 @@ __meta_sheet__ = 'metadata' -# todo: move up to shapeflow.core +class EnforcedStr(str): + """A string that is enforced to be one of several options. + Works like a dynamic ``Enum`` -- options can be added at runtime. + """ + _options: List[str] = [''] + _descriptions: Dict[str, str] = {} + _str: str + + _default: Optional[str] = None + + def __init__(self, string: str = None): + super().__init__() + if string is not None: + if string not in self.options: + if string: + log.warning(f"Illegal {self.__class__.__name__} '{string}', " + f"should be one of {self.options}. " + f"Defaulting to '{self.default}'.") + self._str = str(self.default) + else: + self._str = str(string) + else: + self._str = str(self.default) + + def __repr__(self): + return f"<{self.__class__.__name__} '{self._str}'>" + + def __str__(self): + return str(self._str) # Make SURE it's a string :( + + def __eq__(self, other): + if hasattr(other, '_str'): + return self._str == other._str + elif isinstance(other, str): + return self._str == other + else: + return False + + @property + def options(self): + """The accepted options + """ + return self._options + + @property + def descriptions(self): + """The descriptions of each option + """ + return self._descriptions + + @property + def describe(self): + """The description of the currently selected option + """ + return self.descriptions[self._str] + + @property + def default(self): + """The default option for this :class:`~shapeflow.core.EnforcedStr` + """ + if self._default is not None: + return self._default + else: + return self._options[0] + + @classmethod + def set_default(cls, value: 'EnforcedStr') -> None: + """Explicitly sets the default. + + Parameters + ---------- + value : EnforcedStr + The default value to set + """ + if isinstance(value, cls) and value in cls().options: + log.debug(f"setting default of '{cls.__name__}' to '{value}'") + cls._default = value + else: + raise ValueError( + f"cannot set default of '{cls.__name__}' to '{value}'" + ) + + def __hash__(self): # todo: why? + return hash(str(self)) + + @classmethod + def __modify_schema__(cls, field_schema): + """Modify ``pydantic`` schema to include default, descriptions and + act as an ``Enum`` + """ + # pydantic + temp = cls() + field_schema.update( + enum=temp.options, + default=temp.default, + descriptions=temp.descriptions + ) + + class Factory(EnforcedStr): # todo: add a _class & issubclass check """An enforced string which maps its options to types. @@ -379,7 +473,6 @@ def config_schema(cls): """ return cls.config_class().schema() - class Instance(Configurable): # todo: why isn't this just in Configurable? _config: BaseConfig diff --git a/shapeflow/core/db.py b/shapeflow/core/db.py index 5dd9a197..729c134f 100644 --- a/shapeflow/core/db.py +++ b/shapeflow/core/db.py @@ -3,7 +3,7 @@ import os import time from contextlib import contextmanager -from typing import Optional, List, Type, Any, Tuple +from typing import Optional, List, Any, Tuple import datetime from sqlalchemy.ext.declarative import declarative_base @@ -13,9 +13,8 @@ from sqlalchemy.sql.sqltypes import Integer, String, DateTime from sqlalchemy.exc import InvalidRequestError -from shapeflow import get_logger -from shapeflow.core import RootException, RootInstance, Lockable -from shapeflow.util import hash_file +from shapeflow.core.logging import get_logger, RootException +from shapeflow.util import hash_file, Lockable log = get_logger(__name__) Base = declarative_base() diff --git a/shapeflow/core/dispatching.py b/shapeflow/core/dispatching.py new file mode 100644 index 00000000..673378e4 --- /dev/null +++ b/shapeflow/core/dispatching.py @@ -0,0 +1,277 @@ +import collections +from typing import Type, Callable, Optional, List, Tuple, Dict, Any, _GenericAlias + +from shapeflow.core.logging import get_logger, RootException +from shapeflow.core.streaming import Stream +from shapeflow.util.meta import bind + + +log = get_logger(__name__) + + +class DispatchingError(RootException): + """An error dispatching a method call or exposing an endpoint. + """ + + +class Endpoint(object): + """An endpoint for an internal method. + """ + _name: str + _registered: bool + _signature: Type[Callable] + _method: Optional[Callable] + _streaming: Stream + _update: Optional[Callable[['Endpoint'], None]] + + def __init__(self, signature: _GenericAlias, streaming: Stream = Stream.off): # todo: type Callable[] correctly + try: + assert signature.__origin__ == collections.abc.Callable + assert hasattr(signature, '__args__') + except Exception: + raise TypeError('Invalid Endpoint signature') + + self._method = None + self._update = None + self._registered = False + self._signature = signature + self._streaming = streaming + + def compatible(self, method: Callable) -> bool: + """Checks whether a method is compatible with the endpoint's signature + + Parameters + ---------- + method : Callable + Any method or function + + Returns + ------- + bool + ``True`` if the method is compatible, ``False`` if it isn't. + """ + if hasattr(method, '__annotations__'): + args: List = [] + for arg in self.signature: + if arg == type(None): + arg = None + args.append(arg) + # Don't be too pedantic unannotated None-type return + return tuple(method.__annotations__.values()) == tuple(args) + else: + return False + + def expose(self): + """ Expose a method at this endpoint. + Used as a decorator:: + @endpoint.expose() + def some_method(): + pass + """ + def wrapper(method): + if self._method is not None: + log.debug( # todo: add traceback + f"Exposing '{method.__qualname__}' at endpoint '{self.name}' will override " + f"previously exposed method '{self._method.__qualname__}'." + ) # todo: keep in mind we're also marking the methods themselves + + if not self.compatible(method): + raise DispatchingError( + f"Cannot expose '{method.__qualname__}' at endpoint '{self.name}'. " + f"Incompatible signature: {method.__annotations__} vs. {self.signature}" + ) + + method._endpoint = self + self._method = method + if self._update is not None: + self._update(self) + + return method + return wrapper + + @property + def method(self) -> Optional[Callable]: + """The method exposed at this endpoint. Can be ``None`` + """ + return self._method + + @property + def signature(self) -> tuple: + """The signature of this endpoint. + """ + return self._signature.__args__ # type: ignore + + @property + def streaming(self) -> Stream: + """What or whether this endpoint streams. + """ + return self._streaming + + @property + def registered(self) -> bool: + """Whether this endpoint is registered. + """ + return self._registered + + @property + def name(self) -> str: + """The name of this endpoint. + Taken from its attribute name in the object where it is registered. + """ + try: + return self._name + except AttributeError: + return '' + + def register(self, name: str, callback: Callable[['Endpoint'], None]): + """Register the endpoint in some other object. + """ + self._registered = True + self._name = name + self._update = callback + + +class Dispatcher(object): # todo: these should also register specific instances & handle dispatching? + """Dispatches requests to :class:`shapeflow.core.Endpoint` objects. + """ + _endpoints: Tuple[Endpoint, ...] #type: ignore + _dispatchers: Tuple['Dispatcher', ...] + + _name: str + _parent: Optional['Dispatcher'] + _address_space: Dict[str, Optional[Callable]] + + _update: Optional[Callable[['Dispatcher'], None]] + + _instance: Optional[object] + + def __init__(self, instance: object = None): + self._update = None + if instance is not None: + self._set_instance(instance) + else: + self._address_space = {} + self._endpoints = tuple() + self._dispatchers = tuple() + + @property + def name(self) -> str: + """The name of this dispatcher. + """ + try: + return self._name + except AttributeError: + return self.__class__.__name__ + + @property + def dispatchers(self) -> Tuple['Dispatcher', ...]: + """The dispatchers nested in this dispatcher. + """ + return self._dispatchers + + @property + def endpoints(self) -> Tuple[Endpoint, ...]: + """The endpoints contained in this dispatcher. + """ + return self._endpoints + + @property + def address_space(self) -> Dict[str, Optional[Callable]]: + """The address-method mapping of this dispatcher. + """ + return self._address_space + + def _set_instance(self, instance: object): + self._instance = instance + self._address_space = {} + self._endpoints = tuple() + self._dispatchers = tuple() + + for attr, val in self.__class__.__dict__.items(): + if isinstance(val, Endpoint): # todo: also register dispatchers + self._add_endpoint(attr, val) + elif isinstance(val, Dispatcher): + self._add_dispatcher(attr, val) + + def _register(self, name: str, callback: Callable[['Dispatcher'], None]): + """Register this dispatcher within another dispatcher. + """ + self._update = callback + self._name = name + + def _add_endpoint(self, name: str, endpoint: Endpoint): + endpoint.register(name=name, callback=self._update_endpoint) + + if endpoint.method is not None and self._instance is not None: + method = bind(self._instance, endpoint.method) + else: + method = endpoint.method + + self._address_space[name] = method + self._endpoints = tuple(list(self._endpoints) + [endpoint]) + setattr(self, name, endpoint) + + if self._update is not None: + self._update(self) + + def _add_dispatcher(self, name: str, dispatcher: 'Dispatcher'): + dispatcher._register(name=name, callback=self._update_dispatcher) + + self._address_space.update({ + "/".join([name, address]): method + for address, method in dispatcher.address_space.items() + if method is not None and "__" not in address + }) + self._dispatchers = tuple(list(self._dispatchers) + [dispatcher]) + setattr(self, name, dispatcher) + + if self._update is not None: + self._update(self) + + def _update_endpoint(self, endpoint: Endpoint) -> None: + self._address_space.update({ + endpoint.name: endpoint.method + }) + if self._update is not None: + self._update(self) + + def _update_dispatcher(self, dispatcher: 'Dispatcher') -> None: + self._address_space.update({ # todo: this doesn't take into account deleted keys! + "/".join([dispatcher.name, address]): method + for address, method in dispatcher.address_space.items() + if method is not None and "__" not in address + }) + + if self._update is not None: + self._update(self) + + def dispatch(self, address: str, *args, **kwargs) -> Any: + """Dispatch a request to a method. + + Parameters + ---------- + address : str + The address to dispatch to + args + Any positional arguments to pass on to the method + kwargs + Any keyword arguments to pass on to the method + + Returns + ------- + Any + Whatever the method returns. + """ + try: + method = self.address_space[address] + + if method is not None: + # todo: consider doing some type checking here, args/kwargs vs. method._endpoint.signature + return method(*args, **kwargs) + except KeyError: + raise DispatchingError( + f"'{self.name}' can't dispatch address '{address}'." + ) + + def __getitem__(self, item): + return getattr(self, item) \ No newline at end of file diff --git a/shapeflow/core/features.py b/shapeflow/core/features.py new file mode 100644 index 00000000..b4dba538 --- /dev/null +++ b/shapeflow/core/features.py @@ -0,0 +1,270 @@ +import abc +from typing import Optional, Tuple, Type, Any, List, Mapping + +import numpy as np + +from shapeflow.core.config import BaseConfig, Configurable, Instance +from shapeflow.core.interface import InterfaceType +from shapeflow.maths.colors import Color, HsvColor, as_hsv + + +class FeatureConfig(BaseConfig, abc.ABC): + """Abstract :class:`~shapeflow.core.backend.Feature` parameters""" + pass + + +class Feature(abc.ABC, Configurable): # todo: should probably use Config for parameters after all :) + """A feature implements interactions between BackendElements to + produce a certain value + """ + _color: Optional[Color] + _state: Optional[np.ndarray] + + _label: str = '' # todo: keep these in the config instead? + """Label string, to be used in exported data and the user interface + """ + _unit: str = '' + """Unit string, to be used in exported data and the user interface + """ + _elements: Tuple[Instance, ...] = () + + _config: Optional[FeatureConfig] + _global_config: FeatureConfig + _config_class: Type[FeatureConfig] = FeatureConfig + + def __init__(self, elements: Tuple[Instance, ...], global_config: FeatureConfig, config: Optional[dict] = None): + self._skip = False + self._ready = False + + self._elements = elements + self._global_config = global_config + + if config is not None: + self._config = global_config.__class__(**config) + else: + self._config = None + + self._color = HsvColor(h=0,s=200,v=255) # start out as red + + def calculate(self, frame: np.ndarray, state: np.ndarray = None) \ + -> Tuple[Any, Optional[np.ndarray]]: + """Calculate the feature for the given frame + + Parameters + ---------- + frame : np.ndarray + A video frame + state : Optional[np.ndarray] + An ``np.ndarray`` for the state image. Should have the same + dimensions as the ``frame`` + + Returns + ------- + Any + The calculated feature value + Optional[np.ndarray] + If a state image was provided, return this state frame with the + feature's state frame added onto it. If not, return ``None``. + """ + """Calculate the feature for given frame + """ + if state is not None: + state = self.state(frame, state) + return self.value(frame), state + + @classmethod + def label(cls) -> str: + """The label of this feature. Used in the user interface and results. + """ + return cls._label + + @classmethod + def unit(cls) -> str: + """The unit of this feature. Used in the user interface and results. + """ + return cls._unit + + @property + def skip(self) -> bool: + """Whether this feature should be skipped + """ + raise NotImplementedError + + @property + def ready(self) -> bool: + """Whether this feature is ready to be calculated + """ + raise NotImplementedError + + def set_color(self, color: Color): + self._color = color + + @property + def color(self) -> Color: + """Color of the feature in figures. + + A feature's color is handled by its :class:`~shapeflow.core.backend.FeatureSet` + as not to overlap with any other features in that set. + """ + if self._color is not None: + return self._color + else: + raise ValueError + + @abc.abstractmethod + def _guideline_color(self) -> Color: + """Returns the 'guideline color' of a feature, which is used to resolve + to the final color within a feature set. + """ + raise NotImplementedError + + @abc.abstractmethod # todo: we're dealing with frames explicitly, so maybe this should be an shapeflow.video thing... + def state(self, frame: np.ndarray, state: np.ndarray) -> np.ndarray: + """Return the feature's state image for a given frame + """ + raise NotImplementedError + + @abc.abstractmethod + def value(self, frame: np.ndarray) -> Any: + """Compute the value of the feature for a given frame + """ + raise NotImplementedError + + @property + def config(self): + """The configuration of the feature. + Default to the global configuration if no specific one is provided. + """ + if self._config is not None: + return self._config + else: + return self._global_config + + +class FeatureSet(Configurable): + """A set of :class:`~shapeflow.core.backend.Feature` instances + """ + _feature: Tuple[Feature, ...] + _colors: Tuple[Color, ...] + _config_class = FeatureConfig + + def __init__(self, features: Tuple[Feature, ...]): + self._features = features + + def resolve_colors(self) -> Tuple[Color, ...]: + """Resolve the colors of all features in this set so that none of them + overlap. + """ + guideline_colors = [ + as_hsv(f._guideline_color()) for f in self._features + ] + + min_v = 20.0 + max_v = 255.0 + tolerance = 15 + + bins: list = [] + # todo: clean up binning + for index, color in enumerate(guideline_colors): + if not bins: + bins.append([index]) + else: + in_bin = False + for bin in bins: + if abs( + float(color.h) - + np.mean([guideline_colors[i].h for i in bin]) + ) < tolerance: + bin.append(index) + in_bin = True + break + if not in_bin: + bins.append([index]) + + for bin in bins: + if len(bin) < 4: + increment = 60.0 + else: + increment = (max_v - min_v) / len(bin) + + for repetition, index in enumerate(bin): + self._features[index].set_color( + HsvColor( + h=guideline_colors[index].h, + s=220, + v=int(max_v - repetition * increment) + ) + ) + + self._colors = tuple([feature.color for feature in self._features]) + return self.colors + + @property + def colors(self) -> Tuple[Color, ...]: + """The resolved colors in this feature set + """ + return self._colors + + @property + def features(self) -> Tuple[Feature, ...]: + """The features in this feature set + """ + return self._features + + def calculate(self, frame: np.ndarray, state: Optional[np.ndarray]) -> Tuple[List[Any], Optional[np.ndarray]]: + """Calculate all features in this set for a given frame + + Parameters + ---------- + frame : np.ndarray + An image + state : Optional[np.ndarray] + An empty ``np.ndarray`` for the state image. Should have the same + dimensions as the ``frame`` + + Returns + ------- + List[Any] + The calculated feature values + Optional[np.ndarray] + If a state image was provided, return the composite state image of + this feature set. If not, return ``None``. + """ + values = [] + + for feature in self.features: + value, state = feature.calculate(frame=frame, state=state) + values.append(value) + + return values, state + + +class FeatureType(InterfaceType): + """:class:`~shapeflow.core.backend.Feature` factory + """ + _type = Feature + _mapping: Mapping[str, Type[Feature]] = {} + _config_type = FeatureConfig + + def get(self) -> Type[Feature]: + """Get the :class:`~shapeflow.core.backend.Feature` for this feature type + """ + feature = super().get() + assert issubclass(feature, Feature) + return feature + + def config_schema(self) -> dict: + """The ``pydantic`` configuration schema for + this type of :class:`~shapeflow.core.backend.Feature` + """ + return self.get().config_schema() + + @classmethod + def __modify_schema__(cls, field_schema): + """Modify ``pydantic`` schema to include units and labels. + """ + super().__modify_schema__(field_schema) + field_schema.update( + units={ k:v._unit for k,v in cls._mapping.items() }, + labels={ k:v._label for k,v in cls._mapping.items() } + ) \ No newline at end of file diff --git a/shapeflow/core/interface.py b/shapeflow/core/interface.py index 07800d6e..7a4a7939 100644 --- a/shapeflow/core/interface.py +++ b/shapeflow/core/interface.py @@ -1,10 +1,9 @@ -import abc -from typing import Type, Tuple, Optional, Dict, Mapping +import abcC +from typing import Type, Tuple, Optional, Mapping, Callable import numpy as np -from shapeflow import get_logger -from shapeflow.core import Described +from shapeflow.core.logging import get_logger from shapeflow.core.config import BaseConfig, Configurable, Factory from shapeflow.maths.colors import Color from shapeflow.maths.coordinates import ShapeCoo, Roi @@ -28,6 +27,9 @@ def get(self) -> Type[Configurable]: assert issubclass(interface, Configurable) return interface + def config_class(self): + return self.get().config_class() + def config_schema(self) -> dict: """Get the config schema for this interface type. """ diff --git a/shapeflow/core/logging.py b/shapeflow/core/logging.py new file mode 100644 index 00000000..6833cba4 --- /dev/null +++ b/shapeflow/core/logging.py @@ -0,0 +1,120 @@ +import re +import logging + +from shapeflow import VDEBUG, __version__ +from shapeflow.settings import settings + + +class Logger(logging.Logger): + """``shapeflow`` logger. + + * Adds a verbose debug logging level :func:`~shapeflow.Logger.vdebug` + + * Strips newlines from log output to keep each log event on its own line + """ + _pattern = re.compile(r'(\n|\r|\t| [ ]+)') + + def debug(self, msg, *args, **kwargs): + """:meta private:""" + super().debug(self._remove_newlines(msg)) + + def info(self, msg, *args, **kwargs): + """:meta private:""" + super().info(self._remove_newlines(msg)) + + def warning(self, msg, *args, **kwargs): + """:meta private:""" + super().warning(self._remove_newlines(msg)) + + def error(self, msg, *args, **kwargs): + """:meta private:""" + super().error(self._remove_newlines(msg)) + + def critical(self, msg, *args, **kwargs): + """:meta private:""" + super().critical(self._remove_newlines(msg)) + + def vdebug(self, message, *args, **kwargs): + """Log message with severity 'VDEBUG'. + A slightly more verbose debug level for really dense logs. + """ + if self.isEnabledFor(VDEBUG): + self.log( + VDEBUG, self._remove_newlines(message), *args, **kwargs + ) + + def _remove_newlines(self, msg: str) -> str: + return self._pattern.sub(' ', msg) + + +_console_handler = logging.StreamHandler() +_file_handler = logging.FileHandler(str(settings.log.path)) +_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - %(name)s - %(message)s' +) +waitress = logging.getLogger("waitress") + + +def get_logger(name: str) -> Logger: + """Get a new :class:`~shapeflow.Logger` object + + Parameters + ---------- + name : str + name of the logger + + Returns + ------- + Logger + a fresh logging handle + """ + logger = Logger(name) + # log at the _least_ restrictive level + logger.setLevel( + min([settings.log.lvl_console.level, settings.log.lvl_file.level]) + ) + + logger.addHandler(_console_handler) + logger.addHandler(_file_handler) + + logger.vdebug(f'new logger') + return logger + + +# Define log handlers +_console_handler.setLevel(settings.log.lvl_console.level) +_file_handler.setLevel(settings.log.lvl_file.level) + +_console_handler.setFormatter(_formatter) +_file_handler.setFormatter(_formatter) + +# Handle logs from other packages +waitress.addHandler(_console_handler) +waitress.addHandler(_file_handler) +waitress.propagate = False + +log = get_logger('shapeflow') + +log.info(f"v{__version__}") +log.debug(f"settings: {settings.dict()}") + + +class RootException(Exception): + """All ``shapeflow`` exceptions should be subclasses of this one. + Automatically logs the exception class and message at the ``ERROR`` level. + """ + msg = '' + """The message to log + """ + + def __init__(self, *args): + # https://stackoverflow.com/questions/49224770/ + # if no arguments are passed set the first positional argument + # to be the default message. To do that, we have to replace the + # 'args' tuple with another one, that will only contain the message. + # (we cannot do an assignment since tuples are immutable) + if not (args): + args = (self.msg,) + + log.error(self.__class__.__name__ + ': ' + ' '.join(args)) + super(Exception, self).__init__(*args) \ No newline at end of file diff --git a/shapeflow/core/streaming.py b/shapeflow/core/streaming.py index 85b317a6..ace247ca 100644 --- a/shapeflow/core/streaming.py +++ b/shapeflow/core/streaming.py @@ -4,13 +4,13 @@ import abc import json from threading import Thread -from typing import Optional, Tuple, Generator, Callable, Dict, Type, Any, Union, List +from typing import Optional, Generator, Callable, Dict, Type, Any from functools import wraps +from enum import IntEnum -from shapeflow import get_logger -from shapeflow.core import Lockable, _Streaming +from shapeflow.core.logging import get_logger -from shapeflow.util import Singleton +from shapeflow.util import Singleton, Lockable from shapeflow.util.meta import unbind import queue @@ -26,6 +26,13 @@ log = get_logger(__name__) +class Stream(IntEnum): + off = 0 + plain = 1 + json = 2 + image = 3 + + class BaseStreamer(abc.ABC): """Abstract streamer. """ @@ -261,10 +268,10 @@ def _encode(self, frame: np.ndarray) -> Optional[bytes]: return None -_stream_mapping: Dict[_Streaming, Type[BaseStreamer]] = { - _Streaming('plain'): PlainFileStreamer, - _Streaming('json'): JsonStreamer, - _Streaming('image'): JpegStreamer, +_stream_mapping: Dict[Stream, Type[BaseStreamer]] = { + Stream.plain: PlainFileStreamer, + Stream.json: JsonStreamer, + Stream.image: JpegStreamer, } """Maps :data:`shapeflow.core._Streaming` to :class:`~shapeflow.core.streaming.BaseStreamer` implementations. diff --git a/shapeflow/db.py b/shapeflow/db.py index a16b1e05..998d374c 100644 --- a/shapeflow/db.py +++ b/shapeflow/db.py @@ -1,6 +1,6 @@ import os import json -from typing import Optional, Tuple, List, Dict, Type +from typing import Optional, Tuple, List, Dict from pathlib import Path import datetime import sqlite3 @@ -11,15 +11,16 @@ import pandas as pd from shapeflow.api import api -from shapeflow.core import RootInstance +from shapeflow.core import get_logger from shapeflow.core.db import Base, DbModel, SessionWrapper, FileModel, BaseAnalysisModel -from shapeflow import settings, get_logger, ResultSaveMode +from shapeflow.settings import ResultSaveMode +from shapeflow.settings import settings from shapeflow.core.config import __meta_sheet__ from shapeflow.config import normalize_config, VideoAnalyzerConfig from shapeflow.core.streaming import EventStreamer -from shapeflow.core.backend import BaseAnalyzer, BaseAnalyzerConfig - +from shapeflow.core.backend import BaseAnalyzer +from shapeflow.util import Lockable log = get_logger(__name__) @@ -533,7 +534,7 @@ def get_redo_config(self, context: str = None) -> Tuple[Optional[dict], Optional else: raise ValueError(f"Invalid redo context '{context}'") -class History(SessionWrapper, RootInstance): +class History(SessionWrapper, Lockable): """Interface to the history database """ _eventstreamer: EventStreamer diff --git a/shapeflow/main.py b/shapeflow/main.py index cc6bf28a..702307ef 100644 --- a/shapeflow/main.py +++ b/shapeflow/main.py @@ -22,12 +22,17 @@ from shapeflow.util import open_path, sizeof_fmt from shapeflow.util.filedialog import filedialog -from shapeflow import get_logger, get_cache, settings, update_settings, ROOTDIR -from shapeflow.core import stream_off, Endpoint, RootException +from shapeflow.settings import update_settings, ROOTDIR +from shapeflow.settings import settings +from shapeflow.core import get_cache, get_logger +from shapeflow.core.logging import RootException +from shapeflow.core.dispatching import Endpoint from shapeflow.api import api, _FilesystemDispatcher, _DatabaseDispatcher, _VideoAnalyzerManagerDispatcher, _VideoAnalyzerDispatcher, _CacheDispatcher, ApiDispatcher -from shapeflow.core.streaming import streams, EventStreamer, PlainFileStreamer, BaseStreamer +from shapeflow.core.streaming import streams, EventStreamer, PlainFileStreamer, \ + BaseStreamer, Stream from shapeflow.core.backend import QueueState, AnalyzerState, BaseAnalyzer -from shapeflow.config import schemas, normalize_config, loads, BaseAnalyzerConfig +from shapeflow.config import normalize_config, loads, BaseAnalyzerConfig, \ + VideoAnalyzerConfig from shapeflow.video import init, VideoAnalyzer import shapeflow.plugins from shapeflow.server import ShapeflowServer @@ -38,6 +43,25 @@ log = get_logger(__name__) +def schemas() -> Dict[str, dict]: + """Get the JSON schemas of + + * :class:`shapeflow.video.VideoAnalyzerConfig` + + * :class:`shapeflow.Settings` + + * :class:`shapeflow.core.backend.AnalyzerState` + + * :class:`shapeflow.core.backend.QueueState` + """ + return { + 'config': VideoAnalyzerConfig.schema(), + 'settings': settings.schema(), + 'analyzer_state': dict(AnalyzerState.__members__), + 'queue_state': dict(QueueState.__members__), + } + + class _Main(object): """Implements root-level :data:`~shapeflow.api.api` endpoints. """ @@ -815,7 +839,7 @@ def _check_streaming(self, id, endpoint): self._valid(id) if not endpoint in map(lambda e: e.name, api.va[id].endpoints): raise AttributeError(f"no such endpoint: '{endpoint}") - if self._dispatcher[id][endpoint].streaming == stream_off: + if self._dispatcher[id][endpoint].streaming == Stream.off: raise ValueError(f"endpoint '{endpoint}' doesn't stream") @@ -850,4 +874,4 @@ def load(server: ShapeflowServer) -> ApiDispatcher: if settings.app.load_state: api.dispatch('va/load_state') - return api + return api \ No newline at end of file diff --git a/shapeflow/maths/colors.py b/shapeflow/maths/colors.py index 75bd6175..e76ae573 100644 --- a/shapeflow/maths/colors.py +++ b/shapeflow/maths/colors.py @@ -6,7 +6,9 @@ from collections import namedtuple from typing import Dict, Type, List -from shapeflow.core.config import BaseConfig, Field, validator +from pydantic import Field, validator + +from shapeflow.core.config import BaseConfig import cv2 import numpy as np diff --git a/shapeflow/plugins/Area_mm2.py b/shapeflow/plugins/Area_mm2.py index 953340cb..89699423 100644 --- a/shapeflow/plugins/Area_mm2.py +++ b/shapeflow/plugins/Area_mm2.py @@ -2,7 +2,8 @@ from shapeflow.config import extend from shapeflow.maths.images import area_pixelsum -from shapeflow.video import MaskFunction, FeatureType +from shapeflow.video import MaskFunction +from shapeflow.core.features import FeatureType @extend(FeatureType, __name__.split('.')[-1]) diff --git a/shapeflow/plugins/BackgroundFilter.py b/shapeflow/plugins/BackgroundFilter.py index ca8650be..a6c69965 100644 --- a/shapeflow/plugins/BackgroundFilter.py +++ b/shapeflow/plugins/BackgroundFilter.py @@ -1,7 +1,7 @@ import numpy as np import cv2 -from shapeflow import get_logger +from shapeflow.core import get_logger from shapeflow.config import extend, ConfigType, Field, validator, BaseConfig from shapeflow.core.interface import FilterConfig, FilterInterface, FilterType diff --git a/shapeflow/plugins/HsvRangeFilter.py b/shapeflow/plugins/HsvRangeFilter.py index 7a5717f4..fa7a7971 100644 --- a/shapeflow/plugins/HsvRangeFilter.py +++ b/shapeflow/plugins/HsvRangeFilter.py @@ -1,7 +1,7 @@ import numpy as np import cv2 -from shapeflow import get_logger +from shapeflow.core import get_logger from shapeflow.config import extend, ConfigType, Field, validator, BaseConfig from shapeflow.core.interface import FilterConfig, FilterInterface, FilterType diff --git a/shapeflow/plugins/PerspectiveTransform.py b/shapeflow/plugins/PerspectiveTransform.py index cfd85aeb..106169ed 100644 --- a/shapeflow/plugins/PerspectiveTransform.py +++ b/shapeflow/plugins/PerspectiveTransform.py @@ -3,7 +3,7 @@ import cv2 import numpy as np -from shapeflow import get_logger +from shapeflow.core import get_logger from shapeflow.config import extend, ConfigType from shapeflow.core.interface import TransformConfig, TransformInterface, TransformType from shapeflow.maths.coordinates import ShapeCoo, Roi diff --git a/shapeflow/plugins/PixelSum.py b/shapeflow/plugins/PixelSum.py index 1b0278a7..d38af51b 100644 --- a/shapeflow/plugins/PixelSum.py +++ b/shapeflow/plugins/PixelSum.py @@ -3,7 +3,8 @@ from shapeflow.config import extend from shapeflow.maths.images import area_pixelsum -from shapeflow.video import MaskFunction, FeatureType +from shapeflow.video import MaskFunction +from shapeflow.core.features import FeatureType @extend(FeatureType, __name__.split('.')[-1]) diff --git a/shapeflow/plugins/Volume_uL.py b/shapeflow/plugins/Volume_uL.py index 8cf0778a..34eb3e03 100644 --- a/shapeflow/plugins/Volume_uL.py +++ b/shapeflow/plugins/Volume_uL.py @@ -2,7 +2,8 @@ from shapeflow.config import extend, ConfigType, Field from shapeflow.maths.images import area_pixelsum -from shapeflow.video import MaskFunction, FeatureType, FeatureConfig +from shapeflow.video import MaskFunction +from shapeflow.core.features import FeatureConfig, FeatureType @extend(ConfigType, __name__.split('.')[-1]) diff --git a/shapeflow/plugins/__init__.py b/shapeflow/plugins/__init__.py index 521ceb9e..03fde680 100644 --- a/shapeflow/plugins/__init__.py +++ b/shapeflow/plugins/__init__.py @@ -1,8 +1,7 @@ -from os.path import dirname, basename, isfile, join +from os.path import basename import os -import glob -from shapeflow import get_logger +from shapeflow.core import get_logger log = get_logger(__name__) @@ -16,8 +15,7 @@ # import plugins from . import * from shapeflow.core.interface import TransformType, FilterType -from shapeflow.core.backend import FeatureType - +from ..core.features import FeatureType TransformType.set_default(TransformType('PerspectiveTransform')) FilterType.set_default(FilterType('HsvRangeFilter')) diff --git a/shapeflow/server.py b/shapeflow/server.py index 4a7c41da..e509dcb4 100644 --- a/shapeflow/server.py +++ b/shapeflow/server.py @@ -12,11 +12,14 @@ import shapeflow import shapeflow.config import shapeflow.util as util -from shapeflow.core import DispatchingError +from shapeflow.settings import settings +from shapeflow.core import get_logger import shapeflow.core.streaming as streaming +from shapeflow.core.dispatching import DispatchingError from shapeflow.api import ApiDispatcher -log = shapeflow.get_logger(__name__) + +log = get_logger(__name__) UI = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))) , 'ui', 'dist' @@ -47,7 +50,7 @@ def run(self): self._app, host=self._host, port=self._port, - threads=shapeflow.settings.app.threads, + threads=settings.app.threads, ) except OSError: log.warning("address already in use") diff --git a/shapeflow/settings.py b/shapeflow/settings.py new file mode 100644 index 00000000..fe75ef6a --- /dev/null +++ b/shapeflow/settings.py @@ -0,0 +1,518 @@ +import copy +import datetime +import glob +import logging +import os +import shutil +from contextlib import contextmanager +from enum import Enum +from multiprocessing import cpu_count +from pathlib import Path +from typing import Type, Any, Dict + +import yaml +from pydantic import BaseModel, FilePath, DirectoryPath, Field, validator + +from shapeflow import ROOTDIR, VDEBUG +from shapeflow.config import FrameIntervalSetting, FeatureType, FlipConfig, TransformType, FilterType, FeatureConfig, TransformConfig, FilterConfig + + +_SETTINGS_FILE = ROOTDIR / 'settings.yaml' + + +class _Settings(BaseModel): + """Abstract application settings + """ + + class Config: + validate_assignment = True + + def to_dict(self) -> dict: + """ + Returns + ------- + dict + Application settings as a dict + + """ + d: dict = {} + for k,v in self.__dict__.items(): + if isinstance(v, _Settings): + d.update({ + k:v.to_dict() + }) + elif isinstance(v, Enum): + d.update({ + k:v.value + }) + elif isinstance(v, Path): + d.update({ + k:str(v) # type: ignore + }) # todo: fix ` Dict entry 0 has incompatible type "str": "str"; expected "str": "Dict[str, Any]" ` + else: + d.update({ + k:v + }) + return d + + @contextmanager + def override(self, overrides: dict): # todo: consider deprecating in favor of mocks + """Override some parameters of the settings in a context. + Settings will only be modified within this context and restored to + their previous values afterwards. + Usage:: + with settings.override({"parameter": "override value"}): + + + Parameters + ---------- + overrides: dict + A ``dict`` mapping field names to values with which to + override those fields + """ + originals: dict = {} + try: + for attribute, value in overrides.items(): + originals[attribute] = copy.deepcopy( + getattr(self, attribute) + ) + setattr(self, attribute, value) + yield + finally: + for attribute, original in originals.items(): + setattr(self, attribute, original) + + @classmethod + def _validate_filepath(cls, value): + if not isinstance(value, Path): + value = Path(value) + + if not value.exists() and not value.is_file(): + value.touch() + + return value + + @classmethod + def _validate_directorypath(cls, value): + if not isinstance(value, Path): + value = Path(value) + + if not value.exists() and not value.is_dir(): + value.mkdir() + + return value + + @classmethod + def schema(cls, by_alias: bool = True, ref_template: str = '') -> Dict[str, Any]: + """Inject title & description into ``pydantic`` schema. + + These get lost due to some `bug`_ with ``Enum``. + + .. _bug: https://github.com/samuelcolvin/pydantic/pull/1749 + """ + + schema = super().schema(by_alias) + + def _inject(class_: Type[_Settings], schema, definitions): + for field in class_.__fields__.values(): + if 'properties' in schema and field.alias in schema['properties']: + if 'title' not in schema['properties'][field.alias]: + schema['properties'][field.alias][ + 'title'] = field.field_info.title + if field.field_info.description is not None and 'description' not in schema['properties'][field.alias]: + schema['properties'][field.alias]['description'] = field.field_info.description + if issubclass(field.type_, _Settings): + # recurse into nested _Settings classes + _inject(field.type_, definitions[field.type_.__name__], definitions) + return schema + + return _inject(cls, schema, schema['definitions']) + + +class FormatSettings(_Settings): + """Formatting settings + """ + datetime_format: str = Field(default='%Y/%m/%d %H:%M:%S.%f', title="date/time format") + """Base ``datetime`` `format string `_. + Defaults to ``'%Y/%m/%d %H:%M:%S.%f'``. + + .. _dtfs: https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior + """ + datetime_format_fs: str = Field(default='%Y-%m-%d_%H-%M-%S', title="file system date/time format") + """Filesystem-safe ``datetime`` `format string `_. + Used to append date & time to file names. + Defaults to ``'%Y-%m-%d_%H-%M-%S'``. + + .. _dtfs: https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior + """ + + +class LoggingLevel(str, Enum): + """Logging level. + """ + critical = "critical" + """Only log critical (unrecoverable) errors + """ + error = "error" + """Only log errors + """ + warning = "warning" + """Only log warnings (or errors) + """ + info = "info" + """Log general information + """ + debug = "debug" + """Log debugging information + """ + vdebug = "vdebug" + """Log verbose debugging information + """ + + @property + def level(self) -> int: + """Return the ``int`` logging level for compatibility with built-in + ``logging`` library + """ + _levels: dict = { + LoggingLevel.critical: logging.CRITICAL, + LoggingLevel.error: logging.ERROR, + LoggingLevel.warning: logging.WARNING, + LoggingLevel.info: logging.INFO, + LoggingLevel.debug: logging.DEBUG, + LoggingLevel.vdebug: VDEBUG + } + return _levels[self] + + +class LogSettings(_Settings): + """Logging settings + """ + path: FilePath = Field(default=str(ROOTDIR / 'current.log'), title='running log file') + """The application logs to this file + """ + dir: DirectoryPath = Field(default=str(ROOTDIR / 'log'), title='log file directory') + """This is the log directory. Logs from previous runs are stored here. + """ + keep: int = Field(default=16, title="# of log files to keep") + """The applications stores a number of old logs. + + When the amount of log files in :attr:`shapeflow.LogSettings.dir` exceeds + this number, the oldest files are deleted. + """ + lvl_console: LoggingLevel = Field(default=LoggingLevel.info, title="logging level (Python console)") + """The level at which the application logs to the Python console. + + Defaults to :attr:`~shapeflow.LoggingLevel.info` to keep the console from + getting too spammy. + Set to a lower level such as :attr:`~shapeflow.LoggingLevel.debug` to show + more detailed logs in the console. + """ + lvl_file: LoggingLevel = Field(default=LoggingLevel.debug, title="logging level (file)") + """The level at which the application logs to the log file at + :attr:`~shapeflow.LogSettings.path`. + + Defaults to :attr:`shapeflow.LoggingLevel.debug`. + """ + + _validate_path = validator('path', allow_reuse=True, pre=True)( + _Settings._validate_filepath) + _validate_dir = validator('dir', allow_reuse=True, pre=True)( + _Settings._validate_directorypath) + + +class CacheSettings(_Settings): + """Caching settings + """ + do_cache: bool = Field(default=True, title="use the cache") + """Enables the cache. Set to ``True`` by default. + + Disabling the cache will make the application significantly slower. + """ + dir: DirectoryPath = Field(default=str(ROOTDIR / 'cache'), title="cache directory") + """Where to keep the cache + """ + size_limit_gb: float = Field(default=4.0, title="cache size limit (GB)") + """How big the cache is allowed to get + """ + resolve_frame_number: bool = Field(default=True, title="resolve to (nearest) cached frame numbers") + """Whether to resolve frame numbers to the nearest requested frame numbers. + + Increases seeking performance, but may make it a bit more 'jumpy' if a low + number of frames is requested for an analysis. + """ + block_timeout: float = Field(default=0.1, title="wait for blocked item (s)") + """How long to keep waiting for data that's actively being committed to the + cache before giving up and computing it instead. + + In the rare case that two cachable data requests are being processed at the + same time, the first request will block the cache for those specific data + and cause the second request to wait until it can grab this data from the + cache. + This timeout prevents the second request from waiting forever until the + first request finishes (for example, in case it crashes). + """ + reset_on_error: bool = Field(default=False, title="reset the cache if it can't be opened") + """Clear the cache if it can't be opened. + + In rare cases, ``diskcache`` may cache data in a way it can't read back. + To recover from such an error, the cache will be cleared completely. + The only downside to this is decreased performance for a short while. + """ + + _validate_dir = validator('dir', allow_reuse=True, pre=True)( + _Settings._validate_directorypath) + + +class RenderSettings(_Settings): + """Rendering settings + """ + dir: DirectoryPath = Field(default=str(ROOTDIR / 'render'), title="render directory") + """The directory where SVG files should be rendered to + """ + keep: bool = Field(default=False, title="keep files after rendering") + """Keep rendered images after they've been used. + + Disabled by default, you may want to enable this if you want to inspect the + renders. + """ + + _validate_dir = validator('dir', allow_reuse=True, pre=True)( + _Settings._validate_directorypath) + + +class DatabaseSettings(_Settings): + """Database settings. + """ + path: FilePath = Field(default=str(ROOTDIR / 'history.db'), title="database file") + """The path to the database file + """ + cleanup_interval: int = Field(default=7, title='clean-up interval (days)') + """The database can get cluttered after a while, and will be cleaned at + this interval + """ + + _validate_path = validator('path', allow_reuse=True, pre=True)( + _Settings._validate_filepath) + + +class ResultSaveMode(str, Enum): + """Where (or whether) to save the results of an analysis + """ + skip = "skip" + """Don't save results at all + """ + next_to_video = "next to video file" + """Save results in the same directory as the video file that was analyzed + """ + next_to_design = "next to design file" + """Save results in the same directory as the design file that was analyzed + """ + directory = "in result directory" + """Save results in their own directory at + :attr:`shapeflow.ApplicationSettings.result_dir` + """ + + +class ApplicationSettings(_Settings): + """Application settings. + """ + save_state: bool = Field(default=True, title="save application state on exit") + """Whether to save the application state when exiting the application + """ + load_state: bool = Field(default=True, title="load application state on start") + """Whether to load the application state when starting the application + """ + state_path: FilePath = Field(default=str(ROOTDIR / 'state'), title="application state file") + """Where to save the application state + """ + recent_files: int = Field(default=16, title="# of recent files to fetch") + """The number of recent files to show in the user interface + """ + video_pattern: str = Field(default="*.mp4 *.avi *.mov *.mpv *.mkv", title="video file pattern") + """Recognized video file extensions. + Defaults to ``"*.mp4 *.avi *.mov *.mpv *.mkv"``. + """ + design_pattern: str = Field(default="*.svg", title="design file pattern") + """Recognized design file extensions. + Defaults to ``"*.svg"``. + """ + save_result_auto: ResultSaveMode = Field(default=ResultSaveMode.next_to_video, title="result save mode (auto)") + """Where or whether to save results after each run of an analysis + """ + save_result_manual: ResultSaveMode = Field(default=ResultSaveMode.next_to_video, title="result save mode (manual)") + """Where or whether to save results that are exported manually + via the user interface + """ + result_dir: DirectoryPath = Field(default=str(ROOTDIR / 'results'), title="result directory") + """The path to the result directory + """ + cancel_on_q_stop: bool = Field(default=False, title="cancel running analyzers when stopping queue") + """Whether to cancel the currently running analysis when stopping a queue. + + Defaults to ``False``, i.e. the currently running analysis will be + completed first. + """ + threads: int = Field(default=cpu_count(), title="# of threads") + f"""The number of threads the server uses. Defaults to {cpu_count()}, the + number of logical cores of your machine's CPU. + """ + + _validate_dir = validator('result_dir', allow_reuse=True, pre=True)( + _Settings._validate_directorypath) + _validate_state_path = validator('state_path', allow_reuse=True, pre=True)( + _Settings._validate_filepath) + + @validator('threads', pre=True, allow_reuse=True) + def _validate_threads(cls, value): + if value < 8: + return 8 # At least 8 threads to run decently + else: + return value + + +class DefaultConfigSettings(_Settings): + fis: FrameIntervalSetting = Field(default=FrameIntervalSetting('Nf'), title="Frame interval setting") + + dt: float = Field(default=5.0, title="Frame interval in seconds") + + Nf: int = Field(default=100, title="Total number of frames") + + feature: FeatureType = Field(default_factory=FeatureType, title="Feature") + + feature_parameters: FeatureConfig = Field(default_factory=FeatureType.config_class, title="Feature parameter(s)") + + flip: FlipConfig = Field(default_factory=FlipConfig, title="Flip the ROI...") + + turn: int = Field(default=0, title="Turn the ROI ... times (clockwise, 90°)") + + transform: TransformType = Field(default_factory=TransformType, title="Transform") + + transform_config: TransformConfig = Field(default_factory=TransformType.config_class, title="Transform configuration") + + filter: FilterType = Field(default_factory=FilterType, title="Filter") + + filter_config: FilterConfig = Field(default_factory=FilterType.config_class, title="Filter configuration") + + mask_skip: bool = Field(default=False, title="Skip masks") + + +class Settings(_Settings): + """``shapeflow`` settings. + + * app: :class:`~shapeflow.ApplicationSettings` + + * log: :class:`~shapeflow.LogSettings` + + * cache: :class:`~shapeflow.CacheSettings` + + * render: :class:`~shapeflow.RenderSettings` + + * format: :class:`~shapeflow.FormatSettings` + + * db: :class:`~shapeflow.DatabaseSettings` + + * default_config: :class:`~shapeflow.DefaultConfigSettings` + """ + app: ApplicationSettings = Field(default=ApplicationSettings(), title="Application") + log: LogSettings = Field(default=LogSettings(), title="Logging") + cache: CacheSettings = Field(default=CacheSettings(), title="Caching") + render: RenderSettings = Field(default=RenderSettings(), title="SVG Rendering") + format: FormatSettings = Field(default=FormatSettings(), title="Formatting") + db: DatabaseSettings = Field(default=DatabaseSettings(), title="Database") + default_config: DefaultConfigSettings = Field(default=DefaultConfigSettings(), title="Default configuration") + + @classmethod + def from_dict(cls, settings: dict): # todo: deprecate; DefaultConfigSettings introduces deeper nesting & other pydantic subclasses + for k in cls.__fields__.keys(): + if k not in settings: + settings.update({k:{}}) + + return cls( + **{field.name:field.type_(**settings[field.name]) + for field in cls.__fields__.values()} + ) + + +settings: Settings +"""This global :class:`~shapeflow.Settings` object is used throughout the + library +""" + + +def _load_settings() -> Settings: # todo: if there are unexpected fields: warn, don't crash + """Load :class:`~shapeflow.Settings` from .yaml + """ + global settings + + if _SETTINGS_FILE.is_file(): + with open(_SETTINGS_FILE, 'r') as f: + settings_yaml = yaml.safe_load(f) # todo: replace with json + + # Get settings + if settings_yaml is not None: + settings = Settings.from_dict(settings_yaml) + else: + settings = Settings() + + # Move the previous log file to ROOTDIR/log + if Path(settings.log.path).is_file(): + shutil.move( + str(settings.log.path), # todo: convert to pathlib + os.path.join( + settings.log.dir, + datetime.datetime.fromtimestamp( + os.path.getmtime(settings.log.path) + ).strftime(settings.format.datetime_format_fs) + '.log' + ) + ) + + # If more files than specified in ini.log.keep, remove the oldest + files = glob.glob(os.path.join(settings.log.dir, '*.log')) # todo: convert to pathlib + files.sort(key=lambda f: os.path.getmtime(f), reverse=True) + while len(files) > settings.log.keep: + os.remove(files.pop()) + else: + settings = Settings() + + return settings + + +def save_settings(path: str = str(_SETTINGS_FILE)): + """Save :data:`~shapeflow.settings` to .yaml + """ + with open(path, 'w+') as f: + yaml.safe_dump(settings.to_dict(), f) + + +def update_settings(s: dict) -> dict: + """Update the global settings object. + + .. note:: + Just doing ``settings = Settings(**new_settings_dict)`` + would prevent other modules from accessing the updated settings! + + Parameters + ---------- + s : dict + new settings to integrate into the global settings + + Returns + ------- + dict + the current global settings as a ``dict`` + """ + global settings + + for cat, cat_new in s.items(): + sub = getattr(settings, cat) + for kw, val in cat_new.items(): + setattr(sub, kw, val) + + save_settings() + return settings.to_dict() + + +# Instantiate global settings object +_load_settings() +save_settings() \ No newline at end of file diff --git a/shapeflow/util/__init__.py b/shapeflow/util/__init__.py index 77f3f854..6bbeeb73 100644 --- a/shapeflow/util/__init__.py +++ b/shapeflow/util/__init__.py @@ -19,6 +19,11 @@ import numpy as np +# from shapeflow.core.logging import get_logger +# from shapeflow.settings import settings +# +# log = get_logger(__name__, settings) + def ndarray2str(array: np.ndarray) -> str: return str(json.dumps(array.tolist())) @@ -304,3 +309,114 @@ def ensure_path(path: Union[str, Path]): raise finally: sys.path.remove(path_str) + + +class Described(object): + """A class with a description. + + This description is taken from the first line of the docstring if there is + one or set to the name of the class if there isn't. + """ + @classmethod + def _description(cls): + if cls.__doc__ is not None: + return cls.__doc__.split('\n')[0] + else: + return cls.__name__ + + +class Lockable(object): + """Wrapper around :class:`threading.Lock` & :class:`threading.Event` + + Defines a :class:`~shapeflow.core.Lockable.lock` context to handle locking + and unlocking along with a ``_cancel`` and ``_error`` events to communicate + with :class:`~shapeflow.core.Lockable` objects from other threads. + + Doesn't need to initialize; lock & events are created when they're needed. + """ + _lock: threading.Lock + _cancel: threading.Event + _error: threading.Event + + @property + def _ensure_lock(self) -> threading.Lock: + try: + return self._lock + except AttributeError: + self._lock = threading.Lock() + return self._lock + + @property + def _ensure_cancel(self) -> threading.Event: + try: + return self._cancel + except AttributeError: + self._cancel = threading.Event() + return self._cancel + + @property + def _ensure_error(self) -> threading.Event: + try: + return self._error + except AttributeError: + self._error = threading.Event() + return self._error + + @contextmanager + def lock(self): + """Locking context. + + If ``_lock`` event doesn't exist yet it is instantiated first. + Upon exiting the context, the :class:`threading.Lock` object + is compared to the original to ensure that no shenanigans took place. + """ + # log.vdebug(f"Acquiring lock {self}...") + locked = self._ensure_lock.acquire() + original_lock = self._lock + # log.vdebug(f"Acquired lock {self}") + try: + # log.vdebug(f"Locking {self}") + yield locked + finally: + # log.vdebug(f"Unlocking {self}") + # Make 'sure' nothing weird happened to self._lock + assert self._lock == original_lock + self._lock.release() + + def cancel(self): + """Sets the ``_cancel`` event. + If ``_cancel`` event doesn't exist yet it is instantiated first. + """ + self._ensure_cancel.set() + + def error(self): + """Sets the ``_error`` event. + If ``_error`` event doesn't exist yet it is instantiated first. + """ + self._ensure_error.set() + + @property + def canceled(self) -> bool: + """Returns ``True`` if the ``_cancel`` event is set. + If ``_cancel`` event doesn't exist yet it is instantiated first. + """ + return self._ensure_cancel.is_set() + + @property + def errored(self) -> bool: + """Returns ``True`` if the ``_error`` event is set. + If ``_error`` event doesn't exist yet it is instantiated first. + """ + return self._ensure_error.is_set() + + def clear_cancel(self): + """Clears the ``_cancel`` event. + If ``_cancel`` event doesn't exist yet it is instantiated first. + """ + return self._ensure_cancel.clear() + + def clear_error(self): + """Clears the ``_error`` event. + If ``_error`` event doesn't exist yet it is instantiated first. + """ + return self._ensure_error.clear() \ No newline at end of file diff --git a/shapeflow/video.py b/shapeflow/video.py index 38bf0cf5..3a1fbb17 100644 --- a/shapeflow/video.py +++ b/shapeflow/video.py @@ -9,17 +9,17 @@ import pandas as pd from OnionSVG import OnionSVG, check_svg -from shapeflow import get_logger, settings, ResultSaveMode +from shapeflow.core.logging import get_logger +from shapeflow.settings import settings from shapeflow.api import api from shapeflow.config import VideoFileHandlerConfig, TransformHandlerConfig, \ FilterHandlerConfig, MaskConfig, \ DesignFileHandlerConfig, VideoAnalyzerConfig, \ - FrameIntervalSetting, BaseAnalyzerConfig, FlipConfig -from shapeflow.core import Lockable + FrameIntervalSetting, BaseAnalyzerConfig from shapeflow.core.backend import Instance, CachingInstance, \ - BaseAnalyzer, BackendSetupError, AnalyzerType, Feature, \ - FeatureSet, \ - FeatureType, AnalyzerState, PushEvent, FeatureConfig, CacheAccessError + BaseAnalyzer, BackendSetupError, AnalyzerType, AnalyzerState, PushEvent, CacheAccessError +from shapeflow.core.features import FeatureConfig, Feature, FeatureSet, \ + FeatureType from shapeflow.core.config import extend from shapeflow.core.interface import TransformInterface, FilterConfig, \ FilterInterface, FilterType, TransformType, Handler @@ -28,7 +28,7 @@ from shapeflow.maths.images import to_mask, crop_mask, ckernel, \ overlay, rect_contains from shapeflow.maths.coordinates import ShapeCoo, Roi -from shapeflow.util import frame_number_iterator +from shapeflow.util import frame_number_iterator, Lockable log = get_logger(__name__) diff --git a/test/test_config.py b/test/test_config.py index 450c5dd9..9b7422fb 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -2,10 +2,12 @@ import yaml +from pydantic import Field, validator + from shapeflow.plugins.HsvRangeFilter import _Filter from shapeflow.video import * -from shapeflow.core.config import Factory, Field, BaseConfig, validator, VERSION, CLASS -from shapeflow.core import EnforcedStr +from shapeflow.core.config import Factory, BaseConfig, VERSION, CLASS, \ + EnforcedStr from shapeflow.core.interface import FilterType from shapeflow.config import normalize_config, ColorSpace diff --git a/test/test_endpoints.py b/test/test_endpoints.py index 6f6b976d..9fe23358 100644 --- a/test/test_endpoints.py +++ b/test/test_endpoints.py @@ -2,7 +2,7 @@ from typing import Callable -from shapeflow.core import Endpoint, Dispatcher +from shapeflow.core.dispatching import Endpoint, Dispatcher from shapeflow.util.meta import bind, unbind from shapeflow.api import _CacheDispatcher, _DatabaseDispatcher, \ _VideoAnalyzerManagerDispatcher, _FilesystemDispatcher, \ diff --git a/test/test_main.py b/test/test_main.py index cf5b0f99..0a389485 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -21,8 +21,7 @@ __VIDEO__ = 'test/' + __VIDEO__ __DESIGN__ = 'test/' + __DESIGN__ -from shapeflow import ROOTDIR - +from shapeflow.settings import ROOTDIR, save_settings CACHE = os.path.join(ROOTDIR, 'test_server-cache') DB = os.path.join(ROOTDIR, 'test_server-history.db') @@ -43,7 +42,7 @@ def clear_files(): @contextmanager def application(keep: bool = False): - from shapeflow import settings, save_settings + from shapeflow.settings import settings if not keep: clear_files() @@ -711,7 +710,8 @@ def test_json_streaming(self): class DbCheckTest(unittest.TestCase): def test_db_check(self): - from shapeflow import settings, save_settings + from shapeflow.settings import save_settings + from shapeflow.settings import settings with settings.db.override({'path': DB}): clear_files() diff --git a/test/test_plugins.py b/test/test_plugins.py index 5169067b..606adc42 100644 --- a/test/test_plugins.py +++ b/test/test_plugins.py @@ -1,16 +1,9 @@ import unittest import abc -from typing import Type, Dict, List -import cv2 -import numpy as np - -from shapeflow.maths.coordinates import Roi, Coo -from shapeflow.maths.images import ckernel -from shapeflow.core.config import Factory -from shapeflow.config import TransformType, ConfigType, TransformConfig -from shapeflow.plugins import * +from shapeflow.maths.coordinates import Coo +from shapeflow.config import TransformConfig from shapeflow.video import * diff --git a/test/test_server.py b/test/test_server.py index d67ad191..e37b1cc2 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -7,7 +7,8 @@ import json import subprocess -from shapeflow import settings, ROOTDIR, save_settings +from shapeflow import ROOTDIR +from shapeflow.settings import settings, save_settings raise unittest.SkipTest('takes too long') @@ -58,9 +59,9 @@ def override_settings(): os.remove(STATE) try: - with settings.cache.override({"dir": CACHE}), \ - settings.db.override({"path": DB}), \ - settings.app.override({"state_path": STATE}): + with gs.cache.override({"dir": CACHE}), \ + gs.db.override({"path": DB}), \ + gs.app.override({"state_path": STATE}): save_settings() yield finally: diff --git a/test/test_streaming.py b/test/test_streaming.py index c7e23c22..25272631 100644 --- a/test/test_streaming.py +++ b/test/test_streaming.py @@ -4,12 +4,11 @@ from threading import Thread import time -import cv2 import numpy as np -from shapeflow.core import Endpoint, Dispatcher, stream_image -from shapeflow.core.config import Instance, BaseConfig -from shapeflow.core.streaming import BaseStreamer, JpegStreamer, JsonStreamer, streams, stream +from shapeflow.core.dispatching import Endpoint, Dispatcher +from shapeflow.core.streaming import BaseStreamer, JpegStreamer, JsonStreamer, \ + streams, stream, Stream class StreamerThread(Thread): @@ -112,8 +111,8 @@ class JsonStreamerTest(BaseStreamerTest): class StreamHandlerTest(unittest.TestCase): def test_normal_operation(self): class TestDispatcher(Dispatcher): - method1 = Endpoint(Callable[[], np.ndarray], stream_image) - method2 = Endpoint(Callable[[], np.ndarray], stream_image) + method1 = Endpoint(Callable[[], np.ndarray], Stream.image) + method2 = Endpoint(Callable[[], np.ndarray], Stream.image) class StreamableClass(object): @stream diff --git a/test/test_video.py b/test/test_video.py index 0886a03b..e4036138 100644 --- a/test/test_video.py +++ b/test/test_video.py @@ -19,7 +19,7 @@ from shapeflow.video import VideoFileHandler, VideoFileTypeError, \ CachingInstance, VideoAnalyzer -from shapeflow import settings +from shapeflow.settings import settings from shapeflow.core.config import * From b0ca40c8286e4cec9b6f0713465d366a3e2398b7 Mon Sep 17 00:00:00 2001 From: ybnd Date: Sat, 30 Jan 2021 13:12:07 +0100 Subject: [PATCH 3/4] Implement dynamic category-based lazy settings The current implementation allows more dynamic, complex and circularly dependent definitions for the application settings by decoupling category definitions from the settings class * shapeflow.core.settings defines settings: Settings * other modules create Category subclasses (~pydantic) * settings are loaded from JSON * stored in the instance under settings._loaded * when a new category is added, check whether it has an entry in settings._loaded. If it does, initialize the category with that state * requests for settings ~ settings.get(SomeCategory) * returns the instance of this category stored in settings * if this category doesn't exist yet, an instance is added to settings --- shapeflow/cli.py | 7 +- shapeflow/config.py | 27 ++ shapeflow/core/__init__.py | 34 ++ shapeflow/core/backend.py | 12 +- shapeflow/core/caching.py | 61 ++- shapeflow/core/interface.py | 2 +- shapeflow/core/logging.py | 124 ++++-- shapeflow/core/settings.py | 312 +++++++++++++ shapeflow/db.py | 47 +- shapeflow/main.py | 38 +- shapeflow/plugins/BackgroundFilter.py | 2 +- shapeflow/plugins/HsvRangeFilter.py | 2 +- shapeflow/plugins/PerspectiveTransform.py | 2 +- shapeflow/plugins/__init__.py | 2 +- shapeflow/server.py | 4 +- shapeflow/settings.py | 518 ---------------------- shapeflow/video.py | 46 +- 17 files changed, 620 insertions(+), 620 deletions(-) create mode 100644 shapeflow/core/settings.py delete mode 100644 shapeflow/settings.py diff --git a/shapeflow/cli.py b/shapeflow/cli.py index a2e5c5f4..3c9c882b 100644 --- a/shapeflow/cli.py +++ b/shapeflow/cli.py @@ -16,9 +16,8 @@ from typing import List, Callable, Optional, Tuple from shapeflow import __version__ -from shapeflow.settings import settings -from shapeflow.core import get_logger -from shapeflow.core.logging import RootException +from shapeflow.core.settings import settings +from shapeflow.core.logging import get_logger, RootException log = get_logger(__name__) @@ -286,7 +285,7 @@ def command(self): self.args.dir.mkdir() self._write('schemas', schemas()) - self._write('settings', settings.to_dict()) + self._write('settings', settings.as_dict()) def _write(self, file, d): with open(self.args.dir / (file + '.json'), 'w+') as f: diff --git a/shapeflow/config.py b/shapeflow/config.py index ce7f60ba..1179efe5 100644 --- a/shapeflow/config.py +++ b/shapeflow/config.py @@ -10,6 +10,7 @@ from shapeflow.core.features import FeatureConfig, FeatureType from shapeflow.core.interface import FilterType, TransformType, TransformConfig, \ FilterConfig, HandlerConfig +from shapeflow.core.settings import Category from shapeflow.maths.coordinates import Roi from shapeflow.maths.colors import HsvColor @@ -487,3 +488,29 @@ def normalizing_to(version): untag(d) return d + + +class DefaultConfigSettings(Category): + fis: FrameIntervalSetting = Field(default=FrameIntervalSetting('Nf'), title="Frame interval setting") + + dt: float = Field(default=5.0, title="Frame interval in seconds") + + Nf: int = Field(default=100, title="Total number of frames") + + feature: FeatureType = Field(default_factory=FeatureType, title="Feature") + + feature_parameters: FeatureConfig = Field(default_factory=FeatureType.config_class, title="Feature parameter(s)") + + flip: FlipConfig = Field(default_factory=FlipConfig, title="Flip the ROI...") + + turn: int = Field(default=0, title="Turn the ROI ... times (clockwise, 90°)") + + transform: TransformType = Field(default_factory=TransformType, title="Transform") + + transform_config: TransformConfig = Field(default_factory=TransformType.config_class, title="Transform configuration") + + filter: FilterType = Field(default_factory=FilterType, title="Filter") + + filter_config: FilterConfig = Field(default_factory=FilterType.config_class, title="Filter configuration") + + mask_skip: bool = Field(default=False, title="Skip masks") \ No newline at end of file diff --git a/shapeflow/core/__init__.py b/shapeflow/core/__init__.py index e69de29b..252204b4 100644 --- a/shapeflow/core/__init__.py +++ b/shapeflow/core/__init__.py @@ -0,0 +1,34 @@ +from logging import Logger + + +class RootException(Exception): + """Automatically logs the exception class and message at the ``ERROR`` level. + """ + _logger: Logger = None + + msg = '' + """The default message to log + """ + + @classmethod + def set_logger(cls, logger: Logger) -> None: + """Set the loggerto log exceptions to + + Parameters + ---------- + logger : Logger + """ + cls._log = logger + + def __init__(self, *args): + # https://stackoverflow.com/questions/49224770/ + # if no arguments are passed set the first positional argument + # to be the default message. To do that, we have to replace the + # 'args' tuple with another one, that will only contain the message. + # (we cannot do an assignment since tuples are immutable) + if not (args): + args = (self.msg,) + + if self._logger is not None: + self._logger.error(self.__class__.__name__ + ': ' + ' '.join(args)) + super(RootException, self).__init__(*args) diff --git a/shapeflow/core/backend.py b/shapeflow/core/backend.py index e65a2c06..57a3882c 100644 --- a/shapeflow/core/backend.py +++ b/shapeflow/core/backend.py @@ -10,7 +10,8 @@ import pandas as pd from shapeflow.config import BaseAnalyzerConfig -from shapeflow.core.caching import get_cache +from shapeflow.core.settings import settings +from shapeflow.core.caching import get_cache, CacheSettings from shapeflow.core.logging import get_logger, RootException from shapeflow.api import api @@ -101,6 +102,7 @@ class CachingInstance(Instance): # todo: consider a waterfall cache: e.g. 2 GB def __init__(self, config: BaseConfig = None): super(CachingInstance, self).__init__(config) + self._cache = None self._open_cache() def __del__(self): @@ -173,7 +175,7 @@ def cached_call(self, method, *args, **kwargs): # todo: kwargs necessary? # Check if the file's already cached if key in self._cache: t0 = time.time() - while self._is_blocked(key) and time.time() < t0 + settings.cache.block_timeout: + while self._is_blocked(key) and time.time() < t0 + settings.get(CacheSettings).block_timeout: # Some other thread is currently reading the same frame # Wait a bit and try to get from cache again log.debug(f'{self.__class__.__qualname__}: ' @@ -203,9 +205,9 @@ def cached_call(self, method, *args, **kwargs): # todo: kwargs necessary? return method(*args, **kwargs) def _open_cache(self, override: bool = False): - if settings.cache.do_cache or override: + if settings.get(CacheSettings).do_cache or override: log.debug(f"{self.__class__.__qualname__}: " - f"opening cache @ {settings.cache.dir}") + f"opening cache @ {settings.get(CacheSettings).dir}") self._cache = get_cache() else: self._cache = None @@ -213,7 +215,7 @@ def _open_cache(self, override: bool = False): def _close_cache(self): if self._cache is not None: log.debug(f"{self.__class__.__qualname__}: " - f"closing cache @ {settings.cache.dir}") + f"closing cache @ {settings.get(CacheSettings).dir}") self._cache.close() self._cache = None diff --git a/shapeflow/core/caching.py b/shapeflow/core/caching.py index a6f8ffa9..09115121 100644 --- a/shapeflow/core/caching.py +++ b/shapeflow/core/caching.py @@ -2,13 +2,62 @@ import sqlite3 import diskcache +from pydantic import Field, DirectoryPath, validator +from shapeflow import ROOTDIR from shapeflow.core.logging import get_logger - +from shapeflow.core.settings import Category, settings log = get_logger(__name__) +class CacheSettings(Category): + """Caching settings + """ + do_cache: bool = Field(default=True, title="use the cache") + """Enables the cache. Set to ``True`` by default. + + Disabling the cache will make the application significantly slower. + """ + dir: DirectoryPath = Field(default=str(ROOTDIR / 'cache'), + title="cache directory") + """Where to keep the cache + """ + size_limit_gb: float = Field(default=4.0, title="cache size limit (GB)") + """How big the cache is allowed to get + """ + resolve_frame_number: bool = Field(default=True, + title="resolve to (nearest) cached frame numbers") + """Whether to resolve frame numbers to the nearest requested frame numbers. + + Increases seeking performance, but may make it a bit more 'jumpy' if a low + number of frames is requested for an analysis. + """ + block_timeout: float = Field(default=0.1, + title="wait for blocked item (s)") + """How long to keep waiting for data that's actively being committed to the + cache before giving up and computing it instead. + + In the rare case that two cachable data requests are being processed at the + same time, the first request will block the cache for those specific data + and cause the second request to wait until it can grab this data from the + cache. + This timeout prevents the second request from waiting forever until the + first request finishes (for example, in case it crashes). + """ + reset_on_error: bool = Field(default=False, + title="reset the cache if it can't be opened") + """Clear the cache if it can't be opened. + + In rare cases, ``diskcache`` may cache data in a way it can't read back. + To recover from such an error, the cache will be cleared completely. + The only downside to this is decreased performance for a short while. + """ + + _validate_dir = validator('dir', allow_reuse=True, pre=True)( + Category._validate_directorypath) + + def get_cache(retry: bool = False) -> diskcache.Cache: """Get a new :class:`diskcache.Cache` object In some rare cases this can fail due to a corrupt cache. @@ -30,17 +79,19 @@ def get_cache(retry: bool = False) -> diskcache.Cache: """ try: return diskcache.Cache( - directory=str(settings.cache.dir), - size_limit=settings.cache.size_limit_gb * 1e9 + directory=str(settings.get(CacheSettings).dir), + size_limit=settings.get(CacheSettings).size_limit_gb * 1e9 ) except sqlite3.OperationalError as e: log.error(f"could not open cache - {e.__class__.__name__}: {str(e)}") if not retry: - if settings.cache.reset_on_error: + if settings.get(CacheSettings).reset_on_error: log.error(f"removing cache directory") - shutil.rmtree(str(settings.cache.dir)) + shutil.rmtree(str(settings.get(CacheSettings).dir)) log.error(f"trying to open cache again...") get_cache(retry=True) else: log.error(f"could not open cache on retry") raise e + + diff --git a/shapeflow/core/interface.py b/shapeflow/core/interface.py index 7a4a7939..ae982725 100644 --- a/shapeflow/core/interface.py +++ b/shapeflow/core/interface.py @@ -1,4 +1,4 @@ -import abcC +import abc from typing import Type, Tuple, Optional, Mapping, Callable import numpy as np diff --git a/shapeflow/core/logging.py b/shapeflow/core/logging.py index 6833cba4..69206401 100644 --- a/shapeflow/core/logging.py +++ b/shapeflow/core/logging.py @@ -1,8 +1,86 @@ import re import logging +from enum import Enum + +from pydantic import FilePath, Field, DirectoryPath, validator from shapeflow import VDEBUG, __version__ -from shapeflow.settings import settings +from shapeflow.core import RootException +from shapeflow.core.settings import Category, settings, ROOTDIR + + +class LoggingLevel(str, Enum): + """Logging level. + """ + critical = "critical" + """Only log critical (unrecoverable) errors + """ + error = "error" + """Only log errors + """ + warning = "warning" + """Only log warnings (or errors) + """ + info = "info" + """Log general information + """ + debug = "debug" + """Log debugging information + """ + vdebug = "vdebug" + """Log verbose debugging information + """ + + @property + def level(self) -> int: + """Return the ``int`` logging level for compatibility with built-in + ``logging`` library + """ + _levels: dict = { + LoggingLevel.critical: logging.CRITICAL, + LoggingLevel.error: logging.ERROR, + LoggingLevel.warning: logging.WARNING, + LoggingLevel.info: logging.INFO, + LoggingLevel.debug: logging.DEBUG, + LoggingLevel.vdebug: VDEBUG + } + return _levels[self] + + +class LogSettings(Category): + """Logging settings + """ + path: FilePath = Field(default=str(ROOTDIR / 'current.log'), title='running log file') + """The application logs to this file + """ + dir: DirectoryPath = Field(default=str(ROOTDIR / 'log'), title='log file directory') + """This is the log directory. Logs from previous runs are stored here. + """ + keep: int = Field(default=16, title="# of log files to keep") + """The applications stores a number of old logs. + + When the amount of log files in :attr:`shapeflow.LogSettings.dir` exceeds + this number, the oldest files are deleted. + """ + lvl_console: LoggingLevel = Field(default=LoggingLevel.info, title="logging level (Python console)") + """The level at which the application logs to the Python console. + + Defaults to :attr:`~shapeflow.LoggingLevel.info` to keep the console from + getting too spammy. + Set to a lower level such as :attr:`~shapeflow.LoggingLevel.debug` to show + more detailed logs in the console. + """ + lvl_file: LoggingLevel = Field(default=LoggingLevel.debug, title="logging level (file)") + """The level at which the application logs to the log file at + :attr:`~shapeflow.LogSettings.path`. + + Defaults to :attr:`shapeflow.LoggingLevel.debug`. + """ + + _validate_path = validator('path', allow_reuse=True, pre=True)( + Category._validate_filepath) + _validate_dir = validator('dir', allow_reuse=True, pre=True)( + Category._validate_directorypath) class Logger(logging.Logger): @@ -45,16 +123,16 @@ def vdebug(self, message, *args, **kwargs): def _remove_newlines(self, msg: str) -> str: return self._pattern.sub(' ', msg) +_console_handler = logging.StreamHandler() +_file_handler = logging.FileHandler(str(settings.get(LogSettings).path)) -_console_handler = logging.StreamHandler() -_file_handler = logging.FileHandler(str(settings.log.path)) _formatter = logging.Formatter( '%(asctime)s - %(levelname)s - %(name)s - %(message)s' ) -waitress = logging.getLogger("waitress") +waitress = logging.getLogger("waitress") def get_logger(name: str) -> Logger: """Get a new :class:`~shapeflow.Logger` object @@ -71,7 +149,8 @@ def get_logger(name: str) -> Logger: logger = Logger(name) # log at the _least_ restrictive level logger.setLevel( - min([settings.log.lvl_console.level, settings.log.lvl_file.level]) + min([settings.get(LogSettings).lvl_console.level, + settings.get(LogSettings).lvl_file.level]) ) logger.addHandler(_console_handler) @@ -79,42 +158,21 @@ def get_logger(name: str) -> Logger: logger.vdebug(f'new logger') return logger - - # Define log handlers -_console_handler.setLevel(settings.log.lvl_console.level) -_file_handler.setLevel(settings.log.lvl_file.level) + +_console_handler.setLevel(settings.get(LogSettings).lvl_console.level) +_file_handler.setLevel(settings.get(LogSettings).lvl_file.level) _console_handler.setFormatter(_formatter) _file_handler.setFormatter(_formatter) - # Handle logs from other packages waitress.addHandler(_console_handler) + waitress.addHandler(_file_handler) -waitress.propagate = False +waitress.propagate = False log = get_logger('shapeflow') +RootException.set_logger(log) log.info(f"v{__version__}") -log.debug(f"settings: {settings.dict()}") - - -class RootException(Exception): - """All ``shapeflow`` exceptions should be subclasses of this one. - Automatically logs the exception class and message at the ``ERROR`` level. - """ - msg = '' - """The message to log - """ - - def __init__(self, *args): - # https://stackoverflow.com/questions/49224770/ - # if no arguments are passed set the first positional argument - # to be the default message. To do that, we have to replace the - # 'args' tuple with another one, that will only contain the message. - # (we cannot do an assignment since tuples are immutable) - if not (args): - args = (self.msg,) - - log.error(self.__class__.__name__ + ': ' + ' '.join(args)) - super(Exception, self).__init__(*args) \ No newline at end of file +log.debug(f"settings: {settings.as_dict()}") \ No newline at end of file diff --git a/shapeflow/core/settings.py b/shapeflow/core/settings.py new file mode 100644 index 00000000..fd30efa1 --- /dev/null +++ b/shapeflow/core/settings.py @@ -0,0 +1,312 @@ +import copy +import datetime +import glob +import os +import shutil +from contextlib import contextmanager +from enum import Enum +from multiprocessing import cpu_count +from pathlib import Path +from typing import Dict, Any, Type, TypeVar, Generic, Tuple + +import json +from pydantic import BaseModel, Field, DirectoryPath, validator, FilePath + +from shapeflow import ROOTDIR +from shapeflow.core import RootException +from shapeflow.util import Singleton + + +class SettingsError(RootException): + pass + + +class Category(BaseModel): + """Abstract application settings + """ + + class Config: + validate_assignment = True + + def to_dict(self) -> dict: + """ + Returns + ------- + dict + Application settings as a dict + + """ + d: dict = {} + for k,v in self.__dict__.items(): + if isinstance(v, Category): + d.update({ + k:v.to_dict() + }) + elif isinstance(v, Enum): + d.update({ + k:v.value + }) + elif isinstance(v, Path): + d.update({ + k:str(v) # type: ignore + }) # todo: fix ` Dict entry 0 has incompatible type "str": "str"; expected "str": "Dict[str, Any]" ` + else: + d.update({ + k:v + }) + return d + + @contextmanager + def override(self, overrides: dict): # todo: consider deprecating in favor of mocks + """Override some parameters of the settings in a context. + Settings will only be modified within this context and restored to + their previous values afterwards. + Usage:: + with settings.override({"parameter": "override value"}): + + + Parameters + ---------- + overrides: dict + A ``dict`` mapping field names to values with which to + override those fields + """ + originals: dict = {} + try: + for attribute, value in overrides.items(): + originals[attribute] = copy.deepcopy( + getattr(self, attribute) + ) + setattr(self, attribute, value) + yield + finally: + for attribute, original in originals.items(): + setattr(self, attribute, original) + + @classmethod + def _validate_filepath(cls, value): + if not isinstance(value, Path): + value = Path(value) + + if not value.exists() and not value.is_file(): + value.touch() + + return value + + @classmethod + def _validate_directorypath(cls, value): + if not isinstance(value, Path): + value = Path(value) + + if not value.exists() and not value.is_dir(): + value.mkdir() + + return value + + @classmethod + def schema(cls, by_alias: bool = True, ref_template: str = '') -> Dict[str, Any]: + """Inject title & description into ``pydantic`` schema. + + These get lost due to some `bug`_ with ``Enum``. + + .. _bug: https://github.com/samuelcolvin/pydantic/pull/1749 + """ + + schema = super().schema(by_alias) + + def _inject(class_: Type[Category], schema, definitions): + for field in class_.__fields__.values(): + if 'properties' in schema and field.alias in schema['properties']: + if 'title' not in schema['properties'][field.alias]: + schema['properties'][field.alias][ + 'title'] = field.field_info.title + if field.field_info.description is not None and 'description' not in schema['properties'][field.alias]: + schema['properties'][field.alias]['description'] = field.field_info.description + if issubclass(field.type_, Category): + # recurse into nested _Settings classes + _inject(field.type_, definitions[field.type_.__name__], definitions) + return schema + + return _inject(cls, schema, schema['definitions']) + + +_SETTINGS_FILE = ROOTDIR / 'settings.json' + + +class Settings(metaclass=Singleton): + _categories: Dict[Type[Category], Category] = {} + _loaded: Dict[str, dict] = {} + + def __init__(self): + self._load() + self.save() + + def add(self, category: Type[Category]) -> None: + """Add a category to the Settings + + Parameters + ---------- + category : Type[Category] + The category class to add + + Returns + ------- + + """ + self._categories[category] = self._initialize_from_loaded(category) + + def get(self, type: Type[Category]) -> Category: + if type is None: + raise SettingsError('no category class provided') + if type not in self._categories: + self.add(type) + return self._categories[type] + + def _load(self): + """Load from JSON + """ + + if _SETTINGS_FILE.is_file(): + with open(_SETTINGS_FILE, 'r') as f: + self._loaded = json.load(f) + + for category in self._categories.keys(): + self._initialize_from_loaded(category) + + # # Move the previous log file to ROOTDIR/log # todo: move to shapeflow.core.logging + # if Path(settings.log.path).is_file(): + # shutil.move( + # str(settings.log.path), # todo: convert to pathlib + # os.path.join( + # settings.log.dir, + # datetime.datetime.fromtimestamp( + # os.path.getmtime(settings.log.path) + # ).strftime( + # settings.format.datetime_format_fs) + '.log' + # ) + # ) + # + # # If more files than specified in ini.log.keep, remove the oldest + # files = glob.glob(os.path.join(settings.log.dir, + # '*.log')) # todo: convert to pathlib + # files.sort(key=lambda f: os.path.getmtime(f), reverse=True) + # while len(files) > settings.log.keep: + # os.remove(files.pop()) + + def _initialize_from_loaded(self, category: Type[Category]) -> Category: + if category.__name__ in self._loaded: + return category(**self._loaded[category.__name__]) + else: + return category() + + def save(self): + with open(_SETTINGS_FILE, 'w+') as f: + json.dump(self.as_dict(), f, indent=2) + + def as_dict(self) -> dict: + return {type.__name__: category.dict() + for (type, category) in self._categories.items()} + + def update(self, d: dict) -> dict: + types = self._categories.keys() + for type in types: + if type.__name__ in d: + self._loaded[type.__name__] = d[type.__name__] + self._categories[type] = self._initialize_from_loaded(type) + return self.as_dict() + + +settings = Settings() + + +class FormatSettings(Category): + """Formatting settings + """ + datetime_format: str = Field(default='%Y/%m/%d %H:%M:%S.%f', title="date/time format") + """Base ``datetime`` `format string `_. + Defaults to ``'%Y/%m/%d %H:%M:%S.%f'``. + + .. _dtfs: https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior + """ + datetime_format_fs: str = Field(default='%Y-%m-%d_%H-%M-%S', title="file system date/time format") + """Filesystem-safe ``datetime`` `format string `_. + Used to append date & time to file names. + Defaults to ``'%Y-%m-%d_%H-%M-%S'``. + + .. _dtfs: https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior + """ + + +class ResultSaveMode(str, Enum): + """Where (or whether) to save the results of an analysis + """ + skip = "skip" + """Don't save results at all + """ + next_to_video = "next to video file" + """Save results in the same directory as the video file that was analyzed + """ + next_to_design = "next to design file" + """Save results in the same directory as the design file that was analyzed + """ + directory = "in result directory" + """Save results in their own directory at + :attr:`shapeflow.ApplicationSettings.result_dir` + """ + + +class ApplicationSettings(Category): + """Application settings. + """ + save_state: bool = Field(default=True, title="save application state on exit") + """Whether to save the application state when exiting the application + """ + load_state: bool = Field(default=True, title="load application state on start") + """Whether to load the application state when starting the application + """ + state_path: FilePath = Field(default=str(ROOTDIR / 'state'), title="application state file") + """Where to save the application state + """ + recent_files: int = Field(default=16, title="# of recent files to fetch") + """The number of recent files to show in the user interface + """ + video_pattern: str = Field(default="*.mp4 *.avi *.mov *.mpv *.mkv", title="video file pattern") + """Recognized video file extensions. + Defaults to ``"*.mp4 *.avi *.mov *.mpv *.mkv"``. + """ + design_pattern: str = Field(default="*.svg", title="design file pattern") + """Recognized design file extensions. + Defaults to ``"*.svg"``. + """ + save_result_auto: ResultSaveMode = Field(default=ResultSaveMode.next_to_video, title="result save mode (auto)") + """Where or whether to save results after each run of an analysis + """ + save_result_manual: ResultSaveMode = Field(default=ResultSaveMode.next_to_video, title="result save mode (manual)") + """Where or whether to save results that are exported manually + via the user interface + """ + result_dir: DirectoryPath = Field(default=str(ROOTDIR / 'results'), title="result directory") + """The path to the result directory + """ + cancel_on_q_stop: bool = Field(default=False, title="cancel running analyzers when stopping queue") + """Whether to cancel the currently running analysis when stopping a queue. + + Defaults to ``False``, i.e. the currently running analysis will be + completed first. + """ + threads: int = Field(default=cpu_count(), title="# of threads") + f"""The number of threads the server uses. Defaults to {cpu_count()}, the + number of logical cores of your machine's CPU. + """ + + _validate_dir = validator('result_dir', allow_reuse=True, pre=True)( + Category._validate_directorypath) + _validate_state_path = validator('state_path', allow_reuse=True, pre=True)( + Category._validate_filepath) + + @validator('threads', pre=True, allow_reuse=True) + def _validate_threads(cls, value): + if value < 8: + return 8 # At least 8 threads to run decently + else: + return value \ No newline at end of file diff --git a/shapeflow/db.py b/shapeflow/db.py index 998d374c..40cff74f 100644 --- a/shapeflow/db.py +++ b/shapeflow/db.py @@ -5,16 +5,19 @@ import datetime import sqlite3 +from pydantic import FilePath, Field, validator from sqlalchemy.orm import sessionmaker, scoped_session from sqlalchemy import create_engine, Column, Integer, Float, String, DateTime, ForeignKey import pandas as pd +from shapeflow import ROOTDIR from shapeflow.api import api -from shapeflow.core import get_logger -from shapeflow.core.db import Base, DbModel, SessionWrapper, FileModel, BaseAnalysisModel -from shapeflow.settings import ResultSaveMode -from shapeflow.settings import settings +from shapeflow.core.logging import get_logger +from shapeflow.core.db import Base, DbModel, SessionWrapper, FileModel, \ + BaseAnalysisModel +from shapeflow.core.settings import settings, FormatSettings, Category, \ + ResultSaveMode, ApplicationSettings from shapeflow.core.config import __meta_sheet__ from shapeflow.config import normalize_config, VideoAnalyzerConfig from shapeflow.core.streaming import EventStreamer @@ -306,29 +309,29 @@ def export_result(self, run: int = None, manual: bool = False): base_f = None if manual: - if settings.app.save_result_manual == ResultSaveMode.next_to_video: + if settings.get(ApplicationSettings).save_result_manual == ResultSaveMode.next_to_video: base_f = str(os.path.splitext(config['video_path'])[0]) - elif settings.app.save_result_manual == ResultSaveMode.next_to_design: + elif settings.get(ApplicationSettings).save_result_manual == ResultSaveMode.next_to_design: base_f = str(os.path.splitext(config['design_path'])[0]) - elif settings.app.save_result_manual == ResultSaveMode.directory: + elif settings.get(ApplicationSettings).save_result_manual == ResultSaveMode.directory: base_f = os.path.join( str(settings.app.result_dir), f"{self.name} run {run}" ) else: - if settings.app.save_result_auto == ResultSaveMode.next_to_video: + if settings.get(ApplicationSettings).save_result_auto == ResultSaveMode.next_to_video: base_f = str(os.path.splitext(config['video_path'])[0]) - elif settings.app.save_result_auto == ResultSaveMode.next_to_design: + elif settings.get(ApplicationSettings).save_result_auto == ResultSaveMode.next_to_design: base_f = str(os.path.splitext(config['design_path'])[0]) - elif settings.app.save_result_auto == ResultSaveMode.directory: + elif settings.get(ApplicationSettings).save_result_auto == ResultSaveMode.directory: base_f = os.path.join( - str(settings.app.result_dir), + str(settings.get(ApplicationSettings).result_dir), f"{self.name} run {run}" ) if base_f is not None: f = base_f + ' ' + datetime.datetime.now().strftime( - settings.format.datetime_format_fs + settings.get(FormatSettings).datetime_format_fs ) + '.xlsx' w = pd.ExcelWriter(f) @@ -543,7 +546,7 @@ def __init__(self, path: Path = None): super().__init__() if path is None: - path = settings.db.path + path = settings.get(DatabaseSettings).path self._engine = create_engine(f'sqlite:///{str(path)}') try: @@ -657,10 +660,10 @@ def get_paths(self) -> Dict[str, List[str]]: return { 'video_path': [r[0] for r in s.query(VideoFileModel.path).\ order_by(VideoFileModel.used.desc()).\ - limit(settings.app.recent_files).all()], + limit(settings.get(ApplicationSettings).recent_files).all()], 'design_path': [r[0] for r in s.query(DesignFileModel.path). \ order_by(DesignFileModel.used.desc()). \ - limit(settings.app.recent_files).all()] + limit(settings.get(ApplicationSettings).recent_files).all()] } # @history.expose(history.get_result_list) @@ -834,7 +837,7 @@ def clean(self) -> None: """ log.debug(f"cleaning history") threshold = datetime.datetime.now() - datetime.timedelta( - days=settings.db.cleanup_interval + days=settings.get(DatabaseSettings).cleanup_interval ) with self.session() as s: @@ -886,4 +889,16 @@ def forget(self) -> None: s.query(model).delete() +class DatabaseSettings(Category): + """Database settings. + """ + path: FilePath = Field(default=str(ROOTDIR / 'history.db'), title="database file") + """The path to the database file + """ + cleanup_interval: int = Field(default=7, title='clean-up interval (days)') + """The database can get cluttered after a while, and will be cleaned at + this interval + """ + _validate_path = validator('path', allow_reuse=True, pre=True)( + Category._validate_filepath) \ No newline at end of file diff --git a/shapeflow/main.py b/shapeflow/main.py index 702307ef..0644524b 100644 --- a/shapeflow/main.py +++ b/shapeflow/main.py @@ -22,10 +22,10 @@ from shapeflow.util import open_path, sizeof_fmt from shapeflow.util.filedialog import filedialog -from shapeflow.settings import update_settings, ROOTDIR -from shapeflow.settings import settings -from shapeflow.core import get_cache, get_logger -from shapeflow.core.logging import RootException +from shapeflow.core.settings import settings, ApplicationSettings, ROOTDIR +from shapeflow.db import DatabaseSettings +from shapeflow.core.caching import get_cache +from shapeflow.core.logging import RootException, get_logger, LogSettings from shapeflow.core.dispatching import Endpoint from shapeflow.api import api, _FilesystemDispatcher, _DatabaseDispatcher, _VideoAnalyzerManagerDispatcher, _VideoAnalyzerDispatcher, _CacheDispatcher, ApiDispatcher from shapeflow.core.streaming import streams, EventStreamer, PlainFileStreamer, \ @@ -159,10 +159,10 @@ def get_settings(self) -> dict: dict The application settings as a ``dict`` """ - return settings.to_dict() + return settings.as_dict() @api.set_settings.expose() - def set_settings(self, settings: dict) -> dict: + def set_settings(self, d: dict) -> dict: """Set the application settings :attr:`shapeflow.api.ApiDispatcher.set_settings` @@ -177,7 +177,7 @@ def set_settings(self, settings: dict) -> dict: dict The new application settings ``dict``, which may have been modified. """ - new_settings = update_settings(settings) + new_settings = settings.update(d) self.restart() return new_settings @@ -219,7 +219,7 @@ def log(self) -> PlainFileStreamer: self.stop_log() log.debug("streaming log file") - self._log = PlainFileStreamer(path=str(settings.log.path)) + self._log = PlainFileStreamer(path=str(settings.get(LogSettings).path)) return self._log @@ -331,7 +331,7 @@ def select_video(self) -> Optional[str]: """ return filedialog.load( title='Select a video file...', - pattern=settings.app.video_pattern, + pattern=settings.get(ApplicationSettings).video_pattern, pattern_description='Video files', ) @@ -348,7 +348,7 @@ def select_design(self) -> Optional[str]: """ return filedialog.load( title='Select a design file...', - pattern=settings.app.design_pattern, + pattern=settings.get(ApplicationSettings).design_pattern, pattern_description='Design files', ) @@ -445,10 +445,10 @@ def check_history(self): # todo: move to shapeflow.db else: timestamp = datetime.datetime.fromtimestamp( time.time() - ).strftime(settings.format.datetime_format_fs) - backup_path = f"{settings.db.path}_broken_{timestamp}" + ).strftime(settings.get(FormatSettings).datetime_format_fs) + backup_path = f"{settings.get(DatabaseSettings).path}_broken_{timestamp}" log.warning(f"backing up old history database @ {backup_path}") - os.rename(settings.db.path, backup_path) + os.rename(settings.get(DatabaseSettings).path, backup_path) @@ -662,7 +662,7 @@ def q_stop(self) -> None: if self._pause_q.is_set(): self._pause_q.clear() self._stop_q.set() - if settings.app.cancel_on_q_stop: + if settings.get(ApplicationSettings).cancel_on_q_stop: self.q_cancel() else: for analyzer in self.__analyzers__.values(): @@ -732,12 +732,12 @@ def save_state(self) -> None: :attr:`shapeflow.api._VideoAnalyzerManagerDispatcher.save_state` """ - if settings.app.save_state: + if settings.get(ApplicationSettings).save_state: log.debug(f"saving application state") self._commit() - with open(settings.app.state_path, 'wb') as f: + with open(settings.get(ApplicationSettings).state_path, 'wb') as f: pickle.dump({ id: analyzer.model.get('id') for id, analyzer in self.__analyzers__.items() @@ -750,11 +750,11 @@ def load_state(self) -> None: :attr:`shapeflow.api._VideoAnalyzerManagerDispatcher.load_state` """ - if settings.app.load_state: + if settings.get(ApplicationSettings).load_state: log.info(f"loading application state") try: - with open(settings.app.state_path, 'rb') as f: + with open(settings.get(ApplicationSettings).state_path, 'rb') as f: S = pickle.load(f) for id, model_id in S.items(): @@ -871,7 +871,7 @@ def load(server: ShapeflowServer) -> ApiDispatcher: api._add_dispatcher('va', _va) - if settings.app.load_state: + if settings.get(ApplicationSettings).load_state: api.dispatch('va/load_state') return api \ No newline at end of file diff --git a/shapeflow/plugins/BackgroundFilter.py b/shapeflow/plugins/BackgroundFilter.py index a6c69965..0dcf5b51 100644 --- a/shapeflow/plugins/BackgroundFilter.py +++ b/shapeflow/plugins/BackgroundFilter.py @@ -1,7 +1,7 @@ import numpy as np import cv2 -from shapeflow.core import get_logger +from shapeflow.core.logging import get_logger from shapeflow.config import extend, ConfigType, Field, validator, BaseConfig from shapeflow.core.interface import FilterConfig, FilterInterface, FilterType diff --git a/shapeflow/plugins/HsvRangeFilter.py b/shapeflow/plugins/HsvRangeFilter.py index fa7a7971..b3d41199 100644 --- a/shapeflow/plugins/HsvRangeFilter.py +++ b/shapeflow/plugins/HsvRangeFilter.py @@ -1,7 +1,7 @@ import numpy as np import cv2 -from shapeflow.core import get_logger +from shapeflow.core.logging import get_logger from shapeflow.config import extend, ConfigType, Field, validator, BaseConfig from shapeflow.core.interface import FilterConfig, FilterInterface, FilterType diff --git a/shapeflow/plugins/PerspectiveTransform.py b/shapeflow/plugins/PerspectiveTransform.py index 106169ed..f8cb0b05 100644 --- a/shapeflow/plugins/PerspectiveTransform.py +++ b/shapeflow/plugins/PerspectiveTransform.py @@ -3,7 +3,7 @@ import cv2 import numpy as np -from shapeflow.core import get_logger +from shapeflow.core.logging import get_logger from shapeflow.config import extend, ConfigType from shapeflow.core.interface import TransformConfig, TransformInterface, TransformType from shapeflow.maths.coordinates import ShapeCoo, Roi diff --git a/shapeflow/plugins/__init__.py b/shapeflow/plugins/__init__.py index 03fde680..5a7b8a4a 100644 --- a/shapeflow/plugins/__init__.py +++ b/shapeflow/plugins/__init__.py @@ -1,7 +1,7 @@ from os.path import basename import os -from shapeflow.core import get_logger +from shapeflow.core.logging import get_logger log = get_logger(__name__) diff --git a/shapeflow/server.py b/shapeflow/server.py index e509dcb4..ba31dcaf 100644 --- a/shapeflow/server.py +++ b/shapeflow/server.py @@ -12,8 +12,8 @@ import shapeflow import shapeflow.config import shapeflow.util as util -from shapeflow.settings import settings -from shapeflow.core import get_logger +from shapeflow.core.settings import settings +from shapeflow.core.logging import get_logger import shapeflow.core.streaming as streaming from shapeflow.core.dispatching import DispatchingError from shapeflow.api import ApiDispatcher diff --git a/shapeflow/settings.py b/shapeflow/settings.py deleted file mode 100644 index fe75ef6a..00000000 --- a/shapeflow/settings.py +++ /dev/null @@ -1,518 +0,0 @@ -import copy -import datetime -import glob -import logging -import os -import shutil -from contextlib import contextmanager -from enum import Enum -from multiprocessing import cpu_count -from pathlib import Path -from typing import Type, Any, Dict - -import yaml -from pydantic import BaseModel, FilePath, DirectoryPath, Field, validator - -from shapeflow import ROOTDIR, VDEBUG -from shapeflow.config import FrameIntervalSetting, FeatureType, FlipConfig, TransformType, FilterType, FeatureConfig, TransformConfig, FilterConfig - - -_SETTINGS_FILE = ROOTDIR / 'settings.yaml' - - -class _Settings(BaseModel): - """Abstract application settings - """ - - class Config: - validate_assignment = True - - def to_dict(self) -> dict: - """ - Returns - ------- - dict - Application settings as a dict - - """ - d: dict = {} - for k,v in self.__dict__.items(): - if isinstance(v, _Settings): - d.update({ - k:v.to_dict() - }) - elif isinstance(v, Enum): - d.update({ - k:v.value - }) - elif isinstance(v, Path): - d.update({ - k:str(v) # type: ignore - }) # todo: fix ` Dict entry 0 has incompatible type "str": "str"; expected "str": "Dict[str, Any]" ` - else: - d.update({ - k:v - }) - return d - - @contextmanager - def override(self, overrides: dict): # todo: consider deprecating in favor of mocks - """Override some parameters of the settings in a context. - Settings will only be modified within this context and restored to - their previous values afterwards. - Usage:: - with settings.override({"parameter": "override value"}): - - - Parameters - ---------- - overrides: dict - A ``dict`` mapping field names to values with which to - override those fields - """ - originals: dict = {} - try: - for attribute, value in overrides.items(): - originals[attribute] = copy.deepcopy( - getattr(self, attribute) - ) - setattr(self, attribute, value) - yield - finally: - for attribute, original in originals.items(): - setattr(self, attribute, original) - - @classmethod - def _validate_filepath(cls, value): - if not isinstance(value, Path): - value = Path(value) - - if not value.exists() and not value.is_file(): - value.touch() - - return value - - @classmethod - def _validate_directorypath(cls, value): - if not isinstance(value, Path): - value = Path(value) - - if not value.exists() and not value.is_dir(): - value.mkdir() - - return value - - @classmethod - def schema(cls, by_alias: bool = True, ref_template: str = '') -> Dict[str, Any]: - """Inject title & description into ``pydantic`` schema. - - These get lost due to some `bug`_ with ``Enum``. - - .. _bug: https://github.com/samuelcolvin/pydantic/pull/1749 - """ - - schema = super().schema(by_alias) - - def _inject(class_: Type[_Settings], schema, definitions): - for field in class_.__fields__.values(): - if 'properties' in schema and field.alias in schema['properties']: - if 'title' not in schema['properties'][field.alias]: - schema['properties'][field.alias][ - 'title'] = field.field_info.title - if field.field_info.description is not None and 'description' not in schema['properties'][field.alias]: - schema['properties'][field.alias]['description'] = field.field_info.description - if issubclass(field.type_, _Settings): - # recurse into nested _Settings classes - _inject(field.type_, definitions[field.type_.__name__], definitions) - return schema - - return _inject(cls, schema, schema['definitions']) - - -class FormatSettings(_Settings): - """Formatting settings - """ - datetime_format: str = Field(default='%Y/%m/%d %H:%M:%S.%f', title="date/time format") - """Base ``datetime`` `format string `_. - Defaults to ``'%Y/%m/%d %H:%M:%S.%f'``. - - .. _dtfs: https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior - """ - datetime_format_fs: str = Field(default='%Y-%m-%d_%H-%M-%S', title="file system date/time format") - """Filesystem-safe ``datetime`` `format string `_. - Used to append date & time to file names. - Defaults to ``'%Y-%m-%d_%H-%M-%S'``. - - .. _dtfs: https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior - """ - - -class LoggingLevel(str, Enum): - """Logging level. - """ - critical = "critical" - """Only log critical (unrecoverable) errors - """ - error = "error" - """Only log errors - """ - warning = "warning" - """Only log warnings (or errors) - """ - info = "info" - """Log general information - """ - debug = "debug" - """Log debugging information - """ - vdebug = "vdebug" - """Log verbose debugging information - """ - - @property - def level(self) -> int: - """Return the ``int`` logging level for compatibility with built-in - ``logging`` library - """ - _levels: dict = { - LoggingLevel.critical: logging.CRITICAL, - LoggingLevel.error: logging.ERROR, - LoggingLevel.warning: logging.WARNING, - LoggingLevel.info: logging.INFO, - LoggingLevel.debug: logging.DEBUG, - LoggingLevel.vdebug: VDEBUG - } - return _levels[self] - - -class LogSettings(_Settings): - """Logging settings - """ - path: FilePath = Field(default=str(ROOTDIR / 'current.log'), title='running log file') - """The application logs to this file - """ - dir: DirectoryPath = Field(default=str(ROOTDIR / 'log'), title='log file directory') - """This is the log directory. Logs from previous runs are stored here. - """ - keep: int = Field(default=16, title="# of log files to keep") - """The applications stores a number of old logs. - - When the amount of log files in :attr:`shapeflow.LogSettings.dir` exceeds - this number, the oldest files are deleted. - """ - lvl_console: LoggingLevel = Field(default=LoggingLevel.info, title="logging level (Python console)") - """The level at which the application logs to the Python console. - - Defaults to :attr:`~shapeflow.LoggingLevel.info` to keep the console from - getting too spammy. - Set to a lower level such as :attr:`~shapeflow.LoggingLevel.debug` to show - more detailed logs in the console. - """ - lvl_file: LoggingLevel = Field(default=LoggingLevel.debug, title="logging level (file)") - """The level at which the application logs to the log file at - :attr:`~shapeflow.LogSettings.path`. - - Defaults to :attr:`shapeflow.LoggingLevel.debug`. - """ - - _validate_path = validator('path', allow_reuse=True, pre=True)( - _Settings._validate_filepath) - _validate_dir = validator('dir', allow_reuse=True, pre=True)( - _Settings._validate_directorypath) - - -class CacheSettings(_Settings): - """Caching settings - """ - do_cache: bool = Field(default=True, title="use the cache") - """Enables the cache. Set to ``True`` by default. - - Disabling the cache will make the application significantly slower. - """ - dir: DirectoryPath = Field(default=str(ROOTDIR / 'cache'), title="cache directory") - """Where to keep the cache - """ - size_limit_gb: float = Field(default=4.0, title="cache size limit (GB)") - """How big the cache is allowed to get - """ - resolve_frame_number: bool = Field(default=True, title="resolve to (nearest) cached frame numbers") - """Whether to resolve frame numbers to the nearest requested frame numbers. - - Increases seeking performance, but may make it a bit more 'jumpy' if a low - number of frames is requested for an analysis. - """ - block_timeout: float = Field(default=0.1, title="wait for blocked item (s)") - """How long to keep waiting for data that's actively being committed to the - cache before giving up and computing it instead. - - In the rare case that two cachable data requests are being processed at the - same time, the first request will block the cache for those specific data - and cause the second request to wait until it can grab this data from the - cache. - This timeout prevents the second request from waiting forever until the - first request finishes (for example, in case it crashes). - """ - reset_on_error: bool = Field(default=False, title="reset the cache if it can't be opened") - """Clear the cache if it can't be opened. - - In rare cases, ``diskcache`` may cache data in a way it can't read back. - To recover from such an error, the cache will be cleared completely. - The only downside to this is decreased performance for a short while. - """ - - _validate_dir = validator('dir', allow_reuse=True, pre=True)( - _Settings._validate_directorypath) - - -class RenderSettings(_Settings): - """Rendering settings - """ - dir: DirectoryPath = Field(default=str(ROOTDIR / 'render'), title="render directory") - """The directory where SVG files should be rendered to - """ - keep: bool = Field(default=False, title="keep files after rendering") - """Keep rendered images after they've been used. - - Disabled by default, you may want to enable this if you want to inspect the - renders. - """ - - _validate_dir = validator('dir', allow_reuse=True, pre=True)( - _Settings._validate_directorypath) - - -class DatabaseSettings(_Settings): - """Database settings. - """ - path: FilePath = Field(default=str(ROOTDIR / 'history.db'), title="database file") - """The path to the database file - """ - cleanup_interval: int = Field(default=7, title='clean-up interval (days)') - """The database can get cluttered after a while, and will be cleaned at - this interval - """ - - _validate_path = validator('path', allow_reuse=True, pre=True)( - _Settings._validate_filepath) - - -class ResultSaveMode(str, Enum): - """Where (or whether) to save the results of an analysis - """ - skip = "skip" - """Don't save results at all - """ - next_to_video = "next to video file" - """Save results in the same directory as the video file that was analyzed - """ - next_to_design = "next to design file" - """Save results in the same directory as the design file that was analyzed - """ - directory = "in result directory" - """Save results in their own directory at - :attr:`shapeflow.ApplicationSettings.result_dir` - """ - - -class ApplicationSettings(_Settings): - """Application settings. - """ - save_state: bool = Field(default=True, title="save application state on exit") - """Whether to save the application state when exiting the application - """ - load_state: bool = Field(default=True, title="load application state on start") - """Whether to load the application state when starting the application - """ - state_path: FilePath = Field(default=str(ROOTDIR / 'state'), title="application state file") - """Where to save the application state - """ - recent_files: int = Field(default=16, title="# of recent files to fetch") - """The number of recent files to show in the user interface - """ - video_pattern: str = Field(default="*.mp4 *.avi *.mov *.mpv *.mkv", title="video file pattern") - """Recognized video file extensions. - Defaults to ``"*.mp4 *.avi *.mov *.mpv *.mkv"``. - """ - design_pattern: str = Field(default="*.svg", title="design file pattern") - """Recognized design file extensions. - Defaults to ``"*.svg"``. - """ - save_result_auto: ResultSaveMode = Field(default=ResultSaveMode.next_to_video, title="result save mode (auto)") - """Where or whether to save results after each run of an analysis - """ - save_result_manual: ResultSaveMode = Field(default=ResultSaveMode.next_to_video, title="result save mode (manual)") - """Where or whether to save results that are exported manually - via the user interface - """ - result_dir: DirectoryPath = Field(default=str(ROOTDIR / 'results'), title="result directory") - """The path to the result directory - """ - cancel_on_q_stop: bool = Field(default=False, title="cancel running analyzers when stopping queue") - """Whether to cancel the currently running analysis when stopping a queue. - - Defaults to ``False``, i.e. the currently running analysis will be - completed first. - """ - threads: int = Field(default=cpu_count(), title="# of threads") - f"""The number of threads the server uses. Defaults to {cpu_count()}, the - number of logical cores of your machine's CPU. - """ - - _validate_dir = validator('result_dir', allow_reuse=True, pre=True)( - _Settings._validate_directorypath) - _validate_state_path = validator('state_path', allow_reuse=True, pre=True)( - _Settings._validate_filepath) - - @validator('threads', pre=True, allow_reuse=True) - def _validate_threads(cls, value): - if value < 8: - return 8 # At least 8 threads to run decently - else: - return value - - -class DefaultConfigSettings(_Settings): - fis: FrameIntervalSetting = Field(default=FrameIntervalSetting('Nf'), title="Frame interval setting") - - dt: float = Field(default=5.0, title="Frame interval in seconds") - - Nf: int = Field(default=100, title="Total number of frames") - - feature: FeatureType = Field(default_factory=FeatureType, title="Feature") - - feature_parameters: FeatureConfig = Field(default_factory=FeatureType.config_class, title="Feature parameter(s)") - - flip: FlipConfig = Field(default_factory=FlipConfig, title="Flip the ROI...") - - turn: int = Field(default=0, title="Turn the ROI ... times (clockwise, 90°)") - - transform: TransformType = Field(default_factory=TransformType, title="Transform") - - transform_config: TransformConfig = Field(default_factory=TransformType.config_class, title="Transform configuration") - - filter: FilterType = Field(default_factory=FilterType, title="Filter") - - filter_config: FilterConfig = Field(default_factory=FilterType.config_class, title="Filter configuration") - - mask_skip: bool = Field(default=False, title="Skip masks") - - -class Settings(_Settings): - """``shapeflow`` settings. - - * app: :class:`~shapeflow.ApplicationSettings` - - * log: :class:`~shapeflow.LogSettings` - - * cache: :class:`~shapeflow.CacheSettings` - - * render: :class:`~shapeflow.RenderSettings` - - * format: :class:`~shapeflow.FormatSettings` - - * db: :class:`~shapeflow.DatabaseSettings` - - * default_config: :class:`~shapeflow.DefaultConfigSettings` - """ - app: ApplicationSettings = Field(default=ApplicationSettings(), title="Application") - log: LogSettings = Field(default=LogSettings(), title="Logging") - cache: CacheSettings = Field(default=CacheSettings(), title="Caching") - render: RenderSettings = Field(default=RenderSettings(), title="SVG Rendering") - format: FormatSettings = Field(default=FormatSettings(), title="Formatting") - db: DatabaseSettings = Field(default=DatabaseSettings(), title="Database") - default_config: DefaultConfigSettings = Field(default=DefaultConfigSettings(), title="Default configuration") - - @classmethod - def from_dict(cls, settings: dict): # todo: deprecate; DefaultConfigSettings introduces deeper nesting & other pydantic subclasses - for k in cls.__fields__.keys(): - if k not in settings: - settings.update({k:{}}) - - return cls( - **{field.name:field.type_(**settings[field.name]) - for field in cls.__fields__.values()} - ) - - -settings: Settings -"""This global :class:`~shapeflow.Settings` object is used throughout the - library -""" - - -def _load_settings() -> Settings: # todo: if there are unexpected fields: warn, don't crash - """Load :class:`~shapeflow.Settings` from .yaml - """ - global settings - - if _SETTINGS_FILE.is_file(): - with open(_SETTINGS_FILE, 'r') as f: - settings_yaml = yaml.safe_load(f) # todo: replace with json - - # Get settings - if settings_yaml is not None: - settings = Settings.from_dict(settings_yaml) - else: - settings = Settings() - - # Move the previous log file to ROOTDIR/log - if Path(settings.log.path).is_file(): - shutil.move( - str(settings.log.path), # todo: convert to pathlib - os.path.join( - settings.log.dir, - datetime.datetime.fromtimestamp( - os.path.getmtime(settings.log.path) - ).strftime(settings.format.datetime_format_fs) + '.log' - ) - ) - - # If more files than specified in ini.log.keep, remove the oldest - files = glob.glob(os.path.join(settings.log.dir, '*.log')) # todo: convert to pathlib - files.sort(key=lambda f: os.path.getmtime(f), reverse=True) - while len(files) > settings.log.keep: - os.remove(files.pop()) - else: - settings = Settings() - - return settings - - -def save_settings(path: str = str(_SETTINGS_FILE)): - """Save :data:`~shapeflow.settings` to .yaml - """ - with open(path, 'w+') as f: - yaml.safe_dump(settings.to_dict(), f) - - -def update_settings(s: dict) -> dict: - """Update the global settings object. - - .. note:: - Just doing ``settings = Settings(**new_settings_dict)`` - would prevent other modules from accessing the updated settings! - - Parameters - ---------- - s : dict - new settings to integrate into the global settings - - Returns - ------- - dict - the current global settings as a ``dict`` - """ - global settings - - for cat, cat_new in s.items(): - sub = getattr(settings, cat) - for kw, val in cat_new.items(): - setattr(sub, kw, val) - - save_settings() - return settings.to_dict() - - -# Instantiate global settings object -_load_settings() -save_settings() \ No newline at end of file diff --git a/shapeflow/video.py b/shapeflow/video.py index 3a1fbb17..e0339b50 100644 --- a/shapeflow/video.py +++ b/shapeflow/video.py @@ -7,10 +7,12 @@ import cv2 import numpy as np import pandas as pd +from pydantic import validator, Field, DirectoryPath from OnionSVG import OnionSVG, check_svg from shapeflow.core.logging import get_logger -from shapeflow.settings import settings +from shapeflow.core.caching import CacheSettings +from shapeflow.core.settings import settings, Category, ROOTDIR from shapeflow.api import api from shapeflow.config import VideoFileHandlerConfig, TransformHandlerConfig, \ FilterHandlerConfig, MaskConfig, \ @@ -194,7 +196,7 @@ def read_frame(self, frame_number: Optional[int] = None) -> np.ndarray: if frame_number is None: frame_number = self.frame_number - if settings.cache.resolve_frame_number: + if settings.get(CacheSettings).resolve_frame_number: frame_number = self._resolve_frame(frame_number) return self.cached_call(self._read_frame, self.path, frame_number) @@ -206,7 +208,7 @@ def seek(self, position: float = None) -> float: # (otherwise streams.update() can get deadocked @ VideoFileHandler if not reading frames from cache) if position is not None: frame_number = int(position * self.frame_count) - if settings.cache.resolve_frame_number: + if settings.get(CacheSettings).resolve_frame_number: self.frame_number = self._resolve_frame(frame_number) else: self.frame_number = frame_number @@ -615,6 +617,24 @@ def skip(self): return self.config.skip +class RenderSettings(Category): + """Rendering settings + """ + dir: DirectoryPath = Field(default=str(ROOTDIR / 'render'), + title="render directory") + """The directory where SVG files should be rendered to + """ + keep: bool = Field(default=False, title="keep files after rendering") + """Keep rendered images after they've been used. + + Disabled by default, you may want to enable this if you want to inspect the + renders. + """ + + _validate_dir = validator('dir', allow_reuse=True, pre=True)( + Category._validate_directorypath) + + class DesignFileHandler(CachingInstance): """Handles design files """ @@ -653,31 +673,31 @@ def config(self) -> DesignFileHandlerConfig: return self._config def _clear_renders(self): - log.debug(f'Clearing render directory {settings.render.dir}') - renders = [f for f in os.listdir(settings.render.dir)] + log.debug(f'Clearing render directory {settings.get(RenderSettings).dir}') + renders = [f for f in os.listdir(settings.get(RenderSettings).dir)] for f in renders: - os.remove(os.path.join(settings.render.dir, f)) + os.remove(os.path.join(settings.get(RenderSettings).dir, f)) def _peel_design(self, design_path, dpi) -> np.ndarray: - if not os.path.isdir(settings.render.dir): - os.mkdir(settings.render.dir) + if not os.path.isdir(settings.get(RenderSettings).dir): + os.mkdir(settings.get(RenderSettings).dir) else: self._clear_renders() check_svg(design_path) OnionSVG(design_path, dpi=dpi).peel( - 'all', to=settings.render.dir # todo: should maybe prepend file name to avoid overwriting previous renders? + 'all', to=settings.get(RenderSettings).dir # todo: should maybe prepend file name to avoid overwriting previous renders? ) print("\n") overlay = cv2.imread( - os.path.join(settings.render.dir, 'overlay.png') + os.path.join(settings.get(RenderSettings).dir, 'overlay.png') ) return overlay def _read_masks(self, _, __) -> Tuple[List[np.ndarray], List[str]]: - files = os.listdir(settings.render.dir) + files = os.listdir(settings.get(RenderSettings).dir) files.remove('overlay.png') # Catch file names of numbered layers @@ -689,7 +709,7 @@ def _read_masks(self, _, __) -> Tuple[List[np.ndarray], List[str]]: for path in files: match = pattern.search(os.path.splitext(path)[0]) - path = os.path.join(settings.render.dir, path) + path = os.path.join(settings.get(RenderSettings).dir, path) if match: matched.update( # numbered layer @@ -720,7 +740,7 @@ def _read_masks(self, _, __) -> Tuple[List[np.ndarray], List[str]]: else: names.append(path) - if not settings.render.keep: + if not settings.get(RenderSettings).keep: self._clear_renders() return masks, names From 7e42b3bb0e2151cec8e5dd7ef306a24f1e57f173 Mon Sep 17 00:00:00 2001 From: ybnd Date: Sat, 30 Jan 2021 13:12:25 +0100 Subject: [PATCH 4/4] WIP -- update tests --- test/test_main.py | 7 +++---- test/test_server.py | 2 +- test/test_video.py | 5 +++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_main.py b/test/test_main.py index 0a389485..d18444cf 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -21,7 +21,7 @@ __VIDEO__ = 'test/' + __VIDEO__ __DESIGN__ = 'test/' + __DESIGN__ -from shapeflow.settings import ROOTDIR, save_settings +from shapeflow import ROOTDIR CACHE = os.path.join(ROOTDIR, 'test_server-cache') DB = os.path.join(ROOTDIR, 'test_server-history.db') @@ -42,7 +42,7 @@ def clear_files(): @contextmanager def application(keep: bool = False): - from shapeflow.settings import settings + from shapeflow.core.settings import settings if not keep: clear_files() @@ -710,8 +710,7 @@ def test_json_streaming(self): class DbCheckTest(unittest.TestCase): def test_db_check(self): - from shapeflow.settings import save_settings - from shapeflow.settings import settings + from shapeflow.core.settings import settings with settings.db.override({'path': DB}): clear_files() diff --git a/test/test_server.py b/test/test_server.py index e37b1cc2..08e9d350 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -8,7 +8,7 @@ import subprocess from shapeflow import ROOTDIR -from shapeflow.settings import settings, save_settings +from shapeflow.core.settings import settings raise unittest.SkipTest('takes too long') diff --git a/test/test_video.py b/test/test_video.py index e4036138..45395323 100644 --- a/test/test_video.py +++ b/test/test_video.py @@ -19,7 +19,8 @@ from shapeflow.video import VideoFileHandler, VideoFileTypeError, \ CachingInstance, VideoAnalyzer -from shapeflow.settings import settings +from shapeflow.core.settings import settings +from shapeflow.core.caching import CacheSettings from shapeflow.core.config import * @@ -63,7 +64,7 @@ TEST_TRANSFORMED_FRAME_HSV[frame_number] = cv2.warpPerspective(frame_hsv, TRANSFORM, dsize, borderValue=(255,255,255)) # Clear cache -with settings.cache.override({'do_cache': True}): +with settings.get(CacheSettings).override({'do_cache': True}): vi = VideoFileHandler(__VIDEO__) assert vi._cache is not None vi._cache.clear()