From 7b713c3a996cd47eb043a3fc2bc7df0d7227ee9e Mon Sep 17 00:00:00 2001 From: Jonathan Visser Date: Thu, 16 Jan 2025 15:50:10 +0100 Subject: [PATCH 1/8] Replace watchdog with pyinotify --- nginx_config_reloader/__init__.py | 116 ++++++++++-------- requirements.txt | 5 +- tests/test_nginx_config_reloader.py | 4 +- ...allbacks.py => test_watchdog_callbacks.py} | 87 +++++-------- 4 files changed, 103 insertions(+), 109 deletions(-) rename tests/{test_inotify_callbacks.py => test_watchdog_callbacks.py} (50%) diff --git a/nginx_config_reloader/__init__.py b/nginx_config_reloader/__init__.py index 6b98d79..04f2442 100644 --- a/nginx_config_reloader/__init__.py +++ b/nginx_config_reloader/__init__.py @@ -13,8 +13,10 @@ import threading import time from typing import Optional +from pathlib import Path -import pyinotify +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler from dasbus.loop import EventLoop from dasbus.signal import Signal @@ -43,8 +45,8 @@ dbus_loop: Optional[EventLoop] = None -class NginxConfigReloader(pyinotify.ProcessEvent): - def my_init( +class NginxConfigReloader(FileSystemEventHandler): + def __init__( self, logger=None, no_magento_config=False, @@ -80,36 +82,51 @@ def my_init( self.applying = False self._on_config_reload = Signal() - def process_IN_DELETE(self, event): + def on_deleted(self, event): """Triggered by inotify on removal of file or removal of dir If the dir itself is removed, inotify will stop watching and also trigger IN_IGNORED. """ - if not event.dir: # Will also capture IN_DELETE_SELF + if not event.is_directory: self.handle_event(event) - def process_IN_MOVED(self, event): + def on_moved(self, event): """Triggered by inotify when a file is moved from or to the dir""" self.handle_event(event) - def process_IN_CREATE(self, event): + def on_created(self, event): """Triggered by inotify when a dir is created in the watch dir""" - if event.dir: + if event.is_directory: self.handle_event(event) - def process_IN_CLOSE_WRITE(self, event): + def on_modified(self, event): """Triggered by inotify when a file is written in the dir""" self.handle_event(event) - def process_IN_MOVE_SELF(self, event): - """Triggered by inotify when watched dir is moved""" - raise ListenTargetTerminated + def on_any_event(self, event): + """Triggered by inotify when watched dir is moved or deleted""" + if event.is_directory and event.event_type in ['moved', 'deleted']: + self.logger.warning(f"Directory {event.src_path} has been {event.event_type}.") + raise ListenTargetTerminated def handle_event(self, event): - if not any(fnmatch.fnmatch(event.name, pat) for pat in WATCH_IGNORE_FILES): - self.logger.info("{} detected on {}.".format(event.maskname, event.name)) + file_path = Path(event.src_path) + if ( + file_path.name.endswith(".swx") + or file_path.name.endswith(".swp") + or file_path.name.endswith("~") + ): + return + + if (event.is_directory): + return + + basename = os.path.basename(event.src_path) + if not any(fnmatch.fnmatch(basename, pat) for pat in WATCH_IGNORE_FILES): + self.logger.debug(f"{event.event_type.upper()} detected on {event.src_path}") self.dirty = True + # Additional handling if necessary def install_magento_config(self): # Check if configs are present @@ -319,6 +336,20 @@ def reload(self, send_signal=True): if send_signal: self._on_config_reload.emit() + def start_observer(self): + self.observer = Observer() + self.observer.schedule( + self, + self.dir_to_watch, + recursive=True, + follow_symlink=True + ) + self.observer.start() + + def stop_observer(self): + self.observer.stop() + self.observer.join() + sys.exit() class ListenTargetTerminated(BaseException): pass @@ -367,20 +398,11 @@ def wait_loop( """ dir_to_watch = os.path.abspath(dir_to_watch) - wm = pyinotify.WatchManager() - notifier = pyinotify.Notifier(wm) - - class SymlinkChangedHandler(pyinotify.ProcessEvent): - def process_IN_DELETE(self, event): - if event.pathname == dir_to_watch: - raise ListenTargetTerminated("watched directory was deleted") - nginx_config_changed_handler = NginxConfigReloader( logger=logger, no_magento_config=no_magento_config, no_custom_config=no_custom_config, dir_to_watch=dir_to_watch, - notifier=notifier, use_systemd=use_systemd, ) @@ -393,35 +415,27 @@ def process_IN_DELETE(self, event): dbus_thread = threading.Thread(target=dbus_event_loop) dbus_thread.start() - while True: - while not os.path.exists(dir_to_watch): - logger.warning( - "Configuration dir {} not found, waiting...".format(dir_to_watch) - ) - time.sleep(5) - - wm.add_watch( - dir_to_watch, - pyinotify.ALL_EVENTS, - nginx_config_changed_handler, - rec=recursive_watch, - auto_add=True, - ) - wm.watch_transient_file( - dir_to_watch, pyinotify.ALL_EVENTS, SymlinkChangedHandler + while not os.path.exists(dir_to_watch): + logger.warning( + "Configuration dir {} not found, waiting...".format(dir_to_watch) ) - - # Install initial configuration - nginx_config_changed_handler.reload(send_signal=False) - - try: - logger.info("Listening for changes to {}".format(dir_to_watch)) - notifier.coalesce_events() - notifier.loop(callback=lambda _: after_loop(nginx_config_changed_handler)) - except pyinotify.NotifierError as err: - logger.critical(err) - except ListenTargetTerminated: - logger.warning("Configuration dir lost, waiting for it to reappear") + time.sleep(5) + + + try: + logger.info(f"Listening for changes to {dir_to_watch}") + nginx_config_changed_handler.start_observer() + while True: + time.sleep(1) + after_loop(nginx_config_changed_handler) + except ListenTargetTerminated: + logger.warning("Configuration dir lost, waiting for it to reappear") + nginx_config_changed_handler.stop_observer() + time.sleep(5) + except KeyboardInterrupt: + logger.info("Shutting down observer.") + nginx_config_changed_handler.stop_observer() + def as_unprivileged_user(): diff --git a/requirements.txt b/requirements.txt index ea3bf07..b732bff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -pyinotify==0.9.6 +# Follow_symlinks isnt released in 6.0.0 yet, so we use the latest current commit +watchdog @ git+https://github.com/gorakhargosh/watchdog.git@f3e78cd4d9500d287bd11ec5d08a1f351601028d mock==5.0.1 pytest==7.2.1 @@ -8,4 +9,4 @@ black==23.1.0 pre-commit==2.21.0 pygobject pygobject-stubs -dasbus==1.7 +dasbus==1.7 \ No newline at end of file diff --git a/tests/test_nginx_config_reloader.py b/tests/test_nginx_config_reloader.py index 49fa8fa..f5a6d65 100644 --- a/tests/test_nginx_config_reloader.py +++ b/tests/test_nginx_config_reloader.py @@ -699,4 +699,6 @@ def _dest(self, name): class Event: def __init__(self, name): self.name = name - self.maskname = "IN_CLOSE_WRITE" + self.event_type = "modified" + self.src_path = name + self.is_directory = False diff --git a/tests/test_inotify_callbacks.py b/tests/test_watchdog_callbacks.py similarity index 50% rename from tests/test_inotify_callbacks.py rename to tests/test_watchdog_callbacks.py index 2093977..acc4b65 100644 --- a/tests/test_inotify_callbacks.py +++ b/tests/test_watchdog_callbacks.py @@ -4,12 +4,20 @@ from tempfile import NamedTemporaryFile, mkdtemp import mock -import pyinotify +from watchdog.events import ( + DirCreatedEvent, + DirDeletedEvent, + DirMovedEvent, + FileCreatedEvent, + FileDeletedEvent, + FileModifiedEvent, + FileMovedEvent, +) import nginx_config_reloader -class TestInotifyCallbacks(unittest.TestCase): +class TestWatchdogCallbacks(unittest.TestCase): def setUp(self): patcher = mock.patch("nginx_config_reloader.NginxConfigReloader.handle_event") self.addCleanup(patcher.stop) @@ -19,87 +27,58 @@ def setUp(self): with open(os.path.join(self.dir, "existing_file"), "w") as f: f.write("blablabla") - wm = pyinotify.WatchManager() - handler = nginx_config_reloader.NginxConfigReloader() - self.notifier = pyinotify.Notifier(wm, default_proc_fun=handler) - wm.add_watch(self.dir, pyinotify.ALL_EVENTS) + self.observer = mock.Mock() + self.handler = nginx_config_reloader.NginxConfigReloader(dir_to_watch=self.dir) + self.handler.observer = self.observer def tearDown(self): - self.notifier.stop() shutil.rmtree(self.dir, ignore_errors=True) - def _process_events(self): - while self.notifier.check_events(0): - self.notifier.read_events() - self.notifier.process_events() - - def test_that_handle_event_is_called_when_new_file_is_created(self): - with open(os.path.join(self.dir, "testfile"), "w") as f: - f.write("blablabla") - - self._process_events() - self.assertEqual(len(self.handle_event.mock_calls), 1) def test_that_handle_event_is_called_when_new_dir_is_created(self): - mkdtemp(dir=self.dir) - self._process_events() + event = DirCreatedEvent(os.path.join(self.dir, "testdir")) + self.handler.on_created(event) self.assertEqual(len(self.handle_event.mock_calls), 1) def test_that_handle_event_is_called_when_a_file_is_removed(self): - os.remove(os.path.join(self.dir, "existing_file")) - - self._process_events() + event = FileDeletedEvent(os.path.join(self.dir, "existing_file")) + self.handler.on_deleted(event) self.assertEqual(len(self.handle_event.mock_calls), 1) def test_that_handle_event_is_called_when_a_file_is_moved_in(self): with NamedTemporaryFile(delete=False) as f: - os.rename(f.name, os.path.join(self.dir, "newfile")) - - self._process_events() + event = FileMovedEvent( + f.name, os.path.join(self.dir, "newfile") + ) + self.handler.on_moved(event) self.assertEqual(len(self.handle_event.mock_calls), 1) def test_that_handle_event_is_called_when_a_file_is_moved_out(self): destdir = mkdtemp() - os.rename( + event = FileMovedEvent( os.path.join(self.dir, "existing_file"), os.path.join(destdir, "existing_file"), ) - - self._process_events() + self.handler.on_moved(event) self.assertEqual(len(self.handle_event.mock_calls), 1) shutil.rmtree(destdir) def test_that_handle_event_is_called_when_a_file_is_renamed(self): - os.rename( - os.path.join(self.dir, "existing_file"), os.path.join(self.dir, "new_name") + event = FileMovedEvent( + os.path.join(self.dir, "existing_file"), + os.path.join(self.dir, "new_name"), ) - - self._process_events() + self.handler.on_moved(event) self.assertGreaterEqual(len(self.handle_event.mock_calls), 1) - def test_that_listen_target_terminated_is_raised_if_dir_is_renamed(self): - destdir = mkdtemp() - os.rename(self.dir, destdir) - - with self.assertRaises(nginx_config_reloader.ListenTargetTerminated): - self._process_events() - - shutil.rmtree(destdir) - - def test_that_listen_target_terminated_is_not_raised_if_dir_is_removed(self): - shutil.rmtree(self.dir) - - self._process_events() - - -class TestInotifyRecursiveCallbacks(TestInotifyCallbacks): +class TestWatchdogRecursiveCallbacks(TestWatchdogCallbacks): # Run all callback tests on a subdir def setUp(self): patcher = mock.patch("nginx_config_reloader.NginxConfigReloader.handle_event") @@ -111,11 +90,9 @@ def setUp(self): with open(os.path.join(self.dir, "existing_file"), "w") as f: f.write("blablabla") - wm = pyinotify.WatchManager() - handler = nginx_config_reloader.NginxConfigReloader() - self.notifier = pyinotify.Notifier(wm, default_proc_fun=handler) - wm.add_watch(self.rootdir, pyinotify.ALL_EVENTS, rec=True) + self.observer = mock.Mock() + self.handler = nginx_config_reloader.NginxConfigReloader(dir_to_watch=self.dir) + self.handler.observer = self.observer def tearDown(self): - self.notifier.stop() - shutil.rmtree(self.rootdir, ignore_errors=True) + shutil.rmtree(self.rootdir, ignore_errors=True) \ No newline at end of file From aa750a96c6c00cf9fb44cf7275916d0d771d74f1 Mon Sep 17 00:00:00 2001 From: Jonathan Visser Date: Thu, 16 Jan 2025 18:59:54 +0100 Subject: [PATCH 2/8] Fix linting issues --- nginx_config_reloader/__init__.py | 30 +++++++++++++++--------------- requirements.txt | 2 +- tests/test_watchdog_callbacks.py | 17 ++++------------- 3 files changed, 20 insertions(+), 29 deletions(-) diff --git a/nginx_config_reloader/__init__.py b/nginx_config_reloader/__init__.py index 04f2442..96cf70b 100644 --- a/nginx_config_reloader/__init__.py +++ b/nginx_config_reloader/__init__.py @@ -12,13 +12,13 @@ import sys import threading import time -from typing import Optional from pathlib import Path +from typing import Optional -from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler from dasbus.loop import EventLoop from dasbus.signal import Signal +from watchdog.events import FileSystemEventHandler +from watchdog.observers import Observer from nginx_config_reloader.copy_files import safe_copy_files from nginx_config_reloader.dbus.common import NGINX_CONFIG_RELOADER, SYSTEM_BUS @@ -106,8 +106,10 @@ def on_modified(self, event): def on_any_event(self, event): """Triggered by inotify when watched dir is moved or deleted""" - if event.is_directory and event.event_type in ['moved', 'deleted']: - self.logger.warning(f"Directory {event.src_path} has been {event.event_type}.") + if event.is_directory and event.event_type in ["moved", "deleted"]: + self.logger.warning( + f"Directory {event.src_path} has been {event.event_type}." + ) raise ListenTargetTerminated def handle_event(self, event): @@ -118,13 +120,15 @@ def handle_event(self, event): or file_path.name.endswith("~") ): return - - if (event.is_directory): + + if event.is_directory: return - + basename = os.path.basename(event.src_path) if not any(fnmatch.fnmatch(basename, pat) for pat in WATCH_IGNORE_FILES): - self.logger.debug(f"{event.event_type.upper()} detected on {event.src_path}") + self.logger.debug( + f"{event.event_type.upper()} detected on {event.src_path}" + ) self.dirty = True # Additional handling if necessary @@ -339,10 +343,7 @@ def reload(self, send_signal=True): def start_observer(self): self.observer = Observer() self.observer.schedule( - self, - self.dir_to_watch, - recursive=True, - follow_symlink=True + self, self.dir_to_watch, recursive=True, follow_symlink=True ) self.observer.start() @@ -351,6 +352,7 @@ def stop_observer(self): self.observer.join() sys.exit() + class ListenTargetTerminated(BaseException): pass @@ -421,7 +423,6 @@ def wait_loop( ) time.sleep(5) - try: logger.info(f"Listening for changes to {dir_to_watch}") nginx_config_changed_handler.start_observer() @@ -435,7 +436,6 @@ def wait_loop( except KeyboardInterrupt: logger.info("Shutting down observer.") nginx_config_changed_handler.stop_observer() - def as_unprivileged_user(): diff --git a/requirements.txt b/requirements.txt index b732bff..92f4906 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,4 @@ black==23.1.0 pre-commit==2.21.0 pygobject pygobject-stubs -dasbus==1.7 \ No newline at end of file +dasbus==1.7 diff --git a/tests/test_watchdog_callbacks.py b/tests/test_watchdog_callbacks.py index acc4b65..76569e6 100644 --- a/tests/test_watchdog_callbacks.py +++ b/tests/test_watchdog_callbacks.py @@ -4,15 +4,7 @@ from tempfile import NamedTemporaryFile, mkdtemp import mock -from watchdog.events import ( - DirCreatedEvent, - DirDeletedEvent, - DirMovedEvent, - FileCreatedEvent, - FileDeletedEvent, - FileModifiedEvent, - FileMovedEvent, -) +from watchdog.events import DirCreatedEvent, FileDeletedEvent, FileMovedEvent import nginx_config_reloader @@ -50,9 +42,7 @@ def test_that_handle_event_is_called_when_a_file_is_removed(self): def test_that_handle_event_is_called_when_a_file_is_moved_in(self): with NamedTemporaryFile(delete=False) as f: - event = FileMovedEvent( - f.name, os.path.join(self.dir, "newfile") - ) + event = FileMovedEvent(f.name, os.path.join(self.dir, "newfile")) self.handler.on_moved(event) self.assertEqual(len(self.handle_event.mock_calls), 1) @@ -78,6 +68,7 @@ def test_that_handle_event_is_called_when_a_file_is_renamed(self): self.assertGreaterEqual(len(self.handle_event.mock_calls), 1) + class TestWatchdogRecursiveCallbacks(TestWatchdogCallbacks): # Run all callback tests on a subdir def setUp(self): @@ -95,4 +86,4 @@ def setUp(self): self.handler.observer = self.observer def tearDown(self): - shutil.rmtree(self.rootdir, ignore_errors=True) \ No newline at end of file + shutil.rmtree(self.rootdir, ignore_errors=True) From eeeb5f668aaaa0c0f3bc442a4c0259385bac3603 Mon Sep 17 00:00:00 2001 From: Timon de Groot Date: Wed, 7 Jan 2026 15:06:10 +0100 Subject: [PATCH 3/8] Fix watchdog regressions Some behavior was changed with the initial watchdog implementation. --- nginx_config_reloader/__init__.py | 52 +++++++++++++++++-------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/nginx_config_reloader/__init__.py b/nginx_config_reloader/__init__.py index 96cf70b..c2a587c 100644 --- a/nginx_config_reloader/__init__.py +++ b/nginx_config_reloader/__init__.py @@ -53,7 +53,6 @@ def __init__( no_custom_config=False, dir_to_watch=DIR_TO_WATCH, magento2_flag=None, - notifier=None, use_systemd=False, ): """Constructor called by ProcessEvent @@ -76,7 +75,6 @@ def __init__( else: self.magento2_flag = magento2_flag self.logger.info(self.dir_to_watch) - self.notifier = notifier self.use_systemd = use_systemd self.dirty = False self.applying = False @@ -107,10 +105,11 @@ def on_modified(self, event): def on_any_event(self, event): """Triggered by inotify when watched dir is moved or deleted""" if event.is_directory and event.event_type in ["moved", "deleted"]: - self.logger.warning( - f"Directory {event.src_path} has been {event.event_type}." - ) - raise ListenTargetTerminated + if event.src_path == self.dir_to_watch: + self.logger.warning( + f"Directory {event.src_path} has been {event.event_type}." + ) + raise ListenTargetTerminated def handle_event(self, event): file_path = Path(event.src_path) @@ -350,7 +349,6 @@ def start_observer(self): def stop_observer(self): self.observer.stop() self.observer.join() - sys.exit() class ListenTargetTerminated(BaseException): @@ -373,7 +371,7 @@ def dbus_event_loop(): def wait_loop( - logger=None, + logger: logging.Logger, no_magento_config=False, no_custom_config=False, dir_to_watch=DIR_TO_WATCH, @@ -384,9 +382,9 @@ def wait_loop( """Main event loop There is an outer loop that checks the availability of the directory to watch. - As soon as it becomes available, it starts an inotify-monitor that monitors + As soon as it becomes available, it starts a watchdog observer that monitors configuration changes in an inner event loop. When the monitored directory is - renamed or removed, the inotify-handler raises an exception to break out of the + renamed or removed, the handler raises an exception to break out of the inner loop and we're back here in the outer loop. :param logging.Logger logger: The logger object @@ -423,19 +421,25 @@ def wait_loop( ) time.sleep(5) - try: - logger.info(f"Listening for changes to {dir_to_watch}") - nginx_config_changed_handler.start_observer() - while True: - time.sleep(1) - after_loop(nginx_config_changed_handler) - except ListenTargetTerminated: - logger.warning("Configuration dir lost, waiting for it to reappear") - nginx_config_changed_handler.stop_observer() - time.sleep(5) - except KeyboardInterrupt: - logger.info("Shutting down observer.") - nginx_config_changed_handler.stop_observer() + running = True + while running: + # Install initial configuration + nginx_config_changed_handler.reload(send_signal=False) + + try: + logger.info(f"Listening for changes to {dir_to_watch}") + nginx_config_changed_handler.start_observer() + while True: + time.sleep(1) + after_loop(nginx_config_changed_handler) + except ListenTargetTerminated: + logger.warning("Configuration dir lost, waiting for it to reappear") + nginx_config_changed_handler.stop_observer() + time.sleep(5) + except KeyboardInterrupt: + logger.info("Shutting down observer.") + nginx_config_changed_handler.stop_observer() + running = False def as_unprivileged_user(): @@ -487,7 +491,7 @@ def parse_nginx_config_reloader_arguments(): return parser.parse_args() -def get_logger(): +def get_logger() -> logging.Logger: handler = logging.StreamHandler() handler.setFormatter( logging.Formatter("%(asctime)s %(name)-12s %(levelname)-8s %(message)s") From 5d553e57cd7c045ca6f203e0e696e83234780041 Mon Sep 17 00:00:00 2001 From: Timon de Groot Date: Wed, 7 Jan 2026 15:08:13 +0100 Subject: [PATCH 4/8] Remove obsolete watch ignores --- nginx_config_reloader/__init__.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/nginx_config_reloader/__init__.py b/nginx_config_reloader/__init__.py index c2a587c..cd58931 100644 --- a/nginx_config_reloader/__init__.py +++ b/nginx_config_reloader/__init__.py @@ -12,7 +12,6 @@ import sys import threading import time -from pathlib import Path from typing import Optional from dasbus.loop import EventLoop @@ -112,14 +111,6 @@ def on_any_event(self, event): raise ListenTargetTerminated def handle_event(self, event): - file_path = Path(event.src_path) - if ( - file_path.name.endswith(".swx") - or file_path.name.endswith(".swp") - or file_path.name.endswith("~") - ): - return - if event.is_directory: return @@ -129,7 +120,6 @@ def handle_event(self, event): f"{event.event_type.upper()} detected on {event.src_path}" ) self.dirty = True - # Additional handling if necessary def install_magento_config(self): # Check if configs are present From 31b3d70daefb13b262a95d1b1d0b5604fa491c7f Mon Sep 17 00:00:00 2001 From: Timon de Groot Date: Wed, 7 Jan 2026 15:15:16 +0100 Subject: [PATCH 5/8] debian/control: Update dependencies --- debian/control | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/debian/control b/debian/control index c9afbee..47f112b 100644 --- a/debian/control +++ b/debian/control @@ -2,12 +2,12 @@ Source: nginx-config-reloader Maintainer: Hypernode Tech Team Section: misc Priority: optional -Standards-Version: 3.9.2 -Build-Depends: debhelper (>= 9), python3, python3-pyinotify, python3-setuptools, dh-python -X-Python3-Version: >= 3.5 +Standards-Version: 4.7.2 +Build-Depends: debhelper-compat (= 13), python3, python3-watchdog, python3-setuptools, dh-python +X-Python3-Version: >= 3.11 Package: nginx-config-reloader Architecture: all -Depends: ${python3:Depends}, ${misc:Depends}, python3-pyinotify, python3-dasbus +Depends: ${python3:Depends}, ${misc:Depends}, python3-watchdog, python3-dasbus Description: nginx config files reloader Daemon that installs and reloads nginx configuration From 070ab3456cf5610eb810fd8e3fe0778fd0a98b22 Mon Sep 17 00:00:00 2001 From: Timon de Groot Date: Wed, 7 Jan 2026 15:46:35 +0100 Subject: [PATCH 6/8] Fix unit tests --- requirements.txt | 7 ++--- tests/test_after_loop.py | 7 ++--- ...t_assert_forbidden_statements_in_config.py | 28 +++++++++++++----- tests/test_main.py | 3 +- tests/test_nginx_config_reloader.py | 29 +++++++++++++++---- ...t_parse_nginx_config_reloader_arguments.py | 2 +- tests/test_watchdog_callbacks.py | 2 +- tests/testcase.py | 3 +- tox.ini | 5 ++-- 9 files changed, 58 insertions(+), 28 deletions(-) diff --git a/requirements.txt b/requirements.txt index 92f4906..adcada4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,11 @@ # Follow_symlinks isnt released in 6.0.0 yet, so we use the latest current commit watchdog @ git+https://github.com/gorakhargosh/watchdog.git@f3e78cd4d9500d287bd11ec5d08a1f351601028d -mock==5.0.1 -pytest==7.2.1 -pytest-xdist==3.2.0 +pytest==9.0.2 +pytest-xdist==3.8.0 tox==4.4.5 black==23.1.0 pre-commit==2.21.0 -pygobject +pygobject<3.51 pygobject-stubs dasbus==1.7 diff --git a/tests/test_after_loop.py b/tests/test_after_loop.py index 4907317..b603e97 100644 --- a/tests/test_after_loop.py +++ b/tests/test_after_loop.py @@ -1,4 +1,3 @@ -from collections import deque from tempfile import mkdtemp from unittest.mock import Mock @@ -9,7 +8,9 @@ class TestAfterLoop(TestCase): def setUp(self) -> None: self.source = mkdtemp() - self.notifier = Mock(_eventq=deque(range(5))) + self.set_up_patch( + "nginx_config_reloader.directory_is_unmounted", return_value=False + ) def test_it_returns_nothing(self): tm = self._get_nginx_config_reloader_instance() @@ -43,12 +44,10 @@ def _get_nginx_config_reloader_instance( no_magento_config=False, no_custom_config=False, magento2_flag=None, - notifier=None, ): return nginx_config_reloader.NginxConfigReloader( no_magento_config=no_magento_config, no_custom_config=no_custom_config, dir_to_watch=self.source, magento2_flag=magento2_flag, - notifier=notifier or self.notifier, ) diff --git a/tests/test_assert_forbidden_statements_in_config.py b/tests/test_assert_forbidden_statements_in_config.py index b1e7d33..50dea3f 100644 --- a/tests/test_assert_forbidden_statements_in_config.py +++ b/tests/test_assert_forbidden_statements_in_config.py @@ -1,9 +1,17 @@ -import pipes +import shlex +import sys from subprocess import CalledProcessError, check_output +import pytest + from nginx_config_reloader import FORBIDDEN_CONFIG_REGEX, NginxConfigReloader from tests.testcase import TestCase +# Skip marker for tests that require grep -P (PCRE support, not available on macOS) +requires_pcre_grep = pytest.mark.skipif( + sys.platform != "linux", reason="Requires grep -P (PCRE support)" +) + class TestAssertNoForbiddenStatementsInConfig(TestCase): def setUp(self): @@ -24,6 +32,7 @@ def test_assert_no_includes_in_config_does_not_check_config_if_no_dir_to_watch( self.assertFalse(self.check_output.called) + @requires_pcre_grep def test_include_prevention_legal_includes(self): TEST_CASES = [ "include /etc/nginx/fastcgi_params", @@ -41,11 +50,12 @@ def test_include_prevention_legal_includes(self): for line in TEST_CASES: check_output( "[ $(echo {} | grep -P '{}' | wc -l) -lt 1 ]".format( - pipes.quote(line), FORBIDDEN_CONFIG_REGEX[2][0] + shlex.quote(line), FORBIDDEN_CONFIG_REGEX[2][0] ), shell=True, ) + @requires_pcre_grep def test_include_prevention_illegal_includes(self): TEST_CASES = [ "include /data/web/nginx/someexample.allow;", @@ -67,11 +77,12 @@ def test_include_prevention_illegal_includes(self): with self.assertRaises(CalledProcessError): check_output( "[ $(echo {} | grep -P '{}' | wc -l) -lt 1 ]".format( - pipes.quote(line), FORBIDDEN_CONFIG_REGEX[2][0] + shlex.quote(line), FORBIDDEN_CONFIG_REGEX[2][0] ), shell=True, ) + @requires_pcre_grep def test_forbidden_config_client_body_temp_path_regex_unhappy_case(self): TEST_CASES = [ "client_body_temp_path /tmp/path", @@ -85,11 +96,12 @@ def test_forbidden_config_client_body_temp_path_regex_unhappy_case(self): with self.assertRaises(CalledProcessError): check_output( "[ $(echo {} | grep -P '{}' | wc -l) -lt 1 ]".format( - pipes.quote(test), FORBIDDEN_CONFIG_REGEX[0][0] + shlex.quote(test), FORBIDDEN_CONFIG_REGEX[0][0] ), shell=True, ) + @requires_pcre_grep def test_forbidden_access_or_error_log_configuration_options(self): TEST_CASES = [ "access_log /var/log/nginx/acceptatie.log;", @@ -116,11 +128,12 @@ def test_forbidden_access_or_error_log_configuration_options(self): with self.assertRaises(CalledProcessError): check_output( "[ $(echo {} | grep -P '{}' | wc -l) -lt 1 ]".format( - pipes.quote(test), FORBIDDEN_CONFIG_REGEX[1][0] + shlex.quote(test), FORBIDDEN_CONFIG_REGEX[1][0] ), shell=True, ) + @requires_pcre_grep def test_allowed_access_or_error_log_configuration_options(self): TEST_CASES = [ "access_log /data/var/log/access.log;", @@ -137,11 +150,12 @@ def test_allowed_access_or_error_log_configuration_options(self): for test in TEST_CASES: check_output( "[ $(echo {} | grep -P '{}' | wc -l) -lt 1 ]".format( - pipes.quote(test), FORBIDDEN_CONFIG_REGEX[1][0] + shlex.quote(test), FORBIDDEN_CONFIG_REGEX[1][0] ), shell=True, ) + @requires_pcre_grep def test_forbidden_config_init_by_lua_regex_matches_target_directives(self): TEST_CASES = ["init_by_lua", "init_by_lua_block", "init_by_lua_file"] @@ -149,7 +163,7 @@ def test_forbidden_config_init_by_lua_regex_matches_target_directives(self): with self.assertRaises(CalledProcessError): check_output( "[ $(echo {} | grep -P '{}' | wc -l) -lt 1 ]".format( - pipes.quote(test), FORBIDDEN_CONFIG_REGEX[3][0] + shlex.quote(test), FORBIDDEN_CONFIG_REGEX[3][0] ), shell=True, ) diff --git a/tests/test_main.py b/tests/test_main.py index e35c532..18ed317 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,7 +1,6 @@ import shutil from tempfile import mkdtemp - -from mock import Mock +from unittest.mock import Mock from nginx_config_reloader import main from tests.testcase import TestCase diff --git a/tests/test_nginx_config_reloader.py b/tests/test_nginx_config_reloader.py index f5a6d65..536916f 100644 --- a/tests/test_nginx_config_reloader.py +++ b/tests/test_nginx_config_reloader.py @@ -3,15 +3,21 @@ import signal import stat import subprocess -from collections import deque +import sys from tempfile import NamedTemporaryFile, mkdtemp, mkstemp +from unittest import mock from unittest.mock import Mock -import mock +import pytest import nginx_config_reloader from tests.testcase import TestCase +# Skip marker for tests that require Linux-specific features (rsync with --chown, etc.) +requires_linux = pytest.mark.skipif( + sys.platform != "linux", reason="Requires Linux-specific rsync features" +) + class TestConfigReloader(TestCase): def setUp(self): @@ -45,7 +51,6 @@ def setUp(self): self.error_file = os.path.join( nginx_config_reloader.DIR_TO_WATCH, nginx_config_reloader.ERROR_FILE ) - self.notifier = Mock(_eventq=deque(range(5))) def tearDown(self): shutil.rmtree(self.source, ignore_errors=True) @@ -58,6 +63,7 @@ def tearDown(self): except OSError: pass + @requires_linux def test_that_apply_new_config_moves_files_to_dest_dir(self): self._write_file(self._source("myfile"), "config contents") @@ -67,6 +73,7 @@ def test_that_apply_new_config_moves_files_to_dest_dir(self): contents = self._read_file(self._dest("myfile")) self.assertEqual(contents, "config contents") + @requires_linux def test_that_apply_config_moves_files_to_dest_dir_if_it_doesnt_yet_exist(self): self._write_file(self._source("myfile"), "config contents") shutil.rmtree(self.dest, ignore_errors=True) @@ -163,12 +170,14 @@ def test_that_apply_new_config_keeps_files_in_source_dir(self): contents = self._read_file(self._source("myfile")) self.assertEqual(contents, "config contents") + @requires_linux def test_that_apply_new_config_sends_hup_to_nginx(self): tm = self._get_nginx_config_reloader_instance() tm.apply_new_config() self.kill.assert_called_once_with(42, signal.SIGHUP) + @requires_linux def test_that_apply_new_config_removes_error_file_when_config_correct_and_ignores_all_oserrors( self, ): @@ -249,6 +258,7 @@ def test_that_apply_new_config_writes_error_message_to_source_dir_if_include_is_ contents = self._read_file(self._source(nginx_config_reloader.ERROR_FILE)) self.assertIn(nginx_config_reloader.FORBIDDEN_CONFIG_REGEX[0][1], contents) + @requires_linux def test_that_apply_new_config_does_not_check_includes_if_dir_to_watch_does_not_exist( self, ): @@ -551,6 +561,7 @@ def test_reloader_doesnt_crash_if_source_dir_is_empty(self): tm = self._get_nginx_config_reloader_instance() tm.apply_new_config() + @requires_linux def test_files_are_copied(self): with open(os.path.join(self.source, "server.test.cnf"), "w") as fp: fp.write("test") @@ -560,18 +571,21 @@ def test_files_are_copied(self): with open(os.path.join(self.dest, "server.test.cnf")) as fp: self.assertIn("test", fp.read()) + @requires_linux def test_new_dir_is_placed(self): os.mkdir(os.path.join(self.source, "new_dir")) tm = self._get_nginx_config_reloader_instance() tm.apply_new_config() self.assertTrue(os.path.exists(os.path.join(self.dest, "new_dir"))) + @requires_linux def test_dotfiles_are_ignored(self): os.mkdir(os.path.join(self.source, ".git")) tm = self._get_nginx_config_reloader_instance() tm.apply_new_config() self.assertFalse(os.path.exists(os.path.join(self.dest, ".git"))) + @requires_linux def test_symlink_to_file_is_copied_to_file(self): with open(os.path.join(self.source, "server.test.cnf"), "w") as fp: fp.write("test") @@ -584,6 +598,7 @@ def test_symlink_to_file_is_copied_to_file(self): self.assertFalse(os.path.islink(os.path.join(self.dest, "symlink"))) self.assertTrue(os.path.isfile(os.path.join(self.dest, "symlink"))) + @requires_linux def test_symlink_to_dir_is_copied_to_dir(self): os.mkdir(os.path.join(self.source, "new_dir")) os.symlink( @@ -594,6 +609,7 @@ def test_symlink_to_dir_is_copied_to_dir(self): self.assertFalse(os.path.islink(os.path.join(self.dest, "symlink"))) self.assertTrue(os.path.isdir(os.path.join(self.dest, "symlink"))) + @requires_linux def test_sticky_bits_are_removed_from_dir(self): os.mkdir(os.path.join(self.source, "new_dir")) os.chmod(os.path.join(self.source, "new_dir"), 0o4755) @@ -603,6 +619,7 @@ def test_sticky_bits_are_removed_from_dir(self): str(oct(os.stat(os.path.join(self.dest, "new_dir")).st_mode))[-5:], "40755" ) + @requires_linux def test_sticky_bits_are_removed_from_file(self): with open(os.path.join(self.source, "server.test.cnf"), "w") as fp: fp.write("test") @@ -614,6 +631,7 @@ def test_sticky_bits_are_removed_from_file(self): "0644", ) + @requires_linux def test_dir_is_chmodded_to_0755(self): os.mkdir(os.path.join(self.source, "new_dir")) os.chmod(os.path.join(self.source, "new_dir"), 0o777) @@ -623,6 +641,7 @@ def test_dir_is_chmodded_to_0755(self): str(oct(os.stat(os.path.join(self.dest, "new_dir")).st_mode))[-5:], "40755" ) + @requires_linux def test_execute_permissions_are_stripped_for_others(self): with open(os.path.join(self.source, "server.test.cnf"), "w") as fp: fp.write("test") @@ -633,6 +652,7 @@ def test_execute_permissions_are_stripped_for_others(self): os.stat(os.path.join(self.dest, "server.test.cnf")).st_mode & stat.S_IXOTH ) + @requires_linux def test_write_permissions_are_stripped_for_others(self): with open(os.path.join(self.source, "server.test.cnf"), "w") as fp: fp.write("test") @@ -643,6 +663,7 @@ def test_write_permissions_are_stripped_for_others(self): os.stat(os.path.join(self.dest, "server.test.cnf")).st_mode & stat.S_IWOTH ) + @requires_linux def test_permissions_are_masked_for_file_in_subdir(self): os.mkdir(os.path.join(self.source, "new_dir")) with open(os.path.join(self.source, "new_dir/server.test.cnf"), "w") as fp: @@ -671,14 +692,12 @@ def _get_nginx_config_reloader_instance( no_magento_config=False, no_custom_config=False, magento2_flag=None, - notifier=None, ): return nginx_config_reloader.NginxConfigReloader( no_magento_config=no_magento_config, no_custom_config=no_custom_config, dir_to_watch=self.source, magento2_flag=magento2_flag, - notifier=notifier or self.notifier, ) def _write_file(self, name, contents): diff --git a/tests/test_parse_nginx_config_reloader_arguments.py b/tests/test_parse_nginx_config_reloader_arguments.py index 81531d7..076ab9e 100644 --- a/tests/test_parse_nginx_config_reloader_arguments.py +++ b/tests/test_parse_nginx_config_reloader_arguments.py @@ -1,4 +1,4 @@ -from mock import call +from unittest.mock import call import nginx_config_reloader from nginx_config_reloader import parse_nginx_config_reloader_arguments diff --git a/tests/test_watchdog_callbacks.py b/tests/test_watchdog_callbacks.py index 76569e6..b3ef8e4 100644 --- a/tests/test_watchdog_callbacks.py +++ b/tests/test_watchdog_callbacks.py @@ -2,8 +2,8 @@ import shutil import unittest from tempfile import NamedTemporaryFile, mkdtemp +from unittest import mock -import mock from watchdog.events import DirCreatedEvent, FileDeletedEvent, FileMovedEvent import nginx_config_reloader diff --git a/tests/testcase.py b/tests/testcase.py index b511f07..ab65141 100644 --- a/tests/testcase.py +++ b/tests/testcase.py @@ -1,7 +1,6 @@ import sys import unittest - -from mock import Mock, mock_open, patch +from unittest.mock import Mock, mock_open, patch class TestCase(unittest.TestCase): diff --git a/tox.ini b/tox.ini index 93225ba..5b69868 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,14 @@ [tox] -envlist=py311 +envlist=py311,py314 skipsdist=True [gh-actions] python = 3.11: py311 + 3.14: py314 [testenv] deps = -rrequirements.txt allowlist_externals=bash commands= - bash -c "pytest tests -n $(nproc)" + bash -c "pytest tests" From 67b4be590bd7efd6fd716aaee4e4c4c441c5377f Mon Sep 17 00:00:00 2001 From: Timon de Groot Date: Thu, 8 Jan 2026 12:33:12 +0100 Subject: [PATCH 7/8] debian/rules: Remove --with systemd This is now implicit, debian build system fails on it now. --- debian/rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/rules b/debian/rules index ee05022..da83a69 100755 --- a/debian/rules +++ b/debian/rules @@ -1,7 +1,7 @@ #!/usr/bin/make -f %: - dh $@ --with python3 --with systemd --buildsystem=pybuild + dh $@ --with python3 --buildsystem=pybuild override_dh_auto_test: # skipping tests. we don't want to test in the build env, From 491c14da0763ac45fcc685cb66868d9e8ead2c91 Mon Sep 17 00:00:00 2001 From: Timon de Groot Date: Thu, 8 Jan 2026 12:36:03 +0100 Subject: [PATCH 8/8] debian: Remove nginx-config-reloader.upstart --- debian/nginx-config-reloader.upstart | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100755 debian/nginx-config-reloader.upstart diff --git a/debian/nginx-config-reloader.upstart b/debian/nginx-config-reloader.upstart deleted file mode 100755 index b8adab1..0000000 --- a/debian/nginx-config-reloader.upstart +++ /dev/null @@ -1,20 +0,0 @@ -# nginx-config-reloader - Daemon that detects, checks and installs user provided nginx configuration files - -description "Daemon that detects, checks and installs user provided nginx configuration files" -author "Rick van de Loo " - -start on runlevel [2345] -stop on runlevel [016] - -# Infinite respawn -respawn -respawn limit unlimited - -exec /usr/bin/nginx_config_reloader - -post-stop script - goal=$(initctl status $UPSTART_JOB | cut -d' ' -f 2 | cut -d/ -f 1) - if [ "$goal" != "stop" ]; then - sleep 10; - fi -end script