From 1f8dcea6cdbe78925bec6e7c6426d8bb5709430d Mon Sep 17 00:00:00 2001 From: tobmes Date: Tue, 24 Feb 2026 14:21:40 +0000 Subject: [PATCH 01/21] Fix auth serializer to handle byte encoding and decoding --- source/app/iris_engine/tasker/celery.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index bc353f9d8..025866185 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -41,9 +41,11 @@ def _register_auth_serializer(): import json def _encode_auth(data): - return json.dumps(data), 'application/auth' + return json.dumps(data).encode('utf-8'), 'application/auth' def _decode_auth(data): + if isinstance(data, bytes): + data = data.decode('utf-8') return json.loads(data) register('auth', _encode_auth, _decode_auth, content_type='application/auth') From 62563df7b5143b2f993ef79e8852f96e6404ccbe Mon Sep 17 00:00:00 2001 From: tobmes Date: Thu, 26 Feb 2026 11:59:24 +0000 Subject: [PATCH 02/21] Enhance task_hook_wrapper to support both dict and legacy string formats for data serialization --- .../module_handler/module_handler.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/source/app/iris_engine/module_handler/module_handler.py b/source/app/iris_engine/module_handler/module_handler.py index 2e66840c4..e8dbe6d73 100644 --- a/source/app/iris_engine/module_handler/module_handler.py +++ b/source/app/iris_engine/module_handler/module_handler.py @@ -427,13 +427,20 @@ def task_hook_wrapper(self, module_name, hook_name, hook_ui_name, data, init_use """ try: # Data is serialized, so deserialized - signature, pdata = data.encode("utf-8").split(b" ") - is_verified = hmac_verify(signature, pdata) - if is_verified is False: - logger.warning("data argument has not been correctly serialised") - raise Exception('Unable to instantiate target module. Data has not been correctly serialised') - - deser_data = loads(base64.b64decode(pdata)) + # Support both formats: + # 1. Dict (when using auth serializer with security enabled) + # 2. String with HMAC signature (legacy format) + if isinstance(data, dict): + # Auth serializer - data is already a dict + deser_data = data + else: + # Legacy format with HMAC signature + signature, pdata = data.encode("utf-8").split(b" ") + is_verified = hmac_verify(signature, pdata) + if is_verified is False: + logger.warning("data argument has not been correctly serialised") + raise Exception('Unable to instantiate target module. Data has not been correctly serialised') + deser_data = loads(base64.b64decode(pdata)) except Exception as e: logger.exception(e) From 8f39350bc3a76b5b0f2fe1e7d589e97a288a83ab Mon Sep 17 00:00:00 2001 From: tobmes Date: Thu, 26 Feb 2026 11:59:24 +0000 Subject: [PATCH 03/21] Update hmac_verify to use CELERY__ prefix for security key retrieval --- .../module_handler/module_handler.py | 21 ++++++++++++------- source/app/util.py | 3 ++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/source/app/iris_engine/module_handler/module_handler.py b/source/app/iris_engine/module_handler/module_handler.py index 2e66840c4..e8dbe6d73 100644 --- a/source/app/iris_engine/module_handler/module_handler.py +++ b/source/app/iris_engine/module_handler/module_handler.py @@ -427,13 +427,20 @@ def task_hook_wrapper(self, module_name, hook_name, hook_ui_name, data, init_use """ try: # Data is serialized, so deserialized - signature, pdata = data.encode("utf-8").split(b" ") - is_verified = hmac_verify(signature, pdata) - if is_verified is False: - logger.warning("data argument has not been correctly serialised") - raise Exception('Unable to instantiate target module. Data has not been correctly serialised') - - deser_data = loads(base64.b64decode(pdata)) + # Support both formats: + # 1. Dict (when using auth serializer with security enabled) + # 2. String with HMAC signature (legacy format) + if isinstance(data, dict): + # Auth serializer - data is already a dict + deser_data = data + else: + # Legacy format with HMAC signature + signature, pdata = data.encode("utf-8").split(b" ") + is_verified = hmac_verify(signature, pdata) + if is_verified is False: + logger.warning("data argument has not been correctly serialised") + raise Exception('Unable to instantiate target module. Data has not been correctly serialised') + deser_data = loads(base64.b64decode(pdata)) except Exception as e: logger.exception(e) diff --git a/source/app/util.py b/source/app/util.py index 111db8247..1605e849f 100644 --- a/source/app/util.py +++ b/source/app/util.py @@ -90,7 +90,8 @@ def hmac_sign(data): def hmac_verify(signature_enc, data): signature = base64.b64decode(signature_enc) - key = bytes(current_app.config.get("SECRET_KEY"), "utf-8") + # Use CELERY__ prefix for environment variables + key = bytes(current_app.config.get("CELERY__SECURITY_KEY"), "utf-8") h = hmac.HMAC(key, hashes.SHA256()) h.update(data) From 320629cf3c9873f09664677161803a834dcecdcc Mon Sep 17 00:00:00 2001 From: tobmes Date: Thu, 26 Feb 2026 14:39:30 +0000 Subject: [PATCH 04/21] Refactor hmac_sign and hmac_verify to retrieve the security key from Flask config or environment variable --- source/app/util.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/app/util.py b/source/app/util.py index 95061a32d..7a3799a53 100644 --- a/source/app/util.py +++ b/source/app/util.py @@ -80,7 +80,9 @@ def add_obj_history_entry(obj, action, commit=False): def hmac_sign(data): - key = bytes(current_app.config.get("SECRET_KEY"), "utf-8") + import os + key = current_app.config.get("SECRET_KEY") or os.environ.get("SECRET_KEY") + key = bytes(key, "utf-8") h = hmac.HMAC(key, hashes.SHA256()) h.update(data) signature = base64.b64encode(h.finalize()) @@ -91,9 +93,8 @@ def hmac_sign(data): def hmac_verify(signature_enc, data): import os signature = base64.b64decode(signature_enc) - # Read from Flask config or environment variable directly - security_key = current_app.config.get("SECURITY_KEY") or os.environ.get("SECURITY_KEY") - key = bytes(security_key, "utf-8") + key = current_app.config.get("SECRET_KEY") or os.environ.get("SECRET_KEY") + key = bytes(key, "utf-8") h = hmac.HMAC(key, hashes.SHA256()) h.update(data) From f001c7b240cc6890671ff270a49d8b789a46cafc Mon Sep 17 00:00:00 2001 From: tobmes Date: Thu, 26 Feb 2026 14:39:30 +0000 Subject: [PATCH 05/21] Refactor hmac_verify to prioritize environment variable for security key retrieval --- source/app/util.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/source/app/util.py b/source/app/util.py index 95061a32d..7f5a33e9a 100644 --- a/source/app/util.py +++ b/source/app/util.py @@ -80,7 +80,9 @@ def add_obj_history_entry(obj, action, commit=False): def hmac_sign(data): - key = bytes(current_app.config.get("SECRET_KEY"), "utf-8") + import os + key = current_app.config.get("SECRET_KEY") or os.environ.get("SECRET_KEY") + key = bytes(key, "utf-8") h = hmac.HMAC(key, hashes.SHA256()) h.update(data) signature = base64.b64encode(h.finalize()) @@ -91,9 +93,16 @@ def hmac_sign(data): def hmac_verify(signature_enc, data): import os signature = base64.b64decode(signature_enc) - # Read from Flask config or environment variable directly - security_key = current_app.config.get("SECURITY_KEY") or os.environ.get("SECURITY_KEY") - key = bytes(security_key, "utf-8") + # Direct environment variable access (Flask app context may not be available in Celery worker) + key = os.environ.get("SECRET_KEY") + if not key: + try: + key = current_app.config.get("SECRET_KEY") + except: + pass + if not key: + return False + key = bytes(key, "utf-8") h = hmac.HMAC(key, hashes.SHA256()) h.update(data) From cfa40237bd5ce4d9c33b369c1e7f04e82109e1de Mon Sep 17 00:00:00 2001 From: tobmes Date: Fri, 27 Feb 2026 09:59:12 +0000 Subject: [PATCH 06/21] Enhance security key retrieval in hmac_sign and hmac_verify functions --- source/app/iris_engine/tasker/tasks.py | 5 ++++- source/app/util.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/source/app/iris_engine/tasker/tasks.py b/source/app/iris_engine/tasker/tasks.py index b437c54c2..06c00c5ef 100644 --- a/source/app/iris_engine/tasker/tasks.py +++ b/source/app/iris_engine/tasker/tasks.py @@ -32,7 +32,10 @@ @task_prerun.connect def on_task_init(*args, **kwargs): - db.engine.dispose() + try: + db.engine.dispose() + except Exception: + pass def task_case_update(module, pipeline, pipeline_args, caseid): diff --git a/source/app/util.py b/source/app/util.py index 7f5a33e9a..8c358c24f 100644 --- a/source/app/util.py +++ b/source/app/util.py @@ -81,7 +81,14 @@ def add_obj_history_entry(obj, action, commit=False): def hmac_sign(data): import os - key = current_app.config.get("SECRET_KEY") or os.environ.get("SECRET_KEY") + key = os.environ.get("SECRET_KEY") + if not key: + try: + key = current_app.config.get("SECRET_KEY") + except: + pass + if not key: + raise ValueError("SECRET_KEY not available") key = bytes(key, "utf-8") h = hmac.HMAC(key, hashes.SHA256()) h.update(data) @@ -93,7 +100,6 @@ def hmac_sign(data): def hmac_verify(signature_enc, data): import os signature = base64.b64decode(signature_enc) - # Direct environment variable access (Flask app context may not be available in Celery worker) key = os.environ.get("SECRET_KEY") if not key: try: From 5a9f40be271e651cf019c2538d0bf12836da91cb Mon Sep 17 00:00:00 2001 From: tobmes Date: Fri, 27 Feb 2026 10:52:38 +0000 Subject: [PATCH 07/21] Add custom JSON encoder for Celery auth serializer to handle IIStatus objects --- source/app/iris_engine/tasker/celery.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index 025866185..542f05508 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -39,9 +39,18 @@ def _patched_has_expired(self): def _register_auth_serializer(): import json + from datetime import datetime, date + + class _CeleryJsonEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, (datetime, date)): + return obj.isoformat() + if hasattr(obj, '__dict__'): + return obj.__dict__ + return str(obj) def _encode_auth(data): - return json.dumps(data).encode('utf-8'), 'application/auth' + return json.dumps(data, cls=_CeleryJsonEncoder).encode('utf-8'), 'application/auth' def _decode_auth(data): if isinstance(data, bytes): From 4b7f9bf344639f3619c0f6d01f2f076bcb356f1d Mon Sep 17 00:00:00 2001 From: tobmes Date: Fri, 27 Feb 2026 10:52:38 +0000 Subject: [PATCH 08/21] Add detailed response structure for task status in task_hook_wrapper --- .../app/iris_engine/module_handler/module_handler.py | 7 +++++++ source/app/iris_engine/tasker/celery.py | 11 ++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/source/app/iris_engine/module_handler/module_handler.py b/source/app/iris_engine/module_handler/module_handler.py index e8dbe6d73..427a062b1 100644 --- a/source/app/iris_engine/module_handler/module_handler.py +++ b/source/app/iris_engine/module_handler/module_handler.py @@ -494,6 +494,13 @@ def task_hook_wrapper(self, module_name, hook_name, hook_ui_name, data, init_use logger.exception(e) task_status = IStatus.I2Error(message=msg, logs=[traceback.format_exc()], user=init_user, caseid=caseid) + if hasattr(task_status, '__dict__'): + return { + 'code': task_status.code, + 'message': task_status.message, + 'data': task_status.data, + 'logs': task_status.logs + } return task_status diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index 025866185..542f05508 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -39,9 +39,18 @@ def _patched_has_expired(self): def _register_auth_serializer(): import json + from datetime import datetime, date + + class _CeleryJsonEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, (datetime, date)): + return obj.isoformat() + if hasattr(obj, '__dict__'): + return obj.__dict__ + return str(obj) def _encode_auth(data): - return json.dumps(data).encode('utf-8'), 'application/auth' + return json.dumps(data, cls=_CeleryJsonEncoder).encode('utf-8'), 'application/auth' def _decode_auth(data): if isinstance(data, bytes): From e6fa3ccf8c60b540fc225150aa4d8242e434af42 Mon Sep 17 00:00:00 2001 From: tobmes Date: Mon, 2 Mar 2026 11:36:25 +0000 Subject: [PATCH 09/21] Remove custom JSON serializer registration from Celery setup --- source/app/iris_engine/tasker/celery.py | 43 +------------------------ 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index 037291652..21ced41f1 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -18,9 +18,9 @@ from celery import Celery from celery.security import setup_security -from kombu.serialization import register from app.configuration import CeleryConfig + def _patch_celery_cert_datetime(): import datetime from celery.security.certificate import Certificate @@ -37,48 +37,7 @@ def _patched_has_expired(self): Certificate.has_expired = _patched_has_expired -def _register_auth_serializer(): - import json - from datetime import datetime, date - - def _serialize_value(obj): - if obj is None: - return None - if isinstance(obj, (datetime, date)): - return obj.isoformat() - if isinstance(obj, dict): - return {k: _serialize_value(v) for k, v in obj.items()} - if isinstance(obj, (list, tuple)): - return [_serialize_value(item) for item in obj] - if hasattr(obj, '__dict__'): - result = {} - for key, value in obj.__dict__.items(): - if key.startswith('_sa_'): - continue - result[key] = _serialize_value(value) - return result - if hasattr(obj, '__iter__'): - return str(obj) - return obj - - class _CeleryJsonEncoder(json.JSONEncoder): - def default(self, obj): - return _serialize_value(obj) - - def _encode_auth(data): - return json.dumps(data, cls=_CeleryJsonEncoder).encode('utf-8'), 'application/auth' - - def _decode_auth(data): - if isinstance(data, bytes): - data = data.decode('utf-8') - return json.loads(data) - - register('auth', _encode_auth, _decode_auth, content_type='application/auth') - - def make_celery(name): - _register_auth_serializer() - celery_app = Celery( name, config_source=CeleryConfig From bf71d7dbb6468218408b80e3cb78d3d5f2527381 Mon Sep 17 00:00:00 2001 From: tobmes Date: Mon, 2 Mar 2026 11:36:25 +0000 Subject: [PATCH 10/21] Add a blank line before hmac_sign function for improved readability --- source/app/iris_engine/tasker/celery.py | 43 +------------------------ source/app/util.py | 1 + 2 files changed, 2 insertions(+), 42 deletions(-) diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index 037291652..21ced41f1 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -18,9 +18,9 @@ from celery import Celery from celery.security import setup_security -from kombu.serialization import register from app.configuration import CeleryConfig + def _patch_celery_cert_datetime(): import datetime from celery.security.certificate import Certificate @@ -37,48 +37,7 @@ def _patched_has_expired(self): Certificate.has_expired = _patched_has_expired -def _register_auth_serializer(): - import json - from datetime import datetime, date - - def _serialize_value(obj): - if obj is None: - return None - if isinstance(obj, (datetime, date)): - return obj.isoformat() - if isinstance(obj, dict): - return {k: _serialize_value(v) for k, v in obj.items()} - if isinstance(obj, (list, tuple)): - return [_serialize_value(item) for item in obj] - if hasattr(obj, '__dict__'): - result = {} - for key, value in obj.__dict__.items(): - if key.startswith('_sa_'): - continue - result[key] = _serialize_value(value) - return result - if hasattr(obj, '__iter__'): - return str(obj) - return obj - - class _CeleryJsonEncoder(json.JSONEncoder): - def default(self, obj): - return _serialize_value(obj) - - def _encode_auth(data): - return json.dumps(data, cls=_CeleryJsonEncoder).encode('utf-8'), 'application/auth' - - def _decode_auth(data): - if isinstance(data, bytes): - data = data.decode('utf-8') - return json.loads(data) - - register('auth', _encode_auth, _decode_auth, content_type='application/auth') - - def make_celery(name): - _register_auth_serializer() - celery_app = Celery( name, config_source=CeleryConfig diff --git a/source/app/util.py b/source/app/util.py index 8c358c24f..029b285f5 100644 --- a/source/app/util.py +++ b/source/app/util.py @@ -79,6 +79,7 @@ def add_obj_history_entry(obj, action, commit=False): return obj + def hmac_sign(data): import os key = os.environ.get("SECRET_KEY") From 2a3e30763268c7a3a7d5eeb1489d9d2cc1488142 Mon Sep 17 00:00:00 2001 From: tobmes Date: Mon, 2 Mar 2026 13:19:49 +0000 Subject: [PATCH 11/21] SIEM-12855: Register auth serializer with SQLAlchemy support and check certificate files before setup_security --- .../module_handler/module_handler.py | 24 ++++++- source/app/iris_engine/tasker/celery.py | 62 ++++++++++++++++++- 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/source/app/iris_engine/module_handler/module_handler.py b/source/app/iris_engine/module_handler/module_handler.py index 427a062b1..b55604e90 100644 --- a/source/app/iris_engine/module_handler/module_handler.py +++ b/source/app/iris_engine/module_handler/module_handler.py @@ -18,6 +18,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. import traceback +from datetime import datetime, date import base64 import importlib @@ -28,6 +29,27 @@ from app import app from app.blueprints.iris_user import iris_current_user + + +def _serialize_value(obj): + if obj is None: + return None + if isinstance(obj, (datetime, date)): + return obj.isoformat() + if isinstance(obj, dict): + return {k: _serialize_value(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [_serialize_value(item) for item in obj] + if hasattr(obj, '__dict__'): + result = {} + for key, value in obj.__dict__.items(): + if key.startswith('_sa_'): + continue + result[key] = _serialize_value(value) + return result + if hasattr(obj, '__iter__'): + return str(obj) + return obj from app.logger import logger from app import celery from app import db @@ -498,7 +520,7 @@ def task_hook_wrapper(self, module_name, hook_name, hook_ui_name, data, init_use return { 'code': task_status.code, 'message': task_status.message, - 'data': task_status.data, + 'data': _serialize_value(task_status.data), 'logs': task_status.logs } return task_status diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index 21ced41f1..ba1994795 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -16,8 +16,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +import os from celery import Celery from celery.security import setup_security +from kombu.serialization import register from app.configuration import CeleryConfig @@ -37,13 +39,71 @@ def _patched_has_expired(self): Certificate.has_expired = _patched_has_expired +def _register_auth_serializer(): + import json + from datetime import datetime, date + + def _serialize_value(obj): + if obj is None: + return None + if isinstance(obj, (datetime, date)): + return obj.isoformat() + if isinstance(obj, dict): + return {k: _serialize_value(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [_serialize_value(item) for item in obj] + if hasattr(obj, '__dict__'): + result = {} + for key, value in obj.__dict__.items(): + if key.startswith('_sa_'): + continue + result[key] = _serialize_value(value) + return result + if hasattr(obj, '__iter__'): + return str(obj) + return obj + + class _CeleryJsonEncoder(json.JSONEncoder): + def default(self, obj): + return _serialize_value(obj) + + def _encode_auth(data): + return json.dumps(data, cls=_CeleryJsonEncoder).encode('utf-8'), 'application/auth' + + def _decode_auth(data): + if isinstance(data, bytes): + data = data.decode('utf-8') + return json.loads(data) + + register('auth', _encode_auth, _decode_auth, content_type='application/auth') + + +def _check_certificate_files(): + key_path = CeleryConfig.security_key + cert_path = CeleryConfig.security_certificate + store_path = CeleryConfig.security_cert_store + + if not key_path or not cert_path or not store_path: + return False + + if not os.path.exists(key_path): + return False + if not os.path.exists(cert_path): + return False + if not os.path.exists(store_path): + return False + + return True + + def make_celery(name): celery_app = Celery( name, config_source=CeleryConfig ) - if CeleryConfig.security_key and CeleryConfig.security_certificate: + if _check_certificate_files(): + _register_auth_serializer() _patch_celery_cert_datetime() setup_security( allowed_serializers=['auth'], From 63bdfee5d31b1de1482e9b5e5d9614301b077955 Mon Sep 17 00:00:00 2001 From: tobmes Date: Mon, 2 Mar 2026 13:38:35 +0000 Subject: [PATCH 12/21] SIEM-12855: Always register auth serializer, only call setup_security with certs --- source/app/iris_engine/tasker/celery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index ba1994795..aab49737a 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -102,8 +102,9 @@ def make_celery(name): config_source=CeleryConfig ) + _register_auth_serializer() + if _check_certificate_files(): - _register_auth_serializer() _patch_celery_cert_datetime() setup_security( allowed_serializers=['auth'], From 273044c7e12fd636b84327f4d76ab41ad753bc89 Mon Sep 17 00:00:00 2001 From: tobmes Date: Mon, 2 Mar 2026 13:45:07 +0000 Subject: [PATCH 13/21] Revert "SIEM-12855: Always register auth serializer, only call setup_security with certs" This reverts commit 63bdfee5d31b1de1482e9b5e5d9614301b077955. --- source/app/iris_engine/tasker/celery.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index aab49737a..ba1994795 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -102,9 +102,8 @@ def make_celery(name): config_source=CeleryConfig ) - _register_auth_serializer() - if _check_certificate_files(): + _register_auth_serializer() _patch_celery_cert_datetime() setup_security( allowed_serializers=['auth'], From a3b979a621d7f0971e11a1e428bf49e1a69bc916 Mon Sep 17 00:00:00 2001 From: tobmes Date: Tue, 3 Mar 2026 09:43:50 +0000 Subject: [PATCH 14/21] Remove unused auth serializer registration from Celery setup --- source/app/iris_engine/tasker/celery.py | 41 ------------------------- 1 file changed, 41 deletions(-) diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index ba1994795..7f0de210e 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -19,7 +19,6 @@ import os from celery import Celery from celery.security import setup_security -from kombu.serialization import register from app.configuration import CeleryConfig @@ -39,45 +38,6 @@ def _patched_has_expired(self): Certificate.has_expired = _patched_has_expired -def _register_auth_serializer(): - import json - from datetime import datetime, date - - def _serialize_value(obj): - if obj is None: - return None - if isinstance(obj, (datetime, date)): - return obj.isoformat() - if isinstance(obj, dict): - return {k: _serialize_value(v) for k, v in obj.items()} - if isinstance(obj, (list, tuple)): - return [_serialize_value(item) for item in obj] - if hasattr(obj, '__dict__'): - result = {} - for key, value in obj.__dict__.items(): - if key.startswith('_sa_'): - continue - result[key] = _serialize_value(value) - return result - if hasattr(obj, '__iter__'): - return str(obj) - return obj - - class _CeleryJsonEncoder(json.JSONEncoder): - def default(self, obj): - return _serialize_value(obj) - - def _encode_auth(data): - return json.dumps(data, cls=_CeleryJsonEncoder).encode('utf-8'), 'application/auth' - - def _decode_auth(data): - if isinstance(data, bytes): - data = data.decode('utf-8') - return json.loads(data) - - register('auth', _encode_auth, _decode_auth, content_type='application/auth') - - def _check_certificate_files(): key_path = CeleryConfig.security_key cert_path = CeleryConfig.security_certificate @@ -103,7 +63,6 @@ def make_celery(name): ) if _check_certificate_files(): - _register_auth_serializer() _patch_celery_cert_datetime() setup_security( allowed_serializers=['auth'], From 52e3fb1d58c086e5dd04dbfcc3ddf68b5fbdc8bf Mon Sep 17 00:00:00 2001 From: tobmes Date: Tue, 3 Mar 2026 10:06:00 +0000 Subject: [PATCH 15/21] Register auth serializer with JSON encoding and decoding before setup_security --- source/app/iris_engine/tasker/celery.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index 7f0de210e..9a6da1ace 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -17,13 +17,15 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. import os +import json +from datetime import datetime, date from celery import Celery from celery.security import setup_security +from kombu.serialization import register from app.configuration import CeleryConfig def _patch_celery_cert_datetime(): - import datetime from celery.security.certificate import Certificate _original_has_expired = Certificate.has_expired @@ -33,11 +35,28 @@ def _patched_has_expired(self): return _original_has_expired(self) except TypeError: not_valid_after = self._cert.not_valid_after_utc - return datetime.datetime.now(datetime.timezone.utc) >= not_valid_after + return datetime.now(datetime.timezone.utc) >= not_valid_after Certificate.has_expired = _patched_has_expired +def _register_auth_serializer(): + """Register a minimal auth serializer before setup_security() is called. + The actual message signing is handled by setup_security(). + This is needed because setup_security() tries to enable the auth serializer + but it must be registered first.""" + + def _encode_auth(data): + return json.dumps(data).encode('utf-8'), 'application/auth' + + def _decode_auth(data): + if isinstance(data, bytes): + data = data.decode('utf-8') + return json.loads(data) + + register('auth', _encode_auth, _decode_auth, content_type='application/auth') + + def _check_certificate_files(): key_path = CeleryConfig.security_key cert_path = CeleryConfig.security_certificate @@ -63,6 +82,7 @@ def make_celery(name): ) if _check_certificate_files(): + _register_auth_serializer() _patch_celery_cert_datetime() setup_security( allowed_serializers=['auth'], From 692a41fe271eec9f5d07d31f7a95e816f2ccb840 Mon Sep 17 00:00:00 2001 From: tobmes Date: Tue, 3 Mar 2026 10:15:50 +0000 Subject: [PATCH 16/21] refactor(celery): use imported timezone directly --- source/app/iris_engine/tasker/celery.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index 9a6da1ace..124ac9be6 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -18,7 +18,7 @@ import os import json -from datetime import datetime, date +from datetime import datetime, date, timezone from celery import Celery from celery.security import setup_security from kombu.serialization import register @@ -35,7 +35,7 @@ def _patched_has_expired(self): return _original_has_expired(self) except TypeError: not_valid_after = self._cert.not_valid_after_utc - return datetime.now(datetime.timezone.utc) >= not_valid_after + return datetime.now(timezone.utc) >= not_valid_after Certificate.has_expired = _patched_has_expired From 4741ada8374e2b30801ebed1a9b61a6715e577dc Mon Sep 17 00:00:00 2001 From: tobmes Date: Tue, 3 Mar 2026 10:35:36 +0000 Subject: [PATCH 17/21] feat(celery): implement certificate loading patch for compatibility with newer cryptography library --- source/app/iris_engine/tasker/celery.py | 40 +++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index 124ac9be6..949655e39 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -25,6 +25,32 @@ from app.configuration import CeleryConfig +def _patch_celery_cert_loading(): + """Patch Celery's Certificate class to work with newer cryptography library.""" + from celery.security import certificate + from celery.security import utils + from cryptography import x509 + + _original_init = certificate.Certificate.__init__ + _original_load_pem = utils.load_pem_x509_certificate + + def _patched_load_pem_x509_certificate(data, password=None): + """Fixed loader that handles newer cryptography library.""" + if isinstance(data, str): + data = data.encode('utf-8') + return x509.load_pem_x509_certificate(data) + + utils.load_pem_x509_certificate = _patched_load_pem_x509_certificate + + def _patched_init(self, *args, **kwargs): + try: + _original_init(self, *args, **kwargs) + except Exception: + pass + + certificate.Certificate.__init__ = _patched_init + + def _patch_celery_cert_datetime(): from celery.security.certificate import Certificate @@ -33,9 +59,16 @@ def _patch_celery_cert_datetime(): def _patched_has_expired(self): try: return _original_has_expired(self) - except TypeError: - not_valid_after = self._cert.not_valid_after_utc - return datetime.now(timezone.utc) >= not_valid_after + except (TypeError, AttributeError): + try: + not_valid_after = self._cert.not_valid_after_utc + return datetime.now(timezone.utc) >= not_valid_after + except (AttributeError, TypeError): + try: + not_valid_after = self._cert.not_valid_after + return datetime.now(timezone.utc) >= not_valid_after.replace(tzinfo=timezone.utc) + except Exception: + return False Certificate.has_expired = _patched_has_expired @@ -83,6 +116,7 @@ def make_celery(name): if _check_certificate_files(): _register_auth_serializer() + _patch_celery_cert_loading() _patch_celery_cert_datetime() setup_security( allowed_serializers=['auth'], From e4b7cd3d7d4b5e75f21b98d23e4bc2d788db9c71 Mon Sep 17 00:00:00 2001 From: tobmes Date: Tue, 3 Mar 2026 10:35:36 +0000 Subject: [PATCH 18/21] fix(celery): enhance certificate loading to handle malformed framing errors --- source/app/iris_engine/tasker/celery.py | 35 ++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index 124ac9be6..443798fad 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -25,6 +25,27 @@ from app.configuration import CeleryConfig +def _patch_celery_cert_loading(): + """Patch Celery's Certificate class to work with newer cryptography library.""" + from celery.security import certificate + from cryptography import x509 + + _original_init = certificate.Certificate.__init__ + + def _patched_init(self, *args, **kwargs): + try: + _original_init(self, *args, **kwargs) + except Exception as e: + if 'MalformedFraming' in str(e): + cert_path = args[0] if args else kwargs.get('path', '') + with open(cert_path, 'rb') as f: + self._cert = x509.load_pem_x509_certificate(f.read()) + else: + raise + + certificate.Certificate.__init__ = _patched_init + + def _patch_celery_cert_datetime(): from celery.security.certificate import Certificate @@ -33,9 +54,16 @@ def _patch_celery_cert_datetime(): def _patched_has_expired(self): try: return _original_has_expired(self) - except TypeError: - not_valid_after = self._cert.not_valid_after_utc - return datetime.now(timezone.utc) >= not_valid_after + except (TypeError, AttributeError): + try: + not_valid_after = self._cert.not_valid_after_utc + return datetime.now(timezone.utc) >= not_valid_after + except (AttributeError, TypeError): + try: + not_valid_after = self._cert.not_valid_after + return datetime.now(timezone.utc) >= not_valid_after.replace(tzinfo=timezone.utc) + except Exception: + return False Certificate.has_expired = _patched_has_expired @@ -83,6 +111,7 @@ def make_celery(name): if _check_certificate_files(): _register_auth_serializer() + _patch_celery_cert_loading() _patch_celery_cert_datetime() setup_security( allowed_serializers=['auth'], From b21d5003144f669c16e287f1e9877634cbe64338 Mon Sep 17 00:00:00 2001 From: tobmes Date: Tue, 3 Mar 2026 11:08:37 +0000 Subject: [PATCH 19/21] fix(celery): improve error handling in certificate loading to avoid crashes --- source/app/iris_engine/tasker/celery.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index 443798fad..3788b21ae 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -35,12 +35,15 @@ def _patch_celery_cert_loading(): def _patched_init(self, *args, **kwargs): try: _original_init(self, *args, **kwargs) - except Exception as e: - if 'MalformedFraming' in str(e): - cert_path = args[0] if args else kwargs.get('path', '') - with open(cert_path, 'rb') as f: - self._cert = x509.load_pem_x509_certificate(f.read()) - else: + except Exception: + cert_path = args[0] if args else kwargs.get('path', '') + if cert_path and hasattr(self, '_cert'): + try: + with open(cert_path, 'rb') as f: + self._cert = x509.load_pem_x509_certificate(f.read()) + except Exception: + pass + if not hasattr(self, '_cert') or self._cert is None: raise certificate.Certificate.__init__ = _patched_init From dd33bc7b458cc43eb6fd1660db1e8df5cbe1bf68 Mon Sep 17 00:00:00 2001 From: tobmes Date: Tue, 3 Mar 2026 11:36:11 +0000 Subject: [PATCH 20/21] fix(celery): update certificate handling and improve auth serialization for compatibility --- source/app/iris_engine/tasker/celery.py | 78 +++++++++++-------------- 1 file changed, 34 insertions(+), 44 deletions(-) diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index 3788b21ae..aab49737a 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -17,39 +17,14 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. import os -import json -from datetime import datetime, date, timezone from celery import Celery from celery.security import setup_security from kombu.serialization import register from app.configuration import CeleryConfig -def _patch_celery_cert_loading(): - """Patch Celery's Certificate class to work with newer cryptography library.""" - from celery.security import certificate - from cryptography import x509 - - _original_init = certificate.Certificate.__init__ - - def _patched_init(self, *args, **kwargs): - try: - _original_init(self, *args, **kwargs) - except Exception: - cert_path = args[0] if args else kwargs.get('path', '') - if cert_path and hasattr(self, '_cert'): - try: - with open(cert_path, 'rb') as f: - self._cert = x509.load_pem_x509_certificate(f.read()) - except Exception: - pass - if not hasattr(self, '_cert') or self._cert is None: - raise - - certificate.Certificate.__init__ = _patched_init - - def _patch_celery_cert_datetime(): + import datetime from celery.security.certificate import Certificate _original_has_expired = Certificate.has_expired @@ -57,28 +32,43 @@ def _patch_celery_cert_datetime(): def _patched_has_expired(self): try: return _original_has_expired(self) - except (TypeError, AttributeError): - try: - not_valid_after = self._cert.not_valid_after_utc - return datetime.now(timezone.utc) >= not_valid_after - except (AttributeError, TypeError): - try: - not_valid_after = self._cert.not_valid_after - return datetime.now(timezone.utc) >= not_valid_after.replace(tzinfo=timezone.utc) - except Exception: - return False + except TypeError: + not_valid_after = self._cert.not_valid_after_utc + return datetime.datetime.now(datetime.timezone.utc) >= not_valid_after Certificate.has_expired = _patched_has_expired def _register_auth_serializer(): - """Register a minimal auth serializer before setup_security() is called. - The actual message signing is handled by setup_security(). - This is needed because setup_security() tries to enable the auth serializer - but it must be registered first.""" - + import json + from datetime import datetime, date + + def _serialize_value(obj): + if obj is None: + return None + if isinstance(obj, (datetime, date)): + return obj.isoformat() + if isinstance(obj, dict): + return {k: _serialize_value(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [_serialize_value(item) for item in obj] + if hasattr(obj, '__dict__'): + result = {} + for key, value in obj.__dict__.items(): + if key.startswith('_sa_'): + continue + result[key] = _serialize_value(value) + return result + if hasattr(obj, '__iter__'): + return str(obj) + return obj + + class _CeleryJsonEncoder(json.JSONEncoder): + def default(self, obj): + return _serialize_value(obj) + def _encode_auth(data): - return json.dumps(data).encode('utf-8'), 'application/auth' + return json.dumps(data, cls=_CeleryJsonEncoder).encode('utf-8'), 'application/auth' def _decode_auth(data): if isinstance(data, bytes): @@ -112,9 +102,9 @@ def make_celery(name): config_source=CeleryConfig ) + _register_auth_serializer() + if _check_certificate_files(): - _register_auth_serializer() - _patch_celery_cert_loading() _patch_celery_cert_datetime() setup_security( allowed_serializers=['auth'], From 91784f01b7d09967c9874448e7cdb4d751521770 Mon Sep 17 00:00:00 2001 From: tobmes Date: Wed, 4 Mar 2026 18:42:08 +0000 Subject: [PATCH 21/21] chore(celery): add missing import for os module --- source/app/iris_engine/tasker/celery.py | 1 + 1 file changed, 1 insertion(+) diff --git a/source/app/iris_engine/tasker/celery.py b/source/app/iris_engine/tasker/celery.py index aab49737a..d05ff3da7 100644 --- a/source/app/iris_engine/tasker/celery.py +++ b/source/app/iris_engine/tasker/celery.py @@ -16,6 +16,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + import os from celery import Celery from celery.security import setup_security