[kafka_actions] Add plugin architecture for formats and compression codecs

kafka_actions/changelog.d/23650.added

@@ -0,0 +1 @@
+Add a plugin architecture for message format handlers and payload-compression codecs. Format handlers can be registered via the `datadog_kafka_actions.formats` entry-point group, and compression codecs via `datadog_kafka_actions.compressions`. Built-in formats (json, string, raw, bson, avro, protobuf) are now first-class plugins. New `value_compression` and `key_compression` config keys decompress payloads before deserialization. No compression codecs ship in core — install a plugin wheel to add them.

kafka_actions/datadog_checks/kafka_actions/check.py

@@ -280,6 +280,8 @@ def _action_read_messages(self):
             'key_schema': config.get('key_schema'),
             'key_uses_schema_registry': config.get('key_uses_schema_registry', False),
             'key_skip_bytes': config.get('key_skip_bytes', 0),
+            'value_compression': config.get('value_compression'),
+            'key_compression': config.get('key_compression'),
         }
 
         self.log.debug(

kafka_actions/datadog_checks/kafka_actions/compression/__init__.py

@@ -0,0 +1,20 @@
+# (C) Datadog, Inc. 2026-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+"""Compression codec registry for kafka_actions.
+
+Some producers compress message payloads at the application layer (before
+handing bytes to the Kafka producer) using a variety of algorithms, separate
+from the broker-negotiated ``compression.type`` setting. This module exposes
+a pluggable codec interface so consumers can decompress those payloads
+before deserialization.
+
+No codecs ship in the core wheel — install a plugin wheel that registers
+codecs on the ``datadog_kafka_actions.compressions`` entry-point group, or
+register them directly via :func:`register_codec` in tests.
+"""
+
+from .base import CompressionCodec
+from .registry import get_codec, list_codecs, register_codec
+
+__all__ = ['CompressionCodec', 'get_codec', 'list_codecs', 'register_codec']

kafka_actions/datadog_checks/kafka_actions/compression/base.py

@@ -0,0 +1,19 @@
+# (C) Datadog, Inc. 2026-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+"""Base class for app-level payload compression codecs."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+
+class CompressionCodec(ABC):
+    """Plug-in interface for app-level payload decompression."""
+
+    name: str = ''
+
+    @abstractmethod
+    def decompress(self, data: bytes) -> bytes:
+        """Return the uncompressed payload bytes."""
+        raise NotImplementedError

kafka_actions/datadog_checks/kafka_actions/compression/registry.py

@@ -0,0 +1,64 @@
+# (C) Datadog, Inc. 2026-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+"""Lazy registry of compression codecs."""
+
+from __future__ import annotations
+
+import logging
+from importlib.metadata import entry_points
+from threading import Lock
+
+from .base import CompressionCodec
+
+_LOG = logging.getLogger(__name__)
+_ENTRY_POINT_GROUP = 'datadog_kafka_actions.compressions'
+
+_lock = Lock()
+_codecs: dict[str, CompressionCodec] = {}
+_loaded = False
+
+
+def register_codec(codec: CompressionCodec) -> None:
+    if not codec.name:
+        raise ValueError(f"CompressionCodec {type(codec).__name__} has no name set")
+    with _lock:
+        _codecs[codec.name] = codec
+
+
+def _load_entry_points() -> None:
+    global _loaded
+    if _loaded:
+        return
+    with _lock:
+        if _loaded:
+            return
+        try:
+            eps = entry_points(group=_ENTRY_POINT_GROUP)
+        except TypeError:  # pragma: no cover
+            eps = entry_points().get(_ENTRY_POINT_GROUP, [])
+        for ep in eps:
+            if ep.name in _codecs:
+                continue
+            try:
+                cls = ep.load()
+                instance = cls() if isinstance(cls, type) else cls
+                if not isinstance(instance, CompressionCodec):
+                    _LOG.warning("Entry point %s did not produce a CompressionCodec", ep.name)
+                    continue
+                if not instance.name:
+                    instance.name = ep.name
+                _codecs[instance.name] = instance
+            except Exception as e:
+                _LOG.warning("Failed to load compression codec '%s': %s", ep.name, e)
+        _loaded = True
+
+
+def get_codec(name: str) -> CompressionCodec | None:
+    _load_entry_points()
+    return _codecs.get(name)
+
+
+def list_codecs() -> list[str]:
+    _load_entry_points()
+    return sorted(_codecs)

kafka_actions/datadog_checks/kafka_actions/formats/__init__.py

@@ -0,0 +1,18 @@
+# (C) Datadog, Inc. 2026-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+"""Format handler registry for kafka_actions.
+
+External wheels can register additional handlers by exposing them on the
+``datadog_kafka_actions.formats`` entry-point group:
+
+    [project.entry-points."datadog_kafka_actions.formats"]
+    myformat = "my_pkg.handler:MyHandler"
+
+Handlers must subclass :class:`FormatHandler` from ``base``.
+"""
+
+from .base import FormatHandler
+from .registry import get_handler, list_handlers, register_handler
+
+__all__ = ['FormatHandler', 'get_handler', 'list_handlers', 'register_handler']

kafka_actions/datadog_checks/kafka_actions/formats/base.py

@@ -0,0 +1,42 @@
+# (C) Datadog, Inc. 2026-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+"""Base class for kafka_actions format handlers."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class FormatHandler(ABC):
+    """Plug-in interface for message-body deserialization.
+
+    Subclasses are instantiated once and reused across messages, so they
+    should be stateless or maintain only thread-safe caches.
+    """
+
+    name: str = ''
+
+    def build_schema(self, schema_str: str) -> Any:
+        """Build a schema object from an inline (config-supplied) schema string.
+
+        Override for formats that need a parsed schema (e.g. Avro, Protobuf).
+        Schemaless formats (json, msgpack, raw) can leave the default.
+        """
+        return None
+
+    def build_schema_from_registry(self, schema_str: str, dep_schemas: list) -> Any:
+        """Build a schema object from registry-supplied bytes.
+
+        ``dep_schemas`` is a list of ``(name, base64_bytes)`` tuples for
+        dependencies (e.g. imported .proto files).
+
+        Defaults to :meth:`build_schema` for formats that don't distinguish.
+        """
+        return self.build_schema(schema_str)
+
+    @abstractmethod
+    def deserialize(self, message: bytes, schema: Any, *, log, uses_schema_registry: bool) -> str | None:
+        """Decode ``message`` and return a JSON string (or None for empty messages)."""
+        raise NotImplementedError