diff --git a/src/confluent_kafka/schema_registry/_async/avro.py b/src/confluent_kafka/schema_registry/_async/avro.py index 991e8ec99..65ec01a1c 100644 --- a/src/confluent_kafka/schema_registry/_async/avro.py +++ b/src/confluent_kafka/schema_registry/_async/avro.py @@ -26,7 +26,6 @@ Schema, dual_schema_id_deserializer, prefix_schema_id_serializer, - topic_subject_name_strategy, ) from confluent_kafka.schema_registry.common import asyncinit from confluent_kafka.schema_registry.common.avro import ( @@ -125,14 +124,28 @@ class AsyncAvroSerializer(AsyncBaseSerializer): | | | | | | | Defaults to None. | +-----------------------------------+----------+--------------------------------------------------+ + | ``subject.name.strategy.type`` | str | The type of subject name strategy to use. | + | | | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +-----------------------------------+----------+--------------------------------------------------+ + | ``subject.name.strategy.conf`` | dict | Configuration dictionary passed to strategies | + | | | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-----------------------------------+----------+--------------------------------------------------+ | ``subject.name.strategy`` | callable | Callable(SerializationContext, str) -> str | | | | | | | | Defines how Schema Registry subject names are | | | | constructed. Standard naming strategies are | | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-----------------------------------+----------+--------------------------------------------------+ | ``schema.id.serializer`` | callable | Callable(bytes, SerializationContext, schema_id) | | | | -> bytes | @@ -224,7 +237,9 @@ class AsyncAvroSerializer(AsyncBaseSerializer): 'use.schema.id': None, 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.serializer': prefix_schema_id_serializer, 'validate.strict': False, 'validate.strict.allow.default': False, @@ -286,12 +301,11 @@ async def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_serializer = cast( Callable[[bytes, Optional[SerializationContext], Any], bytes], conf_copy.pop('schema.id.serializer') @@ -373,7 +387,11 @@ async def __serialize(self, obj: object, ctx: Optional[SerializationContext] = N if obj is None: return None - subject = self._subject_name_func(ctx, self._schema_name) + subject = ( + await self._subject_name_func(ctx, self._schema_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, self._schema_name) + ) latest_schema = await self._get_reader_schema(subject) if subject else None if latest_schema is not None: self._schema_id = SchemaId(AVRO_TYPE, latest_schema.schema_id, latest_schema.guid) @@ -468,34 +486,48 @@ class AsyncAvroDeserializer(AsyncBaseDeserializer): Deserializer for Avro binary encoded data with Confluent Schema Registry framing. - +-----------------------------+----------+--------------------------------------------------+ - | Property Name | Type | Description | - +-----------------------------+----------+--------------------------------------------------+ - | | | Whether to use the latest subject version for | - | ``use.latest.version`` | bool | deserialization. | - | | | | - | | | Defaults to False. | - +-----------------------------+----------+--------------------------------------------------+ - | | | Whether to use the latest subject version with | - | ``use.latest.with.metadata``| dict | the given metadata. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+--------------------------------------------------+ - | | | Callable(SerializationContext, str) -> str | - | | | | - | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | - | | | | - | | | Defaults to topic_subject_name_strategy. | - +-----------------------------+----------+--------------------------------------------------+ - | | | Callable(bytes, SerializationContext, schema_id) | - | | | -> io.BytesIO | - | | | | - | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | - | | | Defaults to dual_schema_id_deserializer. | - +-----------------------------+----------+--------------------------------------------------+ + +----------------------------------+----------+--------------------------------------------------+ + | Property Name | Type | Description | + +----------------------------------+----------+--------------------------------------------------+ + | | | Whether to use the latest subject version for | + | ``use.latest.version`` | bool | deserialization. | + | | | | + | | | Defaults to False. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Whether to use the latest subject version with | + | ``use.latest.with.metadata`` | dict | the given metadata. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+--------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Callable(SerializationContext, str) -> str | + | | | | + | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Callable(bytes, SerializationContext, schema_id) | + | | | -> io.BytesIO | + | | | | + | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | + | | | Defaults to dual_schema_id_deserializer. | + +----------------------------------+----------+--------------------------------------------------+ Note: By default, Avro complex types are returned as dicts. This behavior can @@ -533,7 +565,9 @@ class AsyncAvroDeserializer(AsyncBaseDeserializer): _default_conf = { 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.deserializer': dual_schema_id_deserializer, } @@ -575,12 +609,11 @@ async def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_deserializer = cast( Callable[[bytes, Optional[SerializationContext], Any], io.BytesIO], conf_copy.pop('schema.id.deserializer') @@ -649,7 +682,15 @@ async def __deserialize( "Schema Registry serializer".format(len(data)) ) - subject = self._subject_name_func(ctx, None) if ctx else None + subject = ( + ( + await self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, None) + ) + if ctx + else None + ) latest_schema = None if subject is not None: latest_schema = await self._get_reader_schema(subject) @@ -659,9 +700,17 @@ async def __deserialize( writer_schema_raw = await self._get_writer_schema(schema_id, subject) writer_schema = await self._get_parsed_schema(writer_schema_raw) - if subject is None: + if subject is None and isinstance(writer_schema, dict): subject = ( - self._subject_name_func(ctx, writer_schema.get("name")) if ctx else None # type: ignore[union-attr] + ( + await self._subject_name_func( + ctx, writer_schema.get("name"), self._registry, self._subject_name_conf + ) + if self._strategy_accepts_client + else self._subject_name_func(ctx, writer_schema.get("name")) + ) + if ctx + else None ) if subject is not None: latest_schema = await self._get_reader_schema(subject) diff --git a/src/confluent_kafka/schema_registry/_async/json_schema.py b/src/confluent_kafka/schema_registry/_async/json_schema.py index 76e5d5e97..048b8100b 100644 --- a/src/confluent_kafka/schema_registry/_async/json_schema.py +++ b/src/confluent_kafka/schema_registry/_async/json_schema.py @@ -32,7 +32,6 @@ Schema, dual_schema_id_deserializer, prefix_schema_id_serializer, - topic_subject_name_strategy, ) from confluent_kafka.schema_registry.common import asyncinit from confluent_kafka.schema_registry.common.json_schema import ( @@ -95,64 +94,78 @@ class AsyncJSONSerializer(AsyncBaseSerializer): Configuration properties: - +-----------------------------+----------+----------------------------------------------------+ - | Property Name | Type | Description | - +=============================+==========+====================================================+ - | | | If True, automatically register the configured | - | ``auto.register.schemas`` | bool | schema with Confluent Schema Registry if it has | - | | | not previously been associated with the relevant | - | | | subject (determined via subject.name.strategy). | - | | | | - | | | Defaults to True. | - | | | | - | | | Raises SchemaRegistryError if the schema was not | - | | | registered against the subject, or could not be | - | | | successfully registered. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to normalize schemas, which will | - | ``normalize.schemas`` | bool | transform schemas to have a consistent format, | - | | | including ordering properties and references. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the given schema ID for | - | ``use.schema.id`` | int | serialization. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version for | - | ``use.latest.version`` | bool | serialization. | - | | | | - | | | WARNING: There is no check that the latest | - | | | schema is backwards compatible with the object | - | | | being serialized. | - | | | | - | | | Defaults to False. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version with | - | ``use.latest.with.metadata``| dict | the given metadata. | - | | | | - | | | WARNING: There is no check that the latest | - | | | schema is backwards compatible with the object | - | | | being serialized. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(SerializationContext, str) -> str | - | | | | - | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | - | | | | - | | | Defaults to topic_subject_name_strategy. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to validate the payload against the | - | ``validate`` | bool | the given schema. | - | | | | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(bytes, SerializationContext, schema_id) | - | | | -> bytes | - | | | | - | ``schema.id.serializer`` | callable | Defines how the schema id/guid is serialized. | - | | | Defaults to prefix_schema_id_serializer. | - +-----------------------------+----------+----------------------------------------------------+ + +----------------------------------+----------+----------------------------------------------------+ + | Property Name | Type | Description | + +==================================+==========+====================================================+ + | | | If True, automatically register the configured | + | ``auto.register.schemas`` | bool | schema with Confluent Schema Registry if it has | + | | | not previously been associated with the relevant | + | | | subject (determined via subject.name.strategy). | + | | | | + | | | Defaults to True. | + | | | | + | | | Raises SchemaRegistryError if the schema was not | + | | | registered against the subject, or could not be | + | | | successfully registered. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to normalize schemas, which will | + | ``normalize.schemas`` | bool | transform schemas to have a consistent format, | + | | | including ordering properties and references. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the given schema ID for | + | ``use.schema.id`` | int | serialization. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version for | + | ``use.latest.version`` | bool | serialization. | + | | | | + | | | WARNING: There is no check that the latest | + | | | schema is backwards compatible with the object | + | | | being serialized. | + | | | | + | | | Defaults to False. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version with | + | ``use.latest.with.metadata`` | dict | the given metadata. | + | | | | + | | | WARNING: There is no check that the latest | + | | | schema is backwards compatible with the object | + | | | being serialized. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(SerializationContext, str) -> str | + | | | | + | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to validate the payload against the | + | ``validate`` | bool | the given schema. | + | | | | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(bytes, SerializationContext, schema_id) | + | | | -> bytes | + | | | | + | ``schema.id.serializer`` | callable | Defines how the schema id/guid is serialized. | + | | | Defaults to prefix_schema_id_serializer. | + +----------------------------------+----------+----------------------------------------------------+ Schemas are registered against subject names in Confluent Schema Registry that define a scope in which the schemas can be evolved. By default, the subject name @@ -226,7 +239,9 @@ class AsyncJSONSerializer(AsyncBaseSerializer): 'use.schema.id': None, 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.serializer': prefix_schema_id_serializer, 'validate': True, } @@ -292,12 +307,11 @@ async def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_serializer = cast( Callable[[bytes, Optional[SerializationContext], Any], bytes], conf_copy.pop('schema.id.serializer') @@ -354,7 +368,11 @@ async def __serialize(self, obj: object, ctx: Optional[SerializationContext] = N if obj is None: return None - subject = self._subject_name_func(ctx, self._schema_name) + subject = ( + await self._subject_name_func(ctx, self._schema_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, self._schema_name) + ) latest_schema = await self._get_reader_schema(subject) if subject else None if latest_schema is not None: self._schema_id = SchemaId(JSON_TYPE, latest_schema.schema_id, latest_schema.guid) @@ -467,39 +485,53 @@ class AsyncJSONDeserializer(AsyncBaseDeserializer): Configuration properties: - +-----------------------------+----------+----------------------------------------------------+ - | Property Name | Type | Description | - +=============================+==========+====================================================+ - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version for | - | ``use.latest.version`` | bool | deserialization. | - | | | | - | | | Defaults to False. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version with | - | ``use.latest.with.metadata``| dict | the given metadata. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(SerializationContext, str) -> str | - | | | | - | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | - | | | | - | | | Defaults to topic_subject_name_strategy. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to validate the payload against the | - | ``validate`` | bool | the given schema. | - | | | | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(bytes, SerializationContext, schema_id) | - | | | -> io.BytesIO | - | | | | - | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | - | | | Defaults to dual_schema_id_deserializer. | - +-----------------------------+----------+----------------------------------------------------+ + +----------------------------------+----------+----------------------------------------------------+ + | Property Name | Type | Description | + +==================================+==========+====================================================+ + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version for | + | ``use.latest.version`` | bool | deserialization. | + | | | | + | | | Defaults to False. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version with | + | ``use.latest.with.metadata`` | dict | the given metadata. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(SerializationContext, str) -> str | + | | | | + | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to validate the payload against the | + | ``validate`` | bool | the given schema. | + | | | | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(bytes, SerializationContext, schema_id) | + | | | -> io.BytesIO | + | | | | + | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | + | | | Defaults to dual_schema_id_deserializer. | + +----------------------------------+----------+----------------------------------------------------+ Args: schema_str (str, Schema, optional): @@ -531,7 +563,9 @@ class AsyncJSONDeserializer(AsyncBaseDeserializer): _default_conf = { 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.deserializer': dual_schema_id_deserializer, 'validate': True, } @@ -584,12 +618,11 @@ async def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_deserializer = cast( Callable[[bytes, Optional[SerializationContext], Any], io.BytesIO], conf_copy.pop('schema.id.deserializer') @@ -647,7 +680,11 @@ async def __deserialize(self, data: Optional[bytes], ctx: Optional[Serialization if data is None: return None - subject = self._subject_name_func(ctx, None) + subject = ( + await self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, None) + ) latest_schema = None if subject is not None and self._registry is not None: latest_schema = await self._get_reader_schema(subject) @@ -660,7 +697,13 @@ async def __deserialize(self, data: Optional[bytes], ctx: Optional[Serialization writer_schema_raw = await self._get_writer_schema(schema_id, subject) writer_schema, writer_ref_registry = await self._get_parsed_schema(writer_schema_raw) if subject is None and isinstance(writer_schema, dict): - subject = self._subject_name_func(ctx, writer_schema.get("title")) + subject = ( + await self._subject_name_func( + ctx, writer_schema.get("title"), self._registry, self._subject_name_conf + ) + if self._strategy_accepts_client + else self._subject_name_func(ctx, writer_schema.get("title")) + ) if subject is not None: latest_schema = await self._get_reader_schema(subject) else: diff --git a/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py b/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py index f782822be..b12a1fd01 100644 --- a/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py @@ -20,7 +20,15 @@ from threading import Lock from typing import Dict, List, Literal, Optional, Union -from ..common.schema_registry_client import RegisteredSchema, Schema, ServerConfig +from ..common.schema_registry_client import ( + Association, + AssociationCreateOrUpdateRequest, + AssociationInfo, + AssociationResponse, + RegisteredSchema, + Schema, + ServerConfig, +) from ..error import SchemaRegistryError from .schema_registry_client import AsyncSchemaRegistryClient @@ -142,11 +150,119 @@ def clear(self): self.subject_schemas.clear() +class _AssociationStore(object): + + def __init__(self): + self.lock = Lock() + # Key: resource_id -> List[Association] + self.associations_by_resource_id: Dict[str, List[Association]] = defaultdict(list) + # Key: (resource_namespace, resource_name) -> resource_id + self.resource_id_index: Dict[tuple, str] = {} + + def create_association(self, request: AssociationCreateOrUpdateRequest) -> AssociationResponse: + with self.lock: + resource_id = request.resource_id + resource_name = request.resource_name + resource_namespace = request.resource_namespace + resource_type = request.resource_type + + # Index resource_id by (namespace, name) + if resource_name and resource_namespace and resource_id: + self.resource_id_index[(resource_namespace, resource_name)] = resource_id + + created_associations = [] + if request.associations and resource_id is not None: + for assoc_info in request.associations: + association = Association( + subject=assoc_info.subject, + guid=None, + resource_name=resource_name, + resource_namespace=resource_namespace, + resource_id=resource_id, + resource_type=resource_type, + association_type=assoc_info.association_type, + frozen=assoc_info.frozen if assoc_info.frozen is not None else False, + ) + self.associations_by_resource_id[resource_id].append(association) + created_associations.append( + AssociationInfo( + subject=assoc_info.subject, + association_type=assoc_info.association_type, + lifecycle=assoc_info.lifecycle, + frozen=assoc_info.frozen if assoc_info.frozen is not None else False, + schema=assoc_info.schema, + ) + ) + + return AssociationResponse( + resource_name=resource_name, + resource_namespace=resource_namespace, + resource_id=resource_id, + resource_type=resource_type, + associations=created_associations, + ) + + def delete_associations( + self, + resource_id: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + ) -> None: + with self.lock: + if resource_id not in self.associations_by_resource_id: + return + + if association_types is None and resource_type is None: + # Delete all associations for this resource + del self.associations_by_resource_id[resource_id] + else: + # Filter and keep only non-matching associations + remaining = [] + for assoc in self.associations_by_resource_id[resource_id]: + keep = False + if resource_type is not None and assoc.resource_type != resource_type: + keep = True + if association_types is not None and assoc.association_type not in association_types: + keep = True + if keep: + remaining.append(assoc) + self.associations_by_resource_id[resource_id] = remaining + + def get_associations_by_resource_name( + self, + resource_name: str, + resource_namespace: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + ) -> List[Association]: + with self.lock: + result = [] + for resource_id, associations in self.associations_by_resource_id.items(): + for assoc in associations: + # Check if namespace matches (or is wildcard) + if resource_namespace != "-" and assoc.resource_namespace != resource_namespace: + continue + if assoc.resource_name != resource_name: + continue + if resource_type is not None and assoc.resource_type != resource_type: + continue + if association_types is not None and assoc.association_type not in association_types: + continue + result.append(assoc) + return result + + def clear(self): + with self.lock: + self.associations_by_resource_id.clear() + self.resource_id_index.clear() + + class AsyncMockSchemaRegistryClient(AsyncSchemaRegistryClient): def __init__(self, conf: dict): super().__init__(conf) self._store = _SchemaStore() + self._association_store = _AssociationStore() async def register_schema(self, subject_name: str, schema: 'Schema', normalize_schemas: bool = False) -> int: registered_schema = await self.register_schema_full_response( @@ -300,3 +416,46 @@ async def set_config( async def get_config(self, subject_name: Optional[str] = None) -> 'ServerConfig': # noqa F821 return None # type: ignore[return-value] + + async def get_associations_by_resource_name( + self, + resource_name: str, + resource_namespace: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + offset: int = 0, + limit: int = -1, + ) -> List['Association']: + return self._association_store.get_associations_by_resource_name( + resource_name, resource_namespace, resource_type, association_types + ) + + async def create_association(self, request: 'AssociationCreateOrUpdateRequest') -> 'AssociationResponse': + """ + Creates an association between a subject and a resource. + + Args: + request (AssociationCreateOrUpdateRequest): The association create or update request. + + Returns: + AssociationResponse: The response containing the created associations. + """ + return self._association_store.create_association(request) + + async def delete_associations( + self, + resource_id: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + cascade_lifecycle: bool = False, + ) -> None: + """ + Deletes associations for a resource. + + Args: + resource_id (str): The resource identifier. + resource_type (str, optional): The type of resource (e.g., "topic"). + association_types (List[str], optional): The types of associations to delete. + cascade_lifecycle (bool): Whether to cascade the lifecycle policy to dependent schemas. + """ + self._association_store.delete_associations(resource_id, resource_type, association_types) diff --git a/src/confluent_kafka/schema_registry/_async/protobuf.py b/src/confluent_kafka/schema_registry/_async/protobuf.py index 33c316e81..cca5e1ed2 100644 --- a/src/confluent_kafka/schema_registry/_async/protobuf.py +++ b/src/confluent_kafka/schema_registry/_async/protobuf.py @@ -31,7 +31,6 @@ dual_schema_id_deserializer, prefix_schema_id_serializer, reference_subject_name_strategy, - topic_subject_name_strategy, ) from confluent_kafka.schema_registry.common import asyncinit from confluent_kafka.schema_registry.common.protobuf import ( @@ -151,14 +150,28 @@ class AsyncProtobufSerializer(AsyncBaseSerializer): | | | | | | | Defaults to True. | +-------------------------------------+----------+------------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +-------------------------------------+----------+------------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | | | | constructed. Standard naming strategies are | | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | @@ -229,7 +242,9 @@ class AsyncProtobufSerializer(AsyncBaseSerializer): 'use.latest.version': False, 'use.latest.with.metadata': None, 'skip.known.types': True, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'reference.subject.name.strategy': reference_subject_name_strategy, 'schema.id.serializer': prefix_schema_id_serializer, 'use.deprecated.format': False, @@ -281,19 +296,18 @@ async def __init_impl( if self._use_deprecated_format: raise ValueError("use.deprecated.format is no longer supported") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._ref_reference_subject_func = cast( Callable[[Optional[SerializationContext], Any], Optional[str]], conf_copy.pop('reference.subject.name.strategy'), ) if not callable(self._ref_reference_subject_func): - raise ValueError("subject.name.strategy must be callable") + raise ValueError("reference.subject.name.strategy must be callable") self._schema_id_serializer = cast( Callable[[bytes, Optional[SerializationContext], Any], bytes], conf_copy.pop('schema.id.serializer') @@ -418,7 +432,17 @@ async def __serialize(self, message: Message, ctx: Optional[SerializationContext if not isinstance(message, self._msg_class): raise ValueError("message must be of type {} not {}".format(self._msg_class, type(message))) - subject = self._subject_name_func(ctx, message.DESCRIPTOR.full_name) if ctx else None + subject = ( + ( + await self._subject_name_func( + ctx, message.DESCRIPTOR.full_name, self._registry, self._subject_name_conf + ) + if self._strategy_accepts_client + else self._subject_name_func(ctx, message.DESCRIPTOR.full_name) + ) + if ctx + else None + ) latest_schema = None if subject is not None: latest_schema = await self._get_reader_schema(subject, fmt='serialized') @@ -513,14 +537,28 @@ class AsyncProtobufDeserializer(AsyncBaseDeserializer): | | | | | | | Defaults to None. | +-------------------------------------+----------+------------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +-------------------------------------+----------+------------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka. schema_registry | - | | | namespace . | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(bytes, SerializationContext, schema_id) | | | | -> io.BytesIO | @@ -539,7 +577,9 @@ class AsyncProtobufDeserializer(AsyncBaseDeserializer): _default_conf = { 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.deserializer': dual_schema_id_deserializer, 'use.deprecated.format': False, } @@ -571,12 +611,11 @@ async def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_deserializer = cast( Callable[[bytes, Optional[SerializationContext], Any], io.BytesIO], conf_copy.pop('schema.id.deserializer') @@ -626,7 +665,15 @@ async def __deserialize(self, data: Optional[bytes], ctx: Optional[Serialization if data is None: return None - subject = self._subject_name_func(ctx, None) + subject = ( + ( + await self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, None) + ) + if ctx + else None + ) latest_schema = None if subject is not None and self._registry is not None: latest_schema = await self._get_reader_schema(subject, fmt='serialized') @@ -641,7 +688,17 @@ async def __deserialize(self, data: Optional[bytes], ctx: Optional[Serialization writer_schema = pool.FindFileByName(fd_proto.name) writer_desc = self._get_message_desc(pool, writer_schema, msg_index if msg_index is not None else []) if subject is None: - subject = self._subject_name_func(ctx, writer_desc.full_name) + subject = ( + ( + await self._subject_name_func( + ctx, writer_desc.full_name, self._registry, self._subject_name_conf + ) + if self._strategy_accepts_client + else self._subject_name_func(ctx, writer_desc.full_name) + ) + if ctx + else None + ) if subject is not None: latest_schema = await self._get_reader_schema(subject, fmt='serialized') else: diff --git a/src/confluent_kafka/schema_registry/_async/schema_registry_client.py b/src/confluent_kafka/schema_registry/_async/schema_registry_client.py index 768e00940..a5da213c2 100644 --- a/src/confluent_kafka/schema_registry/_async/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_async/schema_registry_client.py @@ -42,6 +42,9 @@ _StaticOAuthBearerFieldProviderBuilder, ) from confluent_kafka.schema_registry.common.schema_registry_client import ( + Association, + AssociationCreateOrUpdateRequest, + AssociationResponse, RegisteredSchema, Schema, SchemaVersion, @@ -400,7 +403,7 @@ async def get(self, url: str, query: Optional[dict] = None) -> Any: async def post(self, url: str, body: Optional[dict], **kwargs) -> Any: raise NotImplementedError() - async def delete(self, url: str) -> Any: + async def delete(self, url: str, query: Optional[dict] = None) -> Any: raise NotImplementedError() async def put(self, url: str, body: Optional[dict] = None) -> Any: @@ -450,8 +453,8 @@ async def get(self, url: str, query: Optional[dict] = None) -> Any: async def post(self, url: str, body: Optional[dict], **kwargs) -> Any: return await self.send_request(url, method='POST', body=body) - async def delete(self, url: str) -> Any: - return await self.send_request(url, method='DELETE') + async def delete(self, url: str, query: Optional[dict] = None) -> Any: + return await self.send_request(url, method='DELETE', query=query) async def put(self, url: str, body: Optional[dict] = None) -> Any: return await self.send_request(url, method='PUT', body=body) @@ -508,6 +511,8 @@ async def send_request( response = await self.send_http_request(base_url, url, method, headers, body_str, query) if is_success(response.status_code): + if response.status_code == 204 or not response.content: + return None return response.json() if not is_retriable(response.status_code) or i == len(self.base_urls) - 1: @@ -1579,6 +1584,94 @@ async def clear_caches(self): self._latest_with_metadata_cache.clear() self._cache.clear() + async def get_associations_by_resource_name( + self, + resource_name: str, + resource_namespace: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + offset: int = 0, + limit: int = -1, + ) -> List['Association']: + """ + Retrieves associations for a given resource name and namespace. + + Args: + resource_name (str): The name of the resource (e.g., topic name). + resource_namespace (str): The namespace of the resource (e.g., kafka cluster ID). + Use "-" as a wildcard. + resource_type (str, optional): The type of resource (e.g., "topic"). + association_types (List[str], optional): The types of associations to filter by + (e.g., ["key", "value"]). + offset (int): Pagination offset for results. + limit (int): Pagination size for results. Ignored if negative. + + Returns: + List[Association]: List of associations matching the criteria. + + Raises: + SchemaRegistryError: if the request was unsuccessful. + """ + query: Dict[str, Any] = {} + if resource_type is not None: + query['resourceType'] = resource_type + if association_types is not None: + query['associationType'] = association_types + if offset > 0: + query['offset'] = offset + if limit >= 1: + query['limit'] = limit + + response = await self._rest_client.get( + 'associations/resources/{}/{}'.format(_urlencode(resource_namespace), _urlencode(resource_name)), query + ) + + return [Association.from_dict(a) for a in response] + + async def create_association(self, request: 'AssociationCreateOrUpdateRequest') -> 'AssociationResponse': + """ + Creates an association between a subject and a resource. + + Args: + request (AssociationCreateOrUpdateRequest): The association create or update request. + + Returns: + AssociationResponse: The response containing the created associations. + + Raises: + SchemaRegistryError: if the request was unsuccessful. + """ + response = await self._rest_client.post('associations', body=request.to_dict()) + return AssociationResponse.from_dict(response) + + async def delete_associations( + self, + resource_id: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + cascade_lifecycle: bool = False, + ) -> None: + """ + Deletes associations for a resource. + + Args: + resource_id (str): The resource identifier. + resource_type (str, optional): The type of resource (e.g., "topic"). + association_types (List[str], optional): The types of associations to delete + (e.g., ["key", "value"]). If not specified, all associations are deleted. + cascade_lifecycle (bool): Whether to cascade the lifecycle policy to dependent schemas. + + Raises: + SchemaRegistryError: if the request was unsuccessful. + """ + query: Dict[str, Any] = {'cascadeLifecycle': cascade_lifecycle} + if resource_type is not None: + query['resourceType'] = resource_type + if association_types is not None: + query['associationType'] = association_types + + await self._rest_client.delete('associations/resources/{}'.format(_urlencode(resource_id)), query=query) + @staticmethod def new_client(conf: dict) -> 'AsyncSchemaRegistryClient': from .mock_schema_registry_client import AsyncMockSchemaRegistryClient diff --git a/src/confluent_kafka/schema_registry/_async/serde.py b/src/confluent_kafka/schema_registry/_async/serde.py index 143855a7b..8e1b58983 100644 --- a/src/confluent_kafka/schema_registry/_async/serde.py +++ b/src/confluent_kafka/schema_registry/_async/serde.py @@ -16,12 +16,20 @@ # limitations under the License. # +import asyncio as _locks import logging -from typing import Any, Callable, Dict, List, Optional, Set +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union -from confluent_kafka.schema_registry import RegisteredSchema +from cachetools import LRUCache + +from confluent_kafka.schema_registry import ( + AsyncSchemaRegistryClient, + RegisteredSchema, + topic_subject_name_strategy, +) from confluent_kafka.schema_registry.common.schema_registry_client import RulePhase from confluent_kafka.schema_registry.common.serde import ( + STRATEGY_TYPE_MAP, ErrorAction, FieldTransformer, Migration, @@ -31,19 +39,209 @@ RuleContext, RuleError, SchemaId, + SubjectNameStrategyType, ) +from confluent_kafka.schema_registry.error import SchemaRegistryError from confluent_kafka.schema_registry.schema_registry_client import Rule, RuleKind, RuleMode, RuleSet, Schema -from confluent_kafka.serialization import Deserializer, SerializationContext, SerializationError, Serializer +from confluent_kafka.serialization import ( + Deserializer, + MessageField, + SerializationContext, + SerializationError, + Serializer, +) __all__ = [ + 'AsyncAssociatedNameStrategy', 'AsyncBaseSerde', 'AsyncBaseSerializer', 'AsyncBaseDeserializer', + 'KAFKA_CLUSTER_ID', + 'FALLBACK_TYPE', ] log = logging.getLogger(__name__) +KAFKA_CLUSTER_ID = "subject.name.strategy.kafka.cluster.id" +NAMESPACE_WILDCARD = "-" +FALLBACK_TYPE = "subject.name.strategy.fallback.type" +DEFAULT_CACHE_CAPACITY = 1000 + + +class AsyncAssociatedNameStrategy: + """ + A subject name strategy that retrieves the associated subject name from schema registry + by querying associations for the topic. + + This class encapsulates a cache for subject name lookups to avoid repeated API calls. + + Args: + cache_capacity (int): Maximum number of entries to cache. Defaults to 1000. + """ + + def __init__(self, cache_capacity: int = DEFAULT_CACHE_CAPACITY): + self._cache: LRUCache = LRUCache(maxsize=cache_capacity) + self._lock: _locks.Lock = _locks.Lock() + + def _get_cache_key(self, topic: str, is_key: bool, record_name: Optional[str]) -> Tuple[str, bool, Optional[str]]: + """Create a cache key from topic, is_key, and record_name.""" + return (topic, is_key, record_name) + + async def _load_subject_name( + self, + topic: str, + is_key: bool, + record_name: Optional[str], + ctx: SerializationContext, + schema_registry_client: AsyncSchemaRegistryClient, + conf: Optional[dict], + ) -> Optional[str]: + """Load the subject name from schema registry (not cached).""" + # Determine resource namespace from config + kafka_cluster_id = None + fallback_strategy = SubjectNameStrategyType.TOPIC # default fallback + + # If no client is available, skip association lookup and use fallback directly + if schema_registry_client is None: + return topic_subject_name_strategy(ctx, record_name) + + if conf is not None: + kafka_cluster_id = conf.get(KAFKA_CLUSTER_ID) + fallback_config = conf.get(FALLBACK_TYPE) + if fallback_config is not None: + if isinstance(fallback_config, SubjectNameStrategyType): + fallback_strategy = fallback_config + else: + try: + fallback_strategy = SubjectNameStrategyType(str(fallback_config).upper()) + except ValueError: + valid_fallbacks = [ + e.value for e in SubjectNameStrategyType if e != SubjectNameStrategyType.ASSOCIATED + ] + raise ValueError( + f"Invalid value for {FALLBACK_TYPE}: {fallback_config}. " + f"Valid values are: {', '.join(valid_fallbacks)}" + ) + + resource_namespace = kafka_cluster_id if kafka_cluster_id is not None else NAMESPACE_WILDCARD + + # Determine association type based on whether this is key or value + association_type = "key" if is_key else "value" + + # Query schema registry for associations + try: + associations = await schema_registry_client.get_associations_by_resource_name( + resource_name=topic, + resource_namespace=resource_namespace, + resource_type="topic", + association_types=[association_type], + offset=0, + limit=-1, + ) + except SchemaRegistryError as e: + if e.http_status_code == 404: + # Treat 404 as no associations found and fall through to existing fallback logic + associations = [] + else: + raise + + if len(associations) > 1: + raise SerializationError(f"Multiple associated subjects found for topic {topic}") + elif len(associations) == 1: + return associations[0].subject + else: + # No associations found, use fallback strategy + if fallback_strategy == SubjectNameStrategyType.NONE: + raise SerializationError(f"No associated subject found for topic {topic}") + elif fallback_strategy == SubjectNameStrategyType.ASSOCIATED: + raise ValueError( + f"Invalid value for {FALLBACK_TYPE}: {fallback_strategy.value}. " + f"ASSOCIATED cannot be used as a fallback strategy." + ) + + return STRATEGY_TYPE_MAP[fallback_strategy](ctx, record_name) + + async def __call__( + self, + ctx: Optional[SerializationContext], + record_name: Optional[str], + schema_registry_client: AsyncSchemaRegistryClient, + conf: Optional[dict] = None, + ) -> Optional[str]: + """ + Retrieves the associated subject name from schema registry by querying + associations for the topic. + + The topic is passed as the resource name to schema registry. If there is a + configuration property named "kafka.cluster.id", then its value will be passed + as the resource namespace; otherwise the value "-" will be passed as the + resource namespace. + + If more than one subject is returned from the query, a SerializationError + will be raised. If no subjects are returned from the query, then the behavior + will fall back to topic_subject_name_strategy, unless the configuration property + "subject.name.strategy.fallback.type" is set to "RECORD", "TOPIC_RECORD", or "NONE". + + Results are cached using an LRU cache to avoid repeated API calls. + + Args: + ctx (SerializationContext): Metadata pertaining to the serialization + operation. **Required** - must contain topic and field information. + + record_name (Optional[str]): Record name (used for fallback strategies). + + schema_registry_client (AsyncSchemaRegistryClient): AsyncSchemaRegistryClient instance. + + conf (Optional[dict]): Configuration dictionary. Supports: + - "subject.name.strategy.kafka.cluster.id": Kafka cluster ID to use as resource namespace. + - "subject.name.strategy.fallback.type": Fallback strategy when no + associations are found. One of "TOPIC", "RECORD", "TOPIC_RECORD", or "NONE". + Defaults to "TOPIC". + + Returns: + Optional[str]: The subject name from the association, or from the fallback strategy. + + Raises: + SerializationError: If multiple associated subjects are found for the topic, + or if no subjects are found and fallback is set to "NONE". + ValueError: If ctx is None. + """ + if ctx is None: + raise ValueError( + "SerializationContext is required for AsyncAssociatedNameStrategy. " + "Either provide a SerializationContext or use a different strategy." + ) + + topic = ctx.topic + if topic is None: + return None + + is_key = ctx.field == MessageField.KEY + cache_key = self._get_cache_key(topic, is_key, record_name) + + # Check cache first + async with self._lock: + cached_result = self._cache.get(cache_key) + if cached_result is not None: + return cached_result + + # Not in cache, load from schema registry + result = await self._load_subject_name(topic, is_key, record_name, ctx, schema_registry_client, conf) + + # Cache the result + if result is not None: + async with self._lock: + self._cache[cache_key] = result + + return result + + async def clear_cache(self) -> None: + """Clear the association subject name cache.""" + async with self._lock: + self._cache.clear() + + class AsyncBaseSerde(object): __slots__ = [ '_use_schema_id', @@ -51,6 +249,8 @@ class AsyncBaseSerde(object): '_use_latest_with_metadata', '_registry', '_rule_registry', + '_strategy_accepts_client', + '_subject_name_conf', '_subject_name_func', '_field_transformer', ] @@ -60,9 +260,81 @@ class AsyncBaseSerde(object): _use_latest_with_metadata: Optional[Dict[str, str]] _registry: Any # AsyncSchemaRegistryClient _rule_registry: Any # RuleRegistry - _subject_name_func: Callable[[Optional['SerializationContext'], Optional[str]], Optional[str]] + _strategy_accepts_client: bool + _subject_name_conf: Optional[dict] + _subject_name_func: Callable[..., Any] _field_transformer: Optional[FieldTransformer] + def configure_subject_name_strategy( + self, + subject_name_strategy_type: Optional[Union[SubjectNameStrategyType, str]] = None, + subject_name_strategy_conf: Optional[dict] = None, + subject_name_strategy: Optional[Callable] = None, + ) -> None: + """ + Configure the subject name strategy for this serde. + + This method supports both the legacy callable approach and the new type-based approach. + If both `subject_name_strategy` (as a callable) and `subject_name_strategy_type` are + provided, the callable takes precedence. + + Args: + subject_name_strategy: A callable that implements the subject name strategy. + Signature: (SerializationContext, str) -> str or + (SerializationContext, str, AsyncSchemaRegistryClient, dict) -> str + + subject_name_strategy_type: The type of subject name strategy to use. + Can be a SubjectNameStrategyType enum value or a string + ("TOPIC", "RECORD", "TOPIC_RECORD", "ASSOCIATED"). + + subject_name_strategy_conf: Configuration dictionary passed to strategies + that accept extra parameters (like ASSOCIATED). + + Raises: + ValueError: If the strategy is not callable or the type is invalid. + """ + self._subject_name_conf = subject_name_strategy_conf + + # If a callable is provided, use it directly (backward compatible) + if subject_name_strategy is not None: + if not callable(subject_name_strategy): + raise ValueError("subject.name.strategy must be callable") + self._subject_name_func = subject_name_strategy + self._strategy_accepts_client = isinstance(subject_name_strategy, AsyncAssociatedNameStrategy) + return + + # If a type is provided, resolve it to a callable + if subject_name_strategy_type is not None: + # Convert string to enum if needed + if isinstance(subject_name_strategy_type, str): + try: + subject_name_strategy_type = SubjectNameStrategyType(subject_name_strategy_type.upper()) + except ValueError: + raise ValueError( + f"Invalid subject.name.strategy.type: {subject_name_strategy_type}. " + f"Valid values are: {[e.value for e in SubjectNameStrategyType]}" + ) + + # Handle ASSOCIATED specially since it needs schema_registry_client + if subject_name_strategy_type == SubjectNameStrategyType.ASSOCIATED: + self._subject_name_func = AsyncAssociatedNameStrategy() + self._strategy_accepts_client = True + elif subject_name_strategy_type == SubjectNameStrategyType.NONE: + raise ValueError( + f"Invalid subject.name.strategy.type: {subject_name_strategy_type}. " + f"NONE cannot be used as a subject name strategy." + ) + elif subject_name_strategy_type in STRATEGY_TYPE_MAP: + self._subject_name_func = STRATEGY_TYPE_MAP[subject_name_strategy_type] + self._strategy_accepts_client = False + else: + raise ValueError(f"Unknown subject.name.strategy.type: {subject_name_strategy_type}") + return + + # Default to AsyncAssociatedNameStrategy (falls back to TOPIC when no associations found) + self._subject_name_func = AsyncAssociatedNameStrategy() + self._strategy_accepts_client = True + async def _get_reader_schema(self, subject: str, fmt: Optional[str] = None) -> Optional[RegisteredSchema]: if self._use_schema_id is not None: schema = await self._registry.get_schema(self._use_schema_id, subject, fmt) diff --git a/src/confluent_kafka/schema_registry/_sync/avro.py b/src/confluent_kafka/schema_registry/_sync/avro.py index 289367265..d17aac5a5 100644 --- a/src/confluent_kafka/schema_registry/_sync/avro.py +++ b/src/confluent_kafka/schema_registry/_sync/avro.py @@ -26,7 +26,6 @@ SchemaRegistryClient, dual_schema_id_deserializer, prefix_schema_id_serializer, - topic_subject_name_strategy, ) from confluent_kafka.schema_registry.common.avro import ( AVRO_TYPE, @@ -121,14 +120,28 @@ class AvroSerializer(BaseSerializer): | | | | | | | Defaults to None. | +-----------------------------------+----------+--------------------------------------------------+ + | ``subject.name.strategy.type`` | str | The type of subject name strategy to use. | + | | | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +-----------------------------------+----------+--------------------------------------------------+ + | ``subject.name.strategy.conf`` | dict | Configuration dictionary passed to strategies | + | | | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-----------------------------------+----------+--------------------------------------------------+ | ``subject.name.strategy`` | callable | Callable(SerializationContext, str) -> str | | | | | | | | Defines how Schema Registry subject names are | | | | constructed. Standard naming strategies are | | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-----------------------------------+----------+--------------------------------------------------+ | ``schema.id.serializer`` | callable | Callable(bytes, SerializationContext, schema_id) | | | | -> bytes | @@ -220,7 +233,9 @@ class AvroSerializer(BaseSerializer): 'use.schema.id': None, 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.serializer': prefix_schema_id_serializer, 'validate.strict': False, 'validate.strict.allow.default': False, @@ -282,12 +297,11 @@ def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_serializer = cast( Callable[[bytes, Optional[SerializationContext], Any], bytes], conf_copy.pop('schema.id.serializer') @@ -369,7 +383,11 @@ def __serialize(self, obj: object, ctx: Optional[SerializationContext] = None) - if obj is None: return None - subject = self._subject_name_func(ctx, self._schema_name) + subject = ( + self._subject_name_func(ctx, self._schema_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, self._schema_name) + ) latest_schema = self._get_reader_schema(subject) if subject else None if latest_schema is not None: self._schema_id = SchemaId(AVRO_TYPE, latest_schema.schema_id, latest_schema.guid) @@ -463,34 +481,48 @@ class AvroDeserializer(BaseDeserializer): Deserializer for Avro binary encoded data with Confluent Schema Registry framing. - +-----------------------------+----------+--------------------------------------------------+ - | Property Name | Type | Description | - +-----------------------------+----------+--------------------------------------------------+ - | | | Whether to use the latest subject version for | - | ``use.latest.version`` | bool | deserialization. | - | | | | - | | | Defaults to False. | - +-----------------------------+----------+--------------------------------------------------+ - | | | Whether to use the latest subject version with | - | ``use.latest.with.metadata``| dict | the given metadata. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+--------------------------------------------------+ - | | | Callable(SerializationContext, str) -> str | - | | | | - | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | - | | | | - | | | Defaults to topic_subject_name_strategy. | - +-----------------------------+----------+--------------------------------------------------+ - | | | Callable(bytes, SerializationContext, schema_id) | - | | | -> io.BytesIO | - | | | | - | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | - | | | Defaults to dual_schema_id_deserializer. | - +-----------------------------+----------+--------------------------------------------------+ + +----------------------------------+----------+--------------------------------------------------+ + | Property Name | Type | Description | + +----------------------------------+----------+--------------------------------------------------+ + | | | Whether to use the latest subject version for | + | ``use.latest.version`` | bool | deserialization. | + | | | | + | | | Defaults to False. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Whether to use the latest subject version with | + | ``use.latest.with.metadata`` | dict | the given metadata. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+--------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Callable(SerializationContext, str) -> str | + | | | | + | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Callable(bytes, SerializationContext, schema_id) | + | | | -> io.BytesIO | + | | | | + | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | + | | | Defaults to dual_schema_id_deserializer. | + +----------------------------------+----------+--------------------------------------------------+ Note: By default, Avro complex types are returned as dicts. This behavior can @@ -528,7 +560,9 @@ class AvroDeserializer(BaseDeserializer): _default_conf = { 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.deserializer': dual_schema_id_deserializer, } @@ -570,12 +604,11 @@ def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_deserializer = cast( Callable[[bytes, Optional[SerializationContext], Any], io.BytesIO], conf_copy.pop('schema.id.deserializer') @@ -642,7 +675,15 @@ def __deserialize( "Schema Registry serializer".format(len(data)) ) - subject = self._subject_name_func(ctx, None) if ctx else None + subject = ( + ( + self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, None) + ) + if ctx + else None + ) latest_schema = None if subject is not None: latest_schema = self._get_reader_schema(subject) @@ -652,9 +693,15 @@ def __deserialize( writer_schema_raw = self._get_writer_schema(schema_id, subject) writer_schema = self._get_parsed_schema(writer_schema_raw) - if subject is None: + if subject is None and isinstance(writer_schema, dict): subject = ( - self._subject_name_func(ctx, writer_schema.get("name")) if ctx else None # type: ignore[union-attr] + ( + self._subject_name_func(ctx, writer_schema.get("name"), self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, writer_schema.get("name")) + ) + if ctx + else None ) if subject is not None: latest_schema = self._get_reader_schema(subject) diff --git a/src/confluent_kafka/schema_registry/_sync/json_schema.py b/src/confluent_kafka/schema_registry/_sync/json_schema.py index bc6fd4d4a..ff33aa4df 100644 --- a/src/confluent_kafka/schema_registry/_sync/json_schema.py +++ b/src/confluent_kafka/schema_registry/_sync/json_schema.py @@ -32,7 +32,6 @@ SchemaRegistryClient, dual_schema_id_deserializer, prefix_schema_id_serializer, - topic_subject_name_strategy, ) from confluent_kafka.schema_registry.common.json_schema import ( DEFAULT_SPEC, @@ -93,64 +92,78 @@ class JSONSerializer(BaseSerializer): Configuration properties: - +-----------------------------+----------+----------------------------------------------------+ - | Property Name | Type | Description | - +=============================+==========+====================================================+ - | | | If True, automatically register the configured | - | ``auto.register.schemas`` | bool | schema with Confluent Schema Registry if it has | - | | | not previously been associated with the relevant | - | | | subject (determined via subject.name.strategy). | - | | | | - | | | Defaults to True. | - | | | | - | | | Raises SchemaRegistryError if the schema was not | - | | | registered against the subject, or could not be | - | | | successfully registered. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to normalize schemas, which will | - | ``normalize.schemas`` | bool | transform schemas to have a consistent format, | - | | | including ordering properties and references. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the given schema ID for | - | ``use.schema.id`` | int | serialization. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version for | - | ``use.latest.version`` | bool | serialization. | - | | | | - | | | WARNING: There is no check that the latest | - | | | schema is backwards compatible with the object | - | | | being serialized. | - | | | | - | | | Defaults to False. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version with | - | ``use.latest.with.metadata``| dict | the given metadata. | - | | | | - | | | WARNING: There is no check that the latest | - | | | schema is backwards compatible with the object | - | | | being serialized. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(SerializationContext, str) -> str | - | | | | - | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | - | | | | - | | | Defaults to topic_subject_name_strategy. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to validate the payload against the | - | ``validate`` | bool | the given schema. | - | | | | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(bytes, SerializationContext, schema_id) | - | | | -> bytes | - | | | | - | ``schema.id.serializer`` | callable | Defines how the schema id/guid is serialized. | - | | | Defaults to prefix_schema_id_serializer. | - +-----------------------------+----------+----------------------------------------------------+ + +----------------------------------+----------+----------------------------------------------------+ + | Property Name | Type | Description | + +==================================+==========+====================================================+ + | | | If True, automatically register the configured | + | ``auto.register.schemas`` | bool | schema with Confluent Schema Registry if it has | + | | | not previously been associated with the relevant | + | | | subject (determined via subject.name.strategy). | + | | | | + | | | Defaults to True. | + | | | | + | | | Raises SchemaRegistryError if the schema was not | + | | | registered against the subject, or could not be | + | | | successfully registered. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to normalize schemas, which will | + | ``normalize.schemas`` | bool | transform schemas to have a consistent format, | + | | | including ordering properties and references. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the given schema ID for | + | ``use.schema.id`` | int | serialization. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version for | + | ``use.latest.version`` | bool | serialization. | + | | | | + | | | WARNING: There is no check that the latest | + | | | schema is backwards compatible with the object | + | | | being serialized. | + | | | | + | | | Defaults to False. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version with | + | ``use.latest.with.metadata`` | dict | the given metadata. | + | | | | + | | | WARNING: There is no check that the latest | + | | | schema is backwards compatible with the object | + | | | being serialized. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(SerializationContext, str) -> str | + | | | | + | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to validate the payload against the | + | ``validate`` | bool | the given schema. | + | | | | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(bytes, SerializationContext, schema_id) | + | | | -> bytes | + | | | | + | ``schema.id.serializer`` | callable | Defines how the schema id/guid is serialized. | + | | | Defaults to prefix_schema_id_serializer. | + +----------------------------------+----------+----------------------------------------------------+ Schemas are registered against subject names in Confluent Schema Registry that define a scope in which the schemas can be evolved. By default, the subject name @@ -224,7 +237,9 @@ class JSONSerializer(BaseSerializer): 'use.schema.id': None, 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.serializer': prefix_schema_id_serializer, 'validate': True, } @@ -290,12 +305,11 @@ def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_serializer = cast( Callable[[bytes, Optional[SerializationContext], Any], bytes], conf_copy.pop('schema.id.serializer') @@ -352,7 +366,11 @@ def __serialize(self, obj: object, ctx: Optional[SerializationContext] = None) - if obj is None: return None - subject = self._subject_name_func(ctx, self._schema_name) + subject = ( + self._subject_name_func(ctx, self._schema_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, self._schema_name) + ) latest_schema = self._get_reader_schema(subject) if subject else None if latest_schema is not None: self._schema_id = SchemaId(JSON_TYPE, latest_schema.schema_id, latest_schema.guid) @@ -464,39 +482,53 @@ class JSONDeserializer(BaseDeserializer): Configuration properties: - +-----------------------------+----------+----------------------------------------------------+ - | Property Name | Type | Description | - +=============================+==========+====================================================+ - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version for | - | ``use.latest.version`` | bool | deserialization. | - | | | | - | | | Defaults to False. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version with | - | ``use.latest.with.metadata``| dict | the given metadata. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(SerializationContext, str) -> str | - | | | | - | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | - | | | | - | | | Defaults to topic_subject_name_strategy. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to validate the payload against the | - | ``validate`` | bool | the given schema. | - | | | | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(bytes, SerializationContext, schema_id) | - | | | -> io.BytesIO | - | | | | - | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | - | | | Defaults to dual_schema_id_deserializer. | - +-----------------------------+----------+----------------------------------------------------+ + +----------------------------------+----------+----------------------------------------------------+ + | Property Name | Type | Description | + +==================================+==========+====================================================+ + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version for | + | ``use.latest.version`` | bool | deserialization. | + | | | | + | | | Defaults to False. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version with | + | ``use.latest.with.metadata`` | dict | the given metadata. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(SerializationContext, str) -> str | + | | | | + | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to validate the payload against the | + | ``validate`` | bool | the given schema. | + | | | | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(bytes, SerializationContext, schema_id) | + | | | -> io.BytesIO | + | | | | + | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | + | | | Defaults to dual_schema_id_deserializer. | + +----------------------------------+----------+----------------------------------------------------+ Args: schema_str (str, Schema, optional): @@ -528,7 +560,9 @@ class JSONDeserializer(BaseDeserializer): _default_conf = { 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.deserializer': dual_schema_id_deserializer, 'validate': True, } @@ -581,12 +615,11 @@ def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_deserializer = cast( Callable[[bytes, Optional[SerializationContext], Any], io.BytesIO], conf_copy.pop('schema.id.deserializer') @@ -642,7 +675,11 @@ def __deserialize(self, data: Optional[bytes], ctx: Optional[SerializationContex if data is None: return None - subject = self._subject_name_func(ctx, None) + subject = ( + self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, None) + ) latest_schema = None if subject is not None and self._registry is not None: latest_schema = self._get_reader_schema(subject) @@ -655,7 +692,11 @@ def __deserialize(self, data: Optional[bytes], ctx: Optional[SerializationContex writer_schema_raw = self._get_writer_schema(schema_id, subject) writer_schema, writer_ref_registry = self._get_parsed_schema(writer_schema_raw) if subject is None and isinstance(writer_schema, dict): - subject = self._subject_name_func(ctx, writer_schema.get("title")) + subject = ( + self._subject_name_func(ctx, writer_schema.get("title"), self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, writer_schema.get("title")) + ) if subject is not None: latest_schema = self._get_reader_schema(subject) else: diff --git a/src/confluent_kafka/schema_registry/_sync/mock_schema_registry_client.py b/src/confluent_kafka/schema_registry/_sync/mock_schema_registry_client.py index 539070bae..ae44934b9 100644 --- a/src/confluent_kafka/schema_registry/_sync/mock_schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_sync/mock_schema_registry_client.py @@ -20,7 +20,15 @@ from threading import Lock from typing import Dict, List, Literal, Optional, Union -from ..common.schema_registry_client import RegisteredSchema, Schema, ServerConfig +from ..common.schema_registry_client import ( + Association, + AssociationCreateOrUpdateRequest, + AssociationInfo, + AssociationResponse, + RegisteredSchema, + Schema, + ServerConfig, +) from ..error import SchemaRegistryError from .schema_registry_client import SchemaRegistryClient @@ -142,11 +150,119 @@ def clear(self): self.subject_schemas.clear() +class _AssociationStore(object): + + def __init__(self): + self.lock = Lock() + # Key: resource_id -> List[Association] + self.associations_by_resource_id: Dict[str, List[Association]] = defaultdict(list) + # Key: (resource_namespace, resource_name) -> resource_id + self.resource_id_index: Dict[tuple, str] = {} + + def create_association(self, request: AssociationCreateOrUpdateRequest) -> AssociationResponse: + with self.lock: + resource_id = request.resource_id + resource_name = request.resource_name + resource_namespace = request.resource_namespace + resource_type = request.resource_type + + # Index resource_id by (namespace, name) + if resource_name and resource_namespace and resource_id: + self.resource_id_index[(resource_namespace, resource_name)] = resource_id + + created_associations = [] + if request.associations and resource_id is not None: + for assoc_info in request.associations: + association = Association( + subject=assoc_info.subject, + guid=None, + resource_name=resource_name, + resource_namespace=resource_namespace, + resource_id=resource_id, + resource_type=resource_type, + association_type=assoc_info.association_type, + frozen=assoc_info.frozen if assoc_info.frozen is not None else False, + ) + self.associations_by_resource_id[resource_id].append(association) + created_associations.append( + AssociationInfo( + subject=assoc_info.subject, + association_type=assoc_info.association_type, + lifecycle=assoc_info.lifecycle, + frozen=assoc_info.frozen if assoc_info.frozen is not None else False, + schema=assoc_info.schema, + ) + ) + + return AssociationResponse( + resource_name=resource_name, + resource_namespace=resource_namespace, + resource_id=resource_id, + resource_type=resource_type, + associations=created_associations, + ) + + def delete_associations( + self, + resource_id: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + ) -> None: + with self.lock: + if resource_id not in self.associations_by_resource_id: + return + + if association_types is None and resource_type is None: + # Delete all associations for this resource + del self.associations_by_resource_id[resource_id] + else: + # Filter and keep only non-matching associations + remaining = [] + for assoc in self.associations_by_resource_id[resource_id]: + keep = False + if resource_type is not None and assoc.resource_type != resource_type: + keep = True + if association_types is not None and assoc.association_type not in association_types: + keep = True + if keep: + remaining.append(assoc) + self.associations_by_resource_id[resource_id] = remaining + + def get_associations_by_resource_name( + self, + resource_name: str, + resource_namespace: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + ) -> List[Association]: + with self.lock: + result = [] + for resource_id, associations in self.associations_by_resource_id.items(): + for assoc in associations: + # Check if namespace matches (or is wildcard) + if resource_namespace != "-" and assoc.resource_namespace != resource_namespace: + continue + if assoc.resource_name != resource_name: + continue + if resource_type is not None and assoc.resource_type != resource_type: + continue + if association_types is not None and assoc.association_type not in association_types: + continue + result.append(assoc) + return result + + def clear(self): + with self.lock: + self.associations_by_resource_id.clear() + self.resource_id_index.clear() + + class MockSchemaRegistryClient(SchemaRegistryClient): def __init__(self, conf: dict): super().__init__(conf) self._store = _SchemaStore() + self._association_store = _AssociationStore() def register_schema(self, subject_name: str, schema: 'Schema', normalize_schemas: bool = False) -> int: registered_schema = self.register_schema_full_response( @@ -300,3 +416,46 @@ def set_config( def get_config(self, subject_name: Optional[str] = None) -> 'ServerConfig': # noqa F821 return None # type: ignore[return-value] + + def get_associations_by_resource_name( + self, + resource_name: str, + resource_namespace: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + offset: int = 0, + limit: int = -1, + ) -> List['Association']: + return self._association_store.get_associations_by_resource_name( + resource_name, resource_namespace, resource_type, association_types + ) + + def create_association(self, request: 'AssociationCreateOrUpdateRequest') -> 'AssociationResponse': + """ + Creates an association between a subject and a resource. + + Args: + request (AssociationCreateOrUpdateRequest): The association create or update request. + + Returns: + AssociationResponse: The response containing the created associations. + """ + return self._association_store.create_association(request) + + def delete_associations( + self, + resource_id: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + cascade_lifecycle: bool = False, + ) -> None: + """ + Deletes associations for a resource. + + Args: + resource_id (str): The resource identifier. + resource_type (str, optional): The type of resource (e.g., "topic"). + association_types (List[str], optional): The types of associations to delete. + cascade_lifecycle (bool): Whether to cascade the lifecycle policy to dependent schemas. + """ + self._association_store.delete_associations(resource_id, resource_type, association_types) diff --git a/src/confluent_kafka/schema_registry/_sync/protobuf.py b/src/confluent_kafka/schema_registry/_sync/protobuf.py index 5e1620276..4b66845c5 100644 --- a/src/confluent_kafka/schema_registry/_sync/protobuf.py +++ b/src/confluent_kafka/schema_registry/_sync/protobuf.py @@ -31,7 +31,6 @@ dual_schema_id_deserializer, prefix_schema_id_serializer, reference_subject_name_strategy, - topic_subject_name_strategy, ) from confluent_kafka.schema_registry.common.protobuf import ( PROTOBUF_TYPE, @@ -149,14 +148,28 @@ class ProtobufSerializer(BaseSerializer): | | | | | | | Defaults to True. | +-------------------------------------+----------+------------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +-------------------------------------+----------+------------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | | | | constructed. Standard naming strategies are | | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | @@ -227,7 +240,9 @@ class ProtobufSerializer(BaseSerializer): 'use.latest.version': False, 'use.latest.with.metadata': None, 'skip.known.types': True, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'reference.subject.name.strategy': reference_subject_name_strategy, 'schema.id.serializer': prefix_schema_id_serializer, 'use.deprecated.format': False, @@ -279,19 +294,18 @@ def __init_impl( if self._use_deprecated_format: raise ValueError("use.deprecated.format is no longer supported") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._ref_reference_subject_func = cast( Callable[[Optional[SerializationContext], Any], Optional[str]], conf_copy.pop('reference.subject.name.strategy'), ) if not callable(self._ref_reference_subject_func): - raise ValueError("subject.name.strategy must be callable") + raise ValueError("reference.subject.name.strategy must be callable") self._schema_id_serializer = cast( Callable[[bytes, Optional[SerializationContext], Any], bytes], conf_copy.pop('schema.id.serializer') @@ -414,7 +428,15 @@ def __serialize(self, message: Message, ctx: Optional[SerializationContext] = No if not isinstance(message, self._msg_class): raise ValueError("message must be of type {} not {}".format(self._msg_class, type(message))) - subject = self._subject_name_func(ctx, message.DESCRIPTOR.full_name) if ctx else None + subject = ( + ( + self._subject_name_func(ctx, message.DESCRIPTOR.full_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, message.DESCRIPTOR.full_name) + ) + if ctx + else None + ) latest_schema = None if subject is not None: latest_schema = self._get_reader_schema(subject, fmt='serialized') @@ -508,14 +530,28 @@ class ProtobufDeserializer(BaseDeserializer): | | | | | | | Defaults to None. | +-------------------------------------+----------+------------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +-------------------------------------+----------+------------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka. schema_registry | - | | | namespace . | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(bytes, SerializationContext, schema_id) | | | | -> io.BytesIO | @@ -534,7 +570,9 @@ class ProtobufDeserializer(BaseDeserializer): _default_conf = { 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.deserializer': dual_schema_id_deserializer, 'use.deprecated.format': False, } @@ -566,12 +604,11 @@ def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_deserializer = cast( Callable[[bytes, Optional[SerializationContext], Any], io.BytesIO], conf_copy.pop('schema.id.deserializer') @@ -619,7 +656,15 @@ def __deserialize(self, data: Optional[bytes], ctx: Optional[SerializationContex if data is None: return None - subject = self._subject_name_func(ctx, None) + subject = ( + ( + self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, None) + ) + if ctx + else None + ) latest_schema = None if subject is not None and self._registry is not None: latest_schema = self._get_reader_schema(subject, fmt='serialized') @@ -634,7 +679,15 @@ def __deserialize(self, data: Optional[bytes], ctx: Optional[SerializationContex writer_schema = pool.FindFileByName(fd_proto.name) writer_desc = self._get_message_desc(pool, writer_schema, msg_index if msg_index is not None else []) if subject is None: - subject = self._subject_name_func(ctx, writer_desc.full_name) + subject = ( + ( + self._subject_name_func(ctx, writer_desc.full_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, writer_desc.full_name) + ) + if ctx + else None + ) if subject is not None: latest_schema = self._get_reader_schema(subject, fmt='serialized') else: diff --git a/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py b/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py index 9f0e56a5a..1f01e46c4 100644 --- a/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py @@ -41,6 +41,9 @@ _StaticOAuthBearerFieldProviderBuilder, ) from confluent_kafka.schema_registry.common.schema_registry_client import ( + Association, + AssociationCreateOrUpdateRequest, + AssociationResponse, RegisteredSchema, Schema, SchemaVersion, @@ -399,7 +402,7 @@ def get(self, url: str, query: Optional[dict] = None) -> Any: def post(self, url: str, body: Optional[dict], **kwargs) -> Any: raise NotImplementedError() - def delete(self, url: str) -> Any: + def delete(self, url: str, query: Optional[dict] = None) -> Any: raise NotImplementedError() def put(self, url: str, body: Optional[dict] = None) -> Any: @@ -449,8 +452,8 @@ def get(self, url: str, query: Optional[dict] = None) -> Any: def post(self, url: str, body: Optional[dict], **kwargs) -> Any: return self.send_request(url, method='POST', body=body) - def delete(self, url: str) -> Any: - return self.send_request(url, method='DELETE') + def delete(self, url: str, query: Optional[dict] = None) -> Any: + return self.send_request(url, method='DELETE', query=query) def put(self, url: str, body: Optional[dict] = None) -> Any: return self.send_request(url, method='PUT', body=body) @@ -505,6 +508,8 @@ def send_request(self, url: str, method: str, body: Optional[dict] = None, query response = self.send_http_request(base_url, url, method, headers, body_str, query) if is_success(response.status_code): + if response.status_code == 204 or not response.content: + return None return response.json() if not is_retriable(response.status_code) or i == len(self.base_urls) - 1: @@ -1566,6 +1571,94 @@ def clear_caches(self): self._latest_with_metadata_cache.clear() self._cache.clear() + def get_associations_by_resource_name( + self, + resource_name: str, + resource_namespace: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + offset: int = 0, + limit: int = -1, + ) -> List['Association']: + """ + Retrieves associations for a given resource name and namespace. + + Args: + resource_name (str): The name of the resource (e.g., topic name). + resource_namespace (str): The namespace of the resource (e.g., kafka cluster ID). + Use "-" as a wildcard. + resource_type (str, optional): The type of resource (e.g., "topic"). + association_types (List[str], optional): The types of associations to filter by + (e.g., ["key", "value"]). + offset (int): Pagination offset for results. + limit (int): Pagination size for results. Ignored if negative. + + Returns: + List[Association]: List of associations matching the criteria. + + Raises: + SchemaRegistryError: if the request was unsuccessful. + """ + query: Dict[str, Any] = {} + if resource_type is not None: + query['resourceType'] = resource_type + if association_types is not None: + query['associationType'] = association_types + if offset > 0: + query['offset'] = offset + if limit >= 1: + query['limit'] = limit + + response = self._rest_client.get( + 'associations/resources/{}/{}'.format(_urlencode(resource_namespace), _urlencode(resource_name)), query + ) + + return [Association.from_dict(a) for a in response] + + def create_association(self, request: 'AssociationCreateOrUpdateRequest') -> 'AssociationResponse': + """ + Creates an association between a subject and a resource. + + Args: + request (AssociationCreateOrUpdateRequest): The association create or update request. + + Returns: + AssociationResponse: The response containing the created associations. + + Raises: + SchemaRegistryError: if the request was unsuccessful. + """ + response = self._rest_client.post('associations', body=request.to_dict()) + return AssociationResponse.from_dict(response) + + def delete_associations( + self, + resource_id: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + cascade_lifecycle: bool = False, + ) -> None: + """ + Deletes associations for a resource. + + Args: + resource_id (str): The resource identifier. + resource_type (str, optional): The type of resource (e.g., "topic"). + association_types (List[str], optional): The types of associations to delete + (e.g., ["key", "value"]). If not specified, all associations are deleted. + cascade_lifecycle (bool): Whether to cascade the lifecycle policy to dependent schemas. + + Raises: + SchemaRegistryError: if the request was unsuccessful. + """ + query: Dict[str, Any] = {'cascadeLifecycle': cascade_lifecycle} + if resource_type is not None: + query['resourceType'] = resource_type + if association_types is not None: + query['associationType'] = association_types + + self._rest_client.delete('associations/resources/{}'.format(_urlencode(resource_id)), query=query) + @staticmethod def new_client(conf: dict) -> 'SchemaRegistryClient': from .mock_schema_registry_client import MockSchemaRegistryClient diff --git a/src/confluent_kafka/schema_registry/_sync/serde.py b/src/confluent_kafka/schema_registry/_sync/serde.py index 56868cad0..58b835006 100644 --- a/src/confluent_kafka/schema_registry/_sync/serde.py +++ b/src/confluent_kafka/schema_registry/_sync/serde.py @@ -17,11 +17,19 @@ # import logging -from typing import Any, Callable, Dict, List, Optional, Set +import threading as _locks +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union -from confluent_kafka.schema_registry import RegisteredSchema +from cachetools import LRUCache + +from confluent_kafka.schema_registry import ( + RegisteredSchema, + SchemaRegistryClient, + topic_subject_name_strategy, +) from confluent_kafka.schema_registry.common.schema_registry_client import RulePhase from confluent_kafka.schema_registry.common.serde import ( + STRATEGY_TYPE_MAP, ErrorAction, FieldTransformer, Migration, @@ -31,19 +39,209 @@ RuleContext, RuleError, SchemaId, + SubjectNameStrategyType, ) +from confluent_kafka.schema_registry.error import SchemaRegistryError from confluent_kafka.schema_registry.schema_registry_client import Rule, RuleKind, RuleMode, RuleSet, Schema -from confluent_kafka.serialization import Deserializer, SerializationContext, SerializationError, Serializer +from confluent_kafka.serialization import ( + Deserializer, + MessageField, + SerializationContext, + SerializationError, + Serializer, +) __all__ = [ + 'AssociatedNameStrategy', 'BaseSerde', 'BaseSerializer', 'BaseDeserializer', + 'KAFKA_CLUSTER_ID', + 'FALLBACK_TYPE', ] log = logging.getLogger(__name__) +KAFKA_CLUSTER_ID = "subject.name.strategy.kafka.cluster.id" +NAMESPACE_WILDCARD = "-" +FALLBACK_TYPE = "subject.name.strategy.fallback.type" +DEFAULT_CACHE_CAPACITY = 1000 + + +class AssociatedNameStrategy: + """ + A subject name strategy that retrieves the associated subject name from schema registry + by querying associations for the topic. + + This class encapsulates a cache for subject name lookups to avoid repeated API calls. + + Args: + cache_capacity (int): Maximum number of entries to cache. Defaults to 1000. + """ + + def __init__(self, cache_capacity: int = DEFAULT_CACHE_CAPACITY): + self._cache: LRUCache = LRUCache(maxsize=cache_capacity) + self._lock: _locks.Lock = _locks.Lock() + + def _get_cache_key(self, topic: str, is_key: bool, record_name: Optional[str]) -> Tuple[str, bool, Optional[str]]: + """Create a cache key from topic, is_key, and record_name.""" + return (topic, is_key, record_name) + + def _load_subject_name( + self, + topic: str, + is_key: bool, + record_name: Optional[str], + ctx: SerializationContext, + schema_registry_client: SchemaRegistryClient, + conf: Optional[dict], + ) -> Optional[str]: + """Load the subject name from schema registry (not cached).""" + # Determine resource namespace from config + kafka_cluster_id = None + fallback_strategy = SubjectNameStrategyType.TOPIC # default fallback + + # If no client is available, skip association lookup and use fallback directly + if schema_registry_client is None: + return topic_subject_name_strategy(ctx, record_name) + + if conf is not None: + kafka_cluster_id = conf.get(KAFKA_CLUSTER_ID) + fallback_config = conf.get(FALLBACK_TYPE) + if fallback_config is not None: + if isinstance(fallback_config, SubjectNameStrategyType): + fallback_strategy = fallback_config + else: + try: + fallback_strategy = SubjectNameStrategyType(str(fallback_config).upper()) + except ValueError: + valid_fallbacks = [ + e.value for e in SubjectNameStrategyType if e != SubjectNameStrategyType.ASSOCIATED + ] + raise ValueError( + f"Invalid value for {FALLBACK_TYPE}: {fallback_config}. " + f"Valid values are: {', '.join(valid_fallbacks)}" + ) + + resource_namespace = kafka_cluster_id if kafka_cluster_id is not None else NAMESPACE_WILDCARD + + # Determine association type based on whether this is key or value + association_type = "key" if is_key else "value" + + # Query schema registry for associations + try: + associations = schema_registry_client.get_associations_by_resource_name( + resource_name=topic, + resource_namespace=resource_namespace, + resource_type="topic", + association_types=[association_type], + offset=0, + limit=-1, + ) + except SchemaRegistryError as e: + if e.http_status_code == 404: + # Treat 404 as no associations found and fall through to existing fallback logic + associations = [] + else: + raise + + if len(associations) > 1: + raise SerializationError(f"Multiple associated subjects found for topic {topic}") + elif len(associations) == 1: + return associations[0].subject + else: + # No associations found, use fallback strategy + if fallback_strategy == SubjectNameStrategyType.NONE: + raise SerializationError(f"No associated subject found for topic {topic}") + elif fallback_strategy == SubjectNameStrategyType.ASSOCIATED: + raise ValueError( + f"Invalid value for {FALLBACK_TYPE}: {fallback_strategy.value}. " + f"ASSOCIATED cannot be used as a fallback strategy." + ) + + return STRATEGY_TYPE_MAP[fallback_strategy](ctx, record_name) + + def __call__( + self, + ctx: Optional[SerializationContext], + record_name: Optional[str], + schema_registry_client: SchemaRegistryClient, + conf: Optional[dict] = None, + ) -> Optional[str]: + """ + Retrieves the associated subject name from schema registry by querying + associations for the topic. + + The topic is passed as the resource name to schema registry. If there is a + configuration property named "kafka.cluster.id", then its value will be passed + as the resource namespace; otherwise the value "-" will be passed as the + resource namespace. + + If more than one subject is returned from the query, a SerializationError + will be raised. If no subjects are returned from the query, then the behavior + will fall back to topic_subject_name_strategy, unless the configuration property + "subject.name.strategy.fallback.type" is set to "RECORD", "TOPIC_RECORD", or "NONE". + + Results are cached using an LRU cache to avoid repeated API calls. + + Args: + ctx (SerializationContext): Metadata pertaining to the serialization + operation. **Required** - must contain topic and field information. + + record_name (Optional[str]): Record name (used for fallback strategies). + + schema_registry_client (SchemaRegistryClient): SchemaRegistryClient instance. + + conf (Optional[dict]): Configuration dictionary. Supports: + - "subject.name.strategy.kafka.cluster.id": Kafka cluster ID to use as resource namespace. + - "subject.name.strategy.fallback.type": Fallback strategy when no + associations are found. One of "TOPIC", "RECORD", "TOPIC_RECORD", or "NONE". + Defaults to "TOPIC". + + Returns: + Optional[str]: The subject name from the association, or from the fallback strategy. + + Raises: + SerializationError: If multiple associated subjects are found for the topic, + or if no subjects are found and fallback is set to "NONE". + ValueError: If ctx is None. + """ + if ctx is None: + raise ValueError( + "SerializationContext is required for AssociatedNameStrategy. " + "Either provide a SerializationContext or use a different strategy." + ) + + topic = ctx.topic + if topic is None: + return None + + is_key = ctx.field == MessageField.KEY + cache_key = self._get_cache_key(topic, is_key, record_name) + + # Check cache first + with self._lock: + cached_result = self._cache.get(cache_key) + if cached_result is not None: + return cached_result + + # Not in cache, load from schema registry + result = self._load_subject_name(topic, is_key, record_name, ctx, schema_registry_client, conf) + + # Cache the result + if result is not None: + with self._lock: + self._cache[cache_key] = result + + return result + + def clear_cache(self) -> None: + """Clear the association subject name cache.""" + with self._lock: + self._cache.clear() + + class BaseSerde(object): __slots__ = [ '_use_schema_id', @@ -51,6 +249,8 @@ class BaseSerde(object): '_use_latest_with_metadata', '_registry', '_rule_registry', + '_strategy_accepts_client', + '_subject_name_conf', '_subject_name_func', '_field_transformer', ] @@ -60,9 +260,81 @@ class BaseSerde(object): _use_latest_with_metadata: Optional[Dict[str, str]] _registry: Any # SchemaRegistryClient _rule_registry: Any # RuleRegistry - _subject_name_func: Callable[[Optional['SerializationContext'], Optional[str]], Optional[str]] + _strategy_accepts_client: bool + _subject_name_conf: Optional[dict] + _subject_name_func: Callable[..., Any] _field_transformer: Optional[FieldTransformer] + def configure_subject_name_strategy( + self, + subject_name_strategy_type: Optional[Union[SubjectNameStrategyType, str]] = None, + subject_name_strategy_conf: Optional[dict] = None, + subject_name_strategy: Optional[Callable] = None, + ) -> None: + """ + Configure the subject name strategy for this serde. + + This method supports both the legacy callable approach and the new type-based approach. + If both `subject_name_strategy` (as a callable) and `subject_name_strategy_type` are + provided, the callable takes precedence. + + Args: + subject_name_strategy: A callable that implements the subject name strategy. + Signature: (SerializationContext, str) -> str or + (SerializationContext, str, SchemaRegistryClient, dict) -> str + + subject_name_strategy_type: The type of subject name strategy to use. + Can be a SubjectNameStrategyType enum value or a string + ("TOPIC", "RECORD", "TOPIC_RECORD", "ASSOCIATED"). + + subject_name_strategy_conf: Configuration dictionary passed to strategies + that accept extra parameters (like ASSOCIATED). + + Raises: + ValueError: If the strategy is not callable or the type is invalid. + """ + self._subject_name_conf = subject_name_strategy_conf + + # If a callable is provided, use it directly (backward compatible) + if subject_name_strategy is not None: + if not callable(subject_name_strategy): + raise ValueError("subject.name.strategy must be callable") + self._subject_name_func = subject_name_strategy + self._strategy_accepts_client = isinstance(subject_name_strategy, AssociatedNameStrategy) + return + + # If a type is provided, resolve it to a callable + if subject_name_strategy_type is not None: + # Convert string to enum if needed + if isinstance(subject_name_strategy_type, str): + try: + subject_name_strategy_type = SubjectNameStrategyType(subject_name_strategy_type.upper()) + except ValueError: + raise ValueError( + f"Invalid subject.name.strategy.type: {subject_name_strategy_type}. " + f"Valid values are: {[e.value for e in SubjectNameStrategyType]}" + ) + + # Handle ASSOCIATED specially since it needs schema_registry_client + if subject_name_strategy_type == SubjectNameStrategyType.ASSOCIATED: + self._subject_name_func = AssociatedNameStrategy() + self._strategy_accepts_client = True + elif subject_name_strategy_type == SubjectNameStrategyType.NONE: + raise ValueError( + f"Invalid subject.name.strategy.type: {subject_name_strategy_type}. " + f"NONE cannot be used as a subject name strategy." + ) + elif subject_name_strategy_type in STRATEGY_TYPE_MAP: + self._subject_name_func = STRATEGY_TYPE_MAP[subject_name_strategy_type] + self._strategy_accepts_client = False + else: + raise ValueError(f"Unknown subject.name.strategy.type: {subject_name_strategy_type}") + return + + # Default to AssociatedNameStrategy (falls back to TOPIC when no associations found) + self._subject_name_func = AssociatedNameStrategy() + self._strategy_accepts_client = True + def _get_reader_schema(self, subject: str, fmt: Optional[str] = None) -> Optional[RegisteredSchema]: if self._use_schema_id is not None: schema = self._registry.get_schema(self._use_schema_id, subject, fmt) diff --git a/src/confluent_kafka/schema_registry/common/schema_registry_client.py b/src/confluent_kafka/schema_registry/common/schema_registry_client.py index eaf2430f6..88819d085 100644 --- a/src/confluent_kafka/schema_registry/common/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/common/schema_registry_client.py @@ -42,6 +42,11 @@ 'ServerConfig', 'Schema', 'RegisteredSchema', + 'Association', + 'AssociationInfo', + 'AssociationCreateOrUpdateInfo', + 'AssociationCreateOrUpdateRequest', + 'AssociationResponse', ] VALID_AUTH_PROVIDERS = ['URL', 'USER_INFO'] @@ -972,3 +977,251 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: ) return schema # type: ignore[return-value] + + +@_attrs_define(frozen=True) +class Association: + """ + An association between a subject and a resource. + """ + + subject: Optional[str] + guid: Optional[str] + resource_name: Optional[str] + resource_namespace: Optional[str] + resource_id: Optional[str] + resource_type: Optional[str] + association_type: Optional[str] + frozen: bool = False + + def to_dict(self) -> Dict[str, Any]: + field_dict: Dict[str, Any] = {} + if self.subject is not None: + field_dict["subject"] = self.subject + if self.guid is not None: + field_dict["guid"] = self.guid + if self.resource_name is not None: + field_dict["resourceName"] = self.resource_name + if self.resource_namespace is not None: + field_dict["resourceNamespace"] = self.resource_namespace + if self.resource_id is not None: + field_dict["resourceId"] = self.resource_id + if self.resource_type is not None: + field_dict["resourceType"] = self.resource_type + if self.association_type is not None: + field_dict["associationType"] = self.association_type + field_dict["frozen"] = self.frozen + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + + subject = d.pop("subject", None) + guid = d.pop("guid", None) + resource_name = d.pop("resourceName", None) + resource_namespace = d.pop("resourceNamespace", None) + resource_id = d.pop("resourceId", None) + resource_type = d.pop("resourceType", None) + association_type = d.pop("associationType", None) + frozen = d.pop("frozen", False) + + association = cls( # type: ignore[call-arg] + subject=subject, + guid=guid, + resource_name=resource_name, + resource_namespace=resource_namespace, + resource_id=resource_id, + resource_type=resource_type, + association_type=association_type, + frozen=frozen, + ) + + return association + + +@_attrs_define +class AssociationInfo: + """ + Information about an association in a response. + """ + + subject: Optional[str] = None + association_type: Optional[str] = None + lifecycle: Optional[str] = None + frozen: bool = False + schema: Optional['Schema'] = None + + def to_dict(self) -> Dict[str, Any]: + field_dict: Dict[str, Any] = {} + if self.subject is not None: + field_dict["subject"] = self.subject + if self.association_type is not None: + field_dict["associationType"] = self.association_type + if self.lifecycle is not None: + field_dict["lifecycle"] = self.lifecycle + field_dict["frozen"] = self.frozen + if self.schema is not None: + field_dict["schema"] = self.schema.to_dict() + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + subject = d.pop("subject", None) + association_type = d.pop("associationType", None) + lifecycle = d.pop("lifecycle", None) + frozen = d.pop("frozen", False) + schema_dict = d.pop("schema", None) + schema = Schema.from_dict(schema_dict) if schema_dict else None + + return cls( # type: ignore[call-arg] + subject=subject, + association_type=association_type, + lifecycle=lifecycle, + frozen=frozen, + schema=schema, + ) + + +@_attrs_define +class AssociationCreateOrUpdateInfo: + """ + Information about an association to create or update. + """ + + subject: Optional[str] = None + association_type: Optional[str] = None + lifecycle: Optional[str] = None + frozen: Optional[bool] = None + schema: Optional['Schema'] = None + normalize: Optional[bool] = None + + def to_dict(self) -> Dict[str, Any]: + field_dict: Dict[str, Any] = {} + if self.subject is not None: + field_dict["subject"] = self.subject + if self.association_type is not None: + field_dict["associationType"] = self.association_type + if self.lifecycle is not None: + field_dict["lifecycle"] = self.lifecycle + if self.frozen is not None: + field_dict["frozen"] = self.frozen + if self.schema is not None: + field_dict["schema"] = self.schema.to_dict() + if self.normalize is not None: + field_dict["normalize"] = self.normalize + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + subject = d.pop("subject", None) + association_type = d.pop("associationType", None) + lifecycle = d.pop("lifecycle", None) + frozen = d.pop("frozen", None) + schema_dict = d.pop("schema", None) + schema = Schema.from_dict(schema_dict) if schema_dict else None + normalize = d.pop("normalize", None) + + return cls( # type: ignore[call-arg] + subject=subject, + association_type=association_type, + lifecycle=lifecycle, + frozen=frozen, + schema=schema, + normalize=normalize, + ) + + +@_attrs_define +class AssociationCreateOrUpdateRequest: + """ + Request to create or update associations. + """ + + resource_name: Optional[str] = None + resource_namespace: Optional[str] = None + resource_id: Optional[str] = None + resource_type: Optional[str] = None + associations: Optional[List[AssociationCreateOrUpdateInfo]] = None + + def to_dict(self) -> Dict[str, Any]: + field_dict: Dict[str, Any] = {} + if self.resource_name is not None: + field_dict["resourceName"] = self.resource_name + if self.resource_namespace is not None: + field_dict["resourceNamespace"] = self.resource_namespace + if self.resource_id is not None: + field_dict["resourceId"] = self.resource_id + if self.resource_type is not None: + field_dict["resourceType"] = self.resource_type + if self.associations is not None: + field_dict["associations"] = [a.to_dict() for a in self.associations] + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + resource_name = d.pop("resourceName", None) + resource_namespace = d.pop("resourceNamespace", None) + resource_id = d.pop("resourceId", None) + resource_type = d.pop("resourceType", None) + associations_list = d.pop("associations", None) + associations = ( + [AssociationCreateOrUpdateInfo.from_dict(a) for a in associations_list] if associations_list else None + ) + + return cls( # type: ignore[call-arg] + resource_name=resource_name, + resource_namespace=resource_namespace, + resource_id=resource_id, + resource_type=resource_type, + associations=associations, + ) + + +@_attrs_define +class AssociationResponse: + """ + Response from creating or updating associations. + """ + + resource_name: Optional[str] = None + resource_namespace: Optional[str] = None + resource_id: Optional[str] = None + resource_type: Optional[str] = None + associations: Optional[List[AssociationInfo]] = None + + def to_dict(self) -> Dict[str, Any]: + field_dict: Dict[str, Any] = {} + if self.resource_name is not None: + field_dict["resourceName"] = self.resource_name + if self.resource_namespace is not None: + field_dict["resourceNamespace"] = self.resource_namespace + if self.resource_id is not None: + field_dict["resourceId"] = self.resource_id + if self.resource_type is not None: + field_dict["resourceType"] = self.resource_type + if self.associations is not None: + field_dict["associations"] = [a.to_dict() for a in self.associations] + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + resource_name = d.pop("resourceName", None) + resource_namespace = d.pop("resourceNamespace", None) + resource_id = d.pop("resourceId", None) + resource_type = d.pop("resourceType", None) + associations_list = d.pop("associations", None) + associations = [AssociationInfo.from_dict(a) for a in associations_list] if associations_list else None + + return cls( # type: ignore[call-arg] + resource_name=resource_name, + resource_namespace=resource_namespace, + resource_id=resource_id, + resource_type=resource_type, + associations=associations, + ) diff --git a/src/confluent_kafka/schema_registry/common/serde.py b/src/confluent_kafka/schema_registry/common/serde.py index c41de51fe..decb6e45c 100644 --- a/src/confluent_kafka/schema_registry/common/serde.py +++ b/src/confluent_kafka/schema_registry/common/serde.py @@ -25,12 +25,21 @@ from threading import Lock from typing import Any, Callable, Dict, List, Optional, Set, TypeVar -from confluent_kafka.schema_registry import _MAGIC_BYTE_V0, _MAGIC_BYTE_V1, RegisteredSchema +from confluent_kafka.schema_registry import ( + _MAGIC_BYTE_V0, + _MAGIC_BYTE_V1, + RegisteredSchema, + record_subject_name_strategy, + topic_record_subject_name_strategy, + topic_subject_name_strategy, +) from confluent_kafka.schema_registry.schema_registry_client import Rule, RuleKind, RuleMode, Schema from confluent_kafka.schema_registry.wildcard_matcher import wildcard_match from confluent_kafka.serialization import SerializationContext, SerializationError __all__ = [ + 'STRATEGY_TYPE_MAP', + 'SubjectNameStrategyType', 'FieldType', 'FieldContext', 'RuleContext', @@ -52,6 +61,23 @@ log = logging.getLogger(__name__) +class SubjectNameStrategyType(str, Enum): + NONE = "NONE" + TOPIC = "TOPIC" + RECORD = "RECORD" + TOPIC_RECORD = "TOPIC_RECORD" + ASSOCIATED = "ASSOCIATED" + + +# Mapping from SubjectNameStrategyType to strategy functions. +# NONE and ASSOCIATED are handled specially and not included here. +STRATEGY_TYPE_MAP = { + SubjectNameStrategyType.TOPIC: topic_subject_name_strategy, + SubjectNameStrategyType.RECORD: record_subject_name_strategy, + SubjectNameStrategyType.TOPIC_RECORD: topic_record_subject_name_strategy, +} + + class FieldType(str, Enum): RECORD = "RECORD" ENUM = "ENUM" diff --git a/tests/schema_registry/_async/test_avro.py b/tests/schema_registry/_async/test_avro.py index 24b64d355..06b346321 100644 --- a/tests/schema_registry/_async/test_avro.py +++ b/tests/schema_registry/_async/test_avro.py @@ -193,16 +193,22 @@ async def test_avro_serializer_topic_record_subject_name_strategy_primitive(load assert ctx.headers is None -async def test_avro_serializer_subject_name_strategy_default(load_avsc): +async def test_avro_serializer_subject_name_strategy_default(mock_schema_registry, load_avsc): """ - Ensures record_subject_name_strategy returns the correct record name + Ensures the default subject name strategy returns the correct subject name """ conf = {'url': TEST_URL} test_client = AsyncSchemaRegistryClient(conf) test_serializer = await AsyncAvroSerializer(test_client, load_avsc('basic_schema.avsc')) ctx = SerializationContext('test_subj', MessageField.VALUE) - assert test_serializer._subject_name_func(ctx, test_serializer._schema_name) == 'test_subj-value' + if test_serializer._strategy_accepts_client: + result = await test_serializer._subject_name_func( + ctx, test_serializer._schema_name, test_client, test_serializer._subject_name_conf + ) + else: + result = test_serializer._subject_name_func(ctx, test_serializer._schema_name) + assert result == 'test_subj-value' async def test_avro_serializer_schema_loads_union(load_avsc): diff --git a/tests/schema_registry/_async/test_avro_serdes.py b/tests/schema_registry/_async/test_avro_serdes.py index ce168bdfa..cd6f5e54b 100644 --- a/tests/schema_registry/_async/test_avro_serdes.py +++ b/tests/schema_registry/_async/test_avro_serdes.py @@ -29,7 +29,16 @@ Schema, header_schema_id_serializer, ) +from confluent_kafka.schema_registry._async.serde import ( + FALLBACK_TYPE, + KAFKA_CLUSTER_ID, +) from confluent_kafka.schema_registry.avro import AsyncAvroDeserializer, AsyncAvroSerializer +from confluent_kafka.schema_registry.common.schema_registry_client import ( + AssociationCreateOrUpdateInfo, + AssociationCreateOrUpdateRequest, +) +from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.rule_registry import RuleOverride, RuleRegistry from confluent_kafka.schema_registry.rules.cel.cel_executor import CelExecutor from confluent_kafka.schema_registry.rules.cel.cel_field_executor import CelFieldExecutor @@ -2658,3 +2667,371 @@ def __init__(self, award, user): def __eq__(self, other): return all([self.award == other.award, self.user == other.user]) + + +async def test_associated_name_strategy_with_association(): + """Test that AsyncAssociatedNameStrategy returns subject from association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema and test object + schema = { + 'type': 'record', + 'name': 'TestRecord', + 'fields': [ + {'name': 'intField', 'type': 'int'}, + {'name': 'stringField', 'type': 'string'}, + ], + } + obj = {'intField': 123, 'stringField': 'hello'} + + # Add an association for the custom subject + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="mock-resource-id-1", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-custom-subject-value", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), + ) + ], + ) + await client.create_association(request) + + # Create serializer with associated name strategy + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + # Deserialize and verify + deser = await AsyncAvroDeserializer(client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Verify the schema was registered with the custom subject + registered_schema = await client.get_latest_version("my-custom-subject-value") + assert registered_schema is not None + + await client.delete_associations(resource_id="mock-resource-id-1", cascade_lifecycle=True) + + +async def test_associated_name_strategy_with_key_association(): + """Test that AsyncAssociatedNameStrategy returns subject for key""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema and test object + schema = { + 'type': 'record', + 'name': 'KeyRecord', + 'fields': [ + {'name': 'id', 'type': 'int'}, + ], + } + obj = {'id': 42} + + # Add an association for key + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="mock-resource-id-2", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-key-subject", + association_type="key", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), + ) + ], + ) + await client.create_association(request) + + # Create serializer with associated name strategy for KEY + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.KEY) + obj_bytes = await ser(obj, ser_ctx) + + # Deserialize and verify + deser = await AsyncAvroDeserializer(client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Verify the schema was registered with the key subject + registered_schema = await client.get_latest_version("my-key-subject") + assert registered_schema is not None + + await client.delete_associations(resource_id="mock-resource-id-2", cascade_lifecycle=True) + + +async def test_associated_name_strategy_fallback_to_topic(): + """Test fallback to topic_subject_name_strategy when no association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema and test object + schema = { + 'type': 'record', + 'name': 'TestRecord', + 'fields': [ + {'name': 'intField', 'type': 'int'}, + {'name': 'stringField', 'type': 'string'}, + ], + } + obj = {'intField': 456, 'stringField': 'world'} + + # No associations added, should fall back to topic strategy + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + # Deserialize and verify + deser = await AsyncAvroDeserializer(client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Default fallback is topic_subject_name_strategy which returns topic-value + registered_schema = await client.get_latest_version(_TOPIC + "-value") + assert registered_schema is not None + + +async def test_associated_name_strategy_fallback_to_record(): + """Test fallback to record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema with a specific record name + schema = { + 'type': 'record', + 'name': 'MyRecord', + 'fields': [ + {'name': 'value', 'type': 'string'}, + ], + } + obj = {'value': 'test'} + + # No associations, configure fallback to RECORD + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.RECORD}, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + # Deserialize and verify + deser = await AsyncAvroDeserializer(client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Should have registered under the record name + registered_schema = await client.get_latest_version("MyRecord") + assert registered_schema is not None + + +async def test_associated_name_strategy_fallback_to_topic_record(): + """Test fallback to topic_record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema with a specific record name + schema = { + 'type': 'record', + 'name': 'MyRecord', + 'fields': [ + {'name': 'data', 'type': 'int'}, + ], + } + obj = {'data': 789} + + # No associations, configure fallback to TOPIC_RECORD + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + # Deserialize and verify + deser = await AsyncAvroDeserializer(client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Should have registered under topic-record_name + registered_schema = await client.get_latest_version(_TOPIC + "-MyRecord") + assert registered_schema is not None + + +async def test_associated_name_strategy_fallback_none_raises(): + """Test that NONE fallback raises an error when no association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema + schema = { + 'type': 'record', + 'name': 'MyRecord', + 'fields': [ + {'name': 'value', 'type': 'string'}, + ], + } + obj = {'value': 'test'} + + # No associations, configure fallback to NONE + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: "NONE"}, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + await ser(obj, ser_ctx) + + assert "No associated subject found" in str(exc_info.value) + + +async def test_associated_name_strategy_with_kafka_cluster_id(): + """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema + schema = { + 'type': 'record', + 'name': 'TestRecord', + 'fields': [ + {'name': 'intField', 'type': 'int'}, + ], + } + obj = {'intField': 100} + + # Add an association with specific namespace + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="my-cluster-id", + resource_id="mock-resource-id-4", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cluster-specific-subject", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), + ) + ], + ) + await client.create_association(request) + + # Create serializer with matching cluster ID + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {KAFKA_CLUSTER_ID: "my-cluster-id"}, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + # Deserialize and verify + deser = await AsyncAvroDeserializer(client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Verify the schema was registered with the cluster-specific subject + registered_schema = await client.get_latest_version("cluster-specific-subject") + assert registered_schema is not None + + await client.delete_associations(resource_id="mock-resource-id-4", cascade_lifecycle=True) + + +async def test_associated_name_strategy_caching(): + """Test that results are cached within a strategy instance and serializer works with caching""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema + schema = { + 'type': 'record', + 'name': 'CacheTestRecord', + 'fields': [ + {'name': 'count', 'type': 'int'}, + ], + } + + # Add an association + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="mock-resource-id-5", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cached-subject", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), + ) + ], + ) + await client.create_association(request) + + # Create serializer with associated name strategy + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + # First serialization + obj1 = {'count': 1} + obj_bytes1 = await ser(obj1, ser_ctx) + + # Verify it was registered under cached-subject + registered_schema = await client.get_latest_version("cached-subject") + assert registered_schema is not None + + # Deserialize first message + deser = await AsyncAvroDeserializer(client) + result1 = await deser(obj_bytes1, ser_ctx) + assert obj1 == result1 + + # Delete associations (but serializer should still work due to caching) + await client.delete_associations(resource_id="mock-resource-id-5", cascade_lifecycle=True) + + # Second serialization should still work (schema already registered) + obj2 = {'count': 2} + obj_bytes2 = await ser(obj2, ser_ctx) + + # Deserialize second message + result2 = await deser(obj_bytes2, ser_ctx) + assert obj2 == result2 diff --git a/tests/schema_registry/_async/test_json.py b/tests/schema_registry/_async/test_json.py index 59c826c61..ae610c607 100644 --- a/tests/schema_registry/_async/test_json.py +++ b/tests/schema_registry/_async/test_json.py @@ -77,6 +77,7 @@ async def test_custom_json_encoder(): mock_schema_registry_client.register_schema_full_response.return_value = RegisteredSchema( schema_id=1, guid=None, schema=Schema(schema_str), subject="topic-name-value", version=1 ) + mock_schema_registry_client.get_associations_by_resource_name.return_value = [] # Use orjson.dumps as the custom encoder serializer = await AsyncJSONSerializer( @@ -136,6 +137,7 @@ async def test_custom_encoder_decoder_chain(): mock_schema_registry_client.register_schema_full_response.return_value = RegisteredSchema( schema_id=1, guid=None, schema=Schema(schema_str), subject="topic-name-value", version=1 ) + mock_schema_registry_client.get_associations_by_resource_name.return_value = [] def custom_encoder(obj): return orjson.dumps(obj, option=orjson.OPT_SORT_KEYS) @@ -176,6 +178,7 @@ async def test_custom_encoding_with_complex_data(): mock_schema_registry_client.register_schema_full_response.return_value = RegisteredSchema( schema_id=1, guid=None, schema=Schema(schema_str), subject="topic-name-value", version=1 ) + mock_schema_registry_client.get_associations_by_resource_name.return_value = [] def custom_encoder(obj): return json.dumps(obj, indent=2) diff --git a/tests/schema_registry/_async/test_json_serdes.py b/tests/schema_registry/_async/test_json_serdes.py index 6059cb55c..abde55f18 100644 --- a/tests/schema_registry/_async/test_json_serdes.py +++ b/tests/schema_registry/_async/test_json_serdes.py @@ -27,6 +27,15 @@ Schema, header_schema_id_serializer, ) +from confluent_kafka.schema_registry._async.serde import ( + FALLBACK_TYPE, + KAFKA_CLUSTER_ID, +) +from confluent_kafka.schema_registry.common.schema_registry_client import ( + AssociationCreateOrUpdateInfo, + AssociationCreateOrUpdateRequest, +) +from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.json_schema import AsyncJSONDeserializer, AsyncJSONSerializer from confluent_kafka.schema_registry.rules.cel.cel_executor import CelExecutor from confluent_kafka.schema_registry.rules.cel.cel_field_executor import CelFieldExecutor @@ -1449,3 +1458,276 @@ async def test_json_deeply_nested_refs(): executor.executor.client = dek_client obj2 = await deser(obj_bytes, ser_ctx) assert obj == obj2 + + +_JSON_SCHEMA = json.dumps( + { + "type": "object", + "title": "MyRecord", + "properties": { + "name": {"type": "string"}, + "id": {"type": "integer"}, + }, + } +) +_JSON_OBJ = {"name": "Kafka", "id": 123} + + +async def test_json_associated_name_strategy_with_association(): + """Test that AssociatedNameStrategy returns subject from association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="json-resource-id-1", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-custom-subject-value", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(_JSON_OBJ, ser_ctx) + + deser = await AsyncJSONDeserializer(None, schema_registry_client=client) + obj2 = await deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = await client.get_latest_version("my-custom-subject-value") + assert registered_schema is not None + + await client.delete_associations(resource_id="json-resource-id-1", cascade_lifecycle=True) + + +async def test_json_associated_name_strategy_with_key_association(): + """Test that AssociatedNameStrategy returns subject for key""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="json-resource-id-2", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-key-subject", + association_type="key", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.KEY) + obj_bytes = await ser(_JSON_OBJ, ser_ctx) + + deser = await AsyncJSONDeserializer(None, schema_registry_client=client) + obj2 = await deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = await client.get_latest_version("my-key-subject") + assert registered_schema is not None + + await client.delete_associations(resource_id="json-resource-id-2", cascade_lifecycle=True) + + +async def test_json_associated_name_strategy_fallback_to_topic(): + """Test fallback to topic_subject_name_strategy when no association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(_JSON_OBJ, ser_ctx) + + deser = await AsyncJSONDeserializer(None, schema_registry_client=client) + obj2 = await deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = await client.get_latest_version(_TOPIC + "-value") + assert registered_schema is not None + + +async def test_json_associated_name_strategy_fallback_to_record(): + """Test fallback to record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.RECORD}, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(_JSON_OBJ, ser_ctx) + + deser = await AsyncJSONDeserializer(None, schema_registry_client=client) + obj2 = await deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + # JSON record name comes from schema "title" + registered_schema = await client.get_latest_version("MyRecord") + assert registered_schema is not None + + +async def test_json_associated_name_strategy_fallback_to_topic_record(): + """Test fallback to topic_record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(_JSON_OBJ, ser_ctx) + + deser = await AsyncJSONDeserializer(None, schema_registry_client=client) + obj2 = await deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + # JSON topic-record subject: "topic1-MyRecord" + registered_schema = await client.get_latest_version(_TOPIC + "-MyRecord") + assert registered_schema is not None + + +async def test_json_associated_name_strategy_fallback_none_raises(): + """Test that NONE fallback raises an error when no association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: "NONE"}, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + await ser(_JSON_OBJ, ser_ctx) + + assert "No associated subject found" in str(exc_info.value) + + +async def test_json_associated_name_strategy_with_kafka_cluster_id(): + """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="my-cluster-id", + resource_id="json-resource-id-4", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cluster-specific-json-subject", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {KAFKA_CLUSTER_ID: "my-cluster-id"}, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(_JSON_OBJ, ser_ctx) + + deser = await AsyncJSONDeserializer(None, schema_registry_client=client) + obj2 = await deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = await client.get_latest_version("cluster-specific-json-subject") + assert registered_schema is not None + + await client.delete_associations(resource_id="json-resource-id-4", cascade_lifecycle=True) + + +async def test_json_associated_name_strategy_caching(): + """Test that results are cached within a strategy instance and serializer works with caching""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="json-resource-id-5", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="json-cached-subject", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + obj1 = {"name": "Kafka", "id": 1} + obj_bytes1 = await ser(obj1, ser_ctx) + + registered_schema = await client.get_latest_version("json-cached-subject") + assert registered_schema is not None + + deser = await AsyncJSONDeserializer(None, schema_registry_client=client) + result1 = await deser(obj_bytes1, ser_ctx) + assert obj1 == result1 + + # Delete associations (but serializer should still work due to caching) + await client.delete_associations(resource_id="json-resource-id-5", cascade_lifecycle=True) + + obj2 = {"name": "Kafka", "id": 2} + obj_bytes2 = await ser(obj2, ser_ctx) + + result2 = await deser(obj_bytes2, ser_ctx) + assert obj2 == result2 diff --git a/tests/schema_registry/_async/test_proto_serdes.py b/tests/schema_registry/_async/test_proto_serdes.py index e08e62501..f61118e96 100644 --- a/tests/schema_registry/_async/test_proto_serdes.py +++ b/tests/schema_registry/_async/test_proto_serdes.py @@ -24,6 +24,15 @@ from confluent_kafka.schema_registry import Metadata, MetadataProperties, Schema, header_schema_id_serializer from confluent_kafka.schema_registry._async.protobuf import AsyncProtobufDeserializer, AsyncProtobufSerializer from confluent_kafka.schema_registry._async.schema_registry_client import AsyncSchemaRegistryClient +from confluent_kafka.schema_registry._async.serde import ( + FALLBACK_TYPE, + KAFKA_CLUSTER_ID, +) +from confluent_kafka.schema_registry.common.schema_registry_client import ( + AssociationCreateOrUpdateInfo, + AssociationCreateOrUpdateRequest, +) +from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.protobuf import _schema_to_str from confluent_kafka.schema_registry.rules.cel.cel_executor import CelExecutor from confluent_kafka.schema_registry.rules.cel.cel_field_executor import CelFieldExecutor @@ -615,3 +624,286 @@ async def deserialize_with_all_versions(client, ser_ctx, obj_bytes, obj, obj2, o deser = await AsyncProtobufDeserializer(newerwidget_pb2.NewerWidget, deser_conf, client) newobj = await deser(obj_bytes, ser_ctx) assert obj3.length == newobj.length + + +async def test_associated_name_strategy_with_association(): + """Test that AssociatedNameStrategy returns subject from association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=123, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="proto-resource-id-1", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-custom-subject-value", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + deser = await AsyncProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = await client.get_latest_version("my-custom-subject-value") + assert registered_schema is not None + + await client.delete_associations(resource_id="proto-resource-id-1", cascade_lifecycle=True) + + +async def test_associated_name_strategy_with_key_association(): + """Test that AssociatedNameStrategy returns subject for key""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author(name='Kafka', id=42) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="proto-resource-id-2", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-key-subject", + association_type="key", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.KEY) + obj_bytes = await ser(obj, ser_ctx) + + deser = await AsyncProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = await client.get_latest_version("my-key-subject") + assert registered_schema is not None + + await client.delete_associations(resource_id="proto-resource-id-2", cascade_lifecycle=True) + + +async def test_associated_name_strategy_fallback_to_topic(): + """Test fallback to topic_subject_name_strategy when no association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=456, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + deser = await AsyncProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = await client.get_latest_version(_TOPIC + "-value") + assert registered_schema is not None + + +async def test_associated_name_strategy_fallback_to_record(): + """Test fallback to record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=789, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.RECORD}, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + deser = await AsyncProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = await client.get_latest_version(example_pb2.Author.DESCRIPTOR.full_name) + assert registered_schema is not None + + +async def test_associated_name_strategy_fallback_to_topic_record(): + """Test fallback to topic_record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=100, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + deser = await AsyncProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = await client.get_latest_version(_TOPIC + "-" + example_pb2.Author.DESCRIPTOR.full_name) + assert registered_schema is not None + + +async def test_associated_name_strategy_fallback_none_raises(): + """Test that NONE fallback raises an error when no association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author(name='Kafka', id=1) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: "NONE"}, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + await ser(obj, ser_ctx) + + assert "No associated subject found" in str(exc_info.value) + + +async def test_associated_name_strategy_with_kafka_cluster_id(): + """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=100, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="my-cluster-id", + resource_id="proto-resource-id-4", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cluster-specific-proto-subject", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {KAFKA_CLUSTER_ID: "my-cluster-id"}, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + deser = await AsyncProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = await client.get_latest_version("cluster-specific-proto-subject") + assert registered_schema is not None + + await client.delete_associations(resource_id="proto-resource-id-4", cascade_lifecycle=True) + + +async def test_associated_name_strategy_caching(): + """Test that results are cached within a strategy instance and serializer works with caching""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="proto-resource-id-5", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="proto-cached-subject", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + obj1 = example_pb2.Author(name='Kafka', id=1) + obj_bytes1 = await ser(obj1, ser_ctx) + + registered_schema = await client.get_latest_version("proto-cached-subject") + assert registered_schema is not None + + deser = await AsyncProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + result1 = await deser(obj_bytes1, ser_ctx) + assert obj1 == result1 + + # Delete associations (but serializer should still work due to caching) + await client.delete_associations(resource_id="proto-resource-id-5", cascade_lifecycle=True) + + obj2 = example_pb2.Author(name='Kafka', id=2) + obj_bytes2 = await ser(obj2, ser_ctx) + + result2 = await deser(obj_bytes2, ser_ctx) + assert obj2 == result2 diff --git a/tests/schema_registry/_sync/test_avro.py b/tests/schema_registry/_sync/test_avro.py index 7a9872285..006734157 100644 --- a/tests/schema_registry/_sync/test_avro.py +++ b/tests/schema_registry/_sync/test_avro.py @@ -189,16 +189,22 @@ def test_avro_serializer_topic_record_subject_name_strategy_primitive(load_avsc) assert ctx.headers is None -def test_avro_serializer_subject_name_strategy_default(load_avsc): +def test_avro_serializer_subject_name_strategy_default(mock_schema_registry, load_avsc): """ - Ensures record_subject_name_strategy returns the correct record name + Ensures the default subject name strategy returns the correct subject name """ conf = {'url': TEST_URL} test_client = SchemaRegistryClient(conf) test_serializer = AvroSerializer(test_client, load_avsc('basic_schema.avsc')) ctx = SerializationContext('test_subj', MessageField.VALUE) - assert test_serializer._subject_name_func(ctx, test_serializer._schema_name) == 'test_subj-value' + if test_serializer._strategy_accepts_client: + result = test_serializer._subject_name_func( + ctx, test_serializer._schema_name, test_client, test_serializer._subject_name_conf + ) + else: + result = test_serializer._subject_name_func(ctx, test_serializer._schema_name) + assert result == 'test_subj-value' def test_avro_serializer_schema_loads_union(load_avsc): diff --git a/tests/schema_registry/_sync/test_avro_serdes.py b/tests/schema_registry/_sync/test_avro_serdes.py index 0c8ca7977..c244756bf 100644 --- a/tests/schema_registry/_sync/test_avro_serdes.py +++ b/tests/schema_registry/_sync/test_avro_serdes.py @@ -29,7 +29,16 @@ SchemaRegistryClient, header_schema_id_serializer, ) +from confluent_kafka.schema_registry._sync.serde import ( + FALLBACK_TYPE, + KAFKA_CLUSTER_ID, +) from confluent_kafka.schema_registry.avro import AvroDeserializer, AvroSerializer +from confluent_kafka.schema_registry.common.schema_registry_client import ( + AssociationCreateOrUpdateInfo, + AssociationCreateOrUpdateRequest, +) +from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.rule_registry import RuleOverride, RuleRegistry from confluent_kafka.schema_registry.rules.cel.cel_executor import CelExecutor from confluent_kafka.schema_registry.rules.cel.cel_field_executor import CelFieldExecutor @@ -2658,3 +2667,371 @@ def __init__(self, award, user): def __eq__(self, other): return all([self.award == other.award, self.user == other.user]) + + +def test_associated_name_strategy_with_association(): + """Test that AssociatedNameStrategy returns subject from association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema and test object + schema = { + 'type': 'record', + 'name': 'TestRecord', + 'fields': [ + {'name': 'intField', 'type': 'int'}, + {'name': 'stringField', 'type': 'string'}, + ], + } + obj = {'intField': 123, 'stringField': 'hello'} + + # Add an association for the custom subject + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="mock-resource-id-1", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-custom-subject-value", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), + ) + ], + ) + client.create_association(request) + + # Create serializer with associated name strategy + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + # Deserialize and verify + deser = AvroDeserializer(client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Verify the schema was registered with the custom subject + registered_schema = client.get_latest_version("my-custom-subject-value") + assert registered_schema is not None + + client.delete_associations(resource_id="mock-resource-id-1", cascade_lifecycle=True) + + +def test_associated_name_strategy_with_key_association(): + """Test that AssociatedNameStrategy returns subject for key""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema and test object + schema = { + 'type': 'record', + 'name': 'KeyRecord', + 'fields': [ + {'name': 'id', 'type': 'int'}, + ], + } + obj = {'id': 42} + + # Add an association for key + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="mock-resource-id-2", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-key-subject", + association_type="key", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), + ) + ], + ) + client.create_association(request) + + # Create serializer with associated name strategy for KEY + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.KEY) + obj_bytes = ser(obj, ser_ctx) + + # Deserialize and verify + deser = AvroDeserializer(client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Verify the schema was registered with the key subject + registered_schema = client.get_latest_version("my-key-subject") + assert registered_schema is not None + + client.delete_associations(resource_id="mock-resource-id-2", cascade_lifecycle=True) + + +def test_associated_name_strategy_fallback_to_topic(): + """Test fallback to topic_subject_name_strategy when no association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema and test object + schema = { + 'type': 'record', + 'name': 'TestRecord', + 'fields': [ + {'name': 'intField', 'type': 'int'}, + {'name': 'stringField', 'type': 'string'}, + ], + } + obj = {'intField': 456, 'stringField': 'world'} + + # No associations added, should fall back to topic strategy + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + # Deserialize and verify + deser = AvroDeserializer(client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Default fallback is topic_subject_name_strategy which returns topic-value + registered_schema = client.get_latest_version(_TOPIC + "-value") + assert registered_schema is not None + + +def test_associated_name_strategy_fallback_to_record(): + """Test fallback to record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema with a specific record name + schema = { + 'type': 'record', + 'name': 'MyRecord', + 'fields': [ + {'name': 'value', 'type': 'string'}, + ], + } + obj = {'value': 'test'} + + # No associations, configure fallback to RECORD + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.RECORD}, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + # Deserialize and verify + deser = AvroDeserializer(client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Should have registered under the record name + registered_schema = client.get_latest_version("MyRecord") + assert registered_schema is not None + + +def test_associated_name_strategy_fallback_to_topic_record(): + """Test fallback to topic_record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema with a specific record name + schema = { + 'type': 'record', + 'name': 'MyRecord', + 'fields': [ + {'name': 'data', 'type': 'int'}, + ], + } + obj = {'data': 789} + + # No associations, configure fallback to TOPIC_RECORD + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + # Deserialize and verify + deser = AvroDeserializer(client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Should have registered under topic-record_name + registered_schema = client.get_latest_version(_TOPIC + "-MyRecord") + assert registered_schema is not None + + +def test_associated_name_strategy_fallback_none_raises(): + """Test that NONE fallback raises an error when no association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema + schema = { + 'type': 'record', + 'name': 'MyRecord', + 'fields': [ + {'name': 'value', 'type': 'string'}, + ], + } + obj = {'value': 'test'} + + # No associations, configure fallback to NONE + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: "NONE"}, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + ser(obj, ser_ctx) + + assert "No associated subject found" in str(exc_info.value) + + +def test_associated_name_strategy_with_kafka_cluster_id(): + """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema + schema = { + 'type': 'record', + 'name': 'TestRecord', + 'fields': [ + {'name': 'intField', 'type': 'int'}, + ], + } + obj = {'intField': 100} + + # Add an association with specific namespace + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="my-cluster-id", + resource_id="mock-resource-id-4", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cluster-specific-subject", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), + ) + ], + ) + client.create_association(request) + + # Create serializer with matching cluster ID + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {KAFKA_CLUSTER_ID: "my-cluster-id"}, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + # Deserialize and verify + deser = AvroDeserializer(client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Verify the schema was registered with the cluster-specific subject + registered_schema = client.get_latest_version("cluster-specific-subject") + assert registered_schema is not None + + client.delete_associations(resource_id="mock-resource-id-4", cascade_lifecycle=True) + + +def test_associated_name_strategy_caching(): + """Test that results are cached within a strategy instance and serializer works with caching""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema + schema = { + 'type': 'record', + 'name': 'CacheTestRecord', + 'fields': [ + {'name': 'count', 'type': 'int'}, + ], + } + + # Add an association + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="mock-resource-id-5", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cached-subject", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), + ) + ], + ) + client.create_association(request) + + # Create serializer with associated name strategy + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + # First serialization + obj1 = {'count': 1} + obj_bytes1 = ser(obj1, ser_ctx) + + # Verify it was registered under cached-subject + registered_schema = client.get_latest_version("cached-subject") + assert registered_schema is not None + + # Deserialize first message + deser = AvroDeserializer(client) + result1 = deser(obj_bytes1, ser_ctx) + assert obj1 == result1 + + # Delete associations (but serializer should still work due to caching) + client.delete_associations(resource_id="mock-resource-id-5", cascade_lifecycle=True) + + # Second serialization should still work (schema already registered) + obj2 = {'count': 2} + obj_bytes2 = ser(obj2, ser_ctx) + + # Deserialize second message + result2 = deser(obj_bytes2, ser_ctx) + assert obj2 == result2 diff --git a/tests/schema_registry/_sync/test_json.py b/tests/schema_registry/_sync/test_json.py index 7d3d912e1..b94622957 100644 --- a/tests/schema_registry/_sync/test_json.py +++ b/tests/schema_registry/_sync/test_json.py @@ -77,6 +77,7 @@ def test_custom_json_encoder(): mock_schema_registry_client.register_schema_full_response.return_value = RegisteredSchema( schema_id=1, guid=None, schema=Schema(schema_str), subject="topic-name-value", version=1 ) + mock_schema_registry_client.get_associations_by_resource_name.return_value = [] # Use orjson.dumps as the custom encoder serializer = JSONSerializer( @@ -136,6 +137,7 @@ def test_custom_encoder_decoder_chain(): mock_schema_registry_client.register_schema_full_response.return_value = RegisteredSchema( schema_id=1, guid=None, schema=Schema(schema_str), subject="topic-name-value", version=1 ) + mock_schema_registry_client.get_associations_by_resource_name.return_value = [] def custom_encoder(obj): return orjson.dumps(obj, option=orjson.OPT_SORT_KEYS) @@ -176,6 +178,7 @@ def test_custom_encoding_with_complex_data(): mock_schema_registry_client.register_schema_full_response.return_value = RegisteredSchema( schema_id=1, guid=None, schema=Schema(schema_str), subject="topic-name-value", version=1 ) + mock_schema_registry_client.get_associations_by_resource_name.return_value = [] def custom_encoder(obj): return json.dumps(obj, indent=2) diff --git a/tests/schema_registry/_sync/test_json_serdes.py b/tests/schema_registry/_sync/test_json_serdes.py index 43f10b20f..485c7db7f 100644 --- a/tests/schema_registry/_sync/test_json_serdes.py +++ b/tests/schema_registry/_sync/test_json_serdes.py @@ -27,6 +27,15 @@ SchemaRegistryClient, header_schema_id_serializer, ) +from confluent_kafka.schema_registry._sync.serde import ( + FALLBACK_TYPE, + KAFKA_CLUSTER_ID, +) +from confluent_kafka.schema_registry.common.schema_registry_client import ( + AssociationCreateOrUpdateInfo, + AssociationCreateOrUpdateRequest, +) +from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.json_schema import JSONDeserializer, JSONSerializer from confluent_kafka.schema_registry.rules.cel.cel_executor import CelExecutor from confluent_kafka.schema_registry.rules.cel.cel_field_executor import CelFieldExecutor @@ -1449,3 +1458,276 @@ def test_json_deeply_nested_refs(): executor.executor.client = dek_client obj2 = deser(obj_bytes, ser_ctx) assert obj == obj2 + + +_JSON_SCHEMA = json.dumps( + { + "type": "object", + "title": "MyRecord", + "properties": { + "name": {"type": "string"}, + "id": {"type": "integer"}, + }, + } +) +_JSON_OBJ = {"name": "Kafka", "id": 123} + + +def test_json_associated_name_strategy_with_association(): + """Test that AssociatedNameStrategy returns subject from association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="json-resource-id-1", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-custom-subject-value", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(_JSON_OBJ, ser_ctx) + + deser = JSONDeserializer(None, schema_registry_client=client) + obj2 = deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = client.get_latest_version("my-custom-subject-value") + assert registered_schema is not None + + client.delete_associations(resource_id="json-resource-id-1", cascade_lifecycle=True) + + +def test_json_associated_name_strategy_with_key_association(): + """Test that AssociatedNameStrategy returns subject for key""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="json-resource-id-2", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-key-subject", + association_type="key", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.KEY) + obj_bytes = ser(_JSON_OBJ, ser_ctx) + + deser = JSONDeserializer(None, schema_registry_client=client) + obj2 = deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = client.get_latest_version("my-key-subject") + assert registered_schema is not None + + client.delete_associations(resource_id="json-resource-id-2", cascade_lifecycle=True) + + +def test_json_associated_name_strategy_fallback_to_topic(): + """Test fallback to topic_subject_name_strategy when no association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(_JSON_OBJ, ser_ctx) + + deser = JSONDeserializer(None, schema_registry_client=client) + obj2 = deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = client.get_latest_version(_TOPIC + "-value") + assert registered_schema is not None + + +def test_json_associated_name_strategy_fallback_to_record(): + """Test fallback to record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.RECORD}, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(_JSON_OBJ, ser_ctx) + + deser = JSONDeserializer(None, schema_registry_client=client) + obj2 = deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + # JSON record name comes from schema "title" + registered_schema = client.get_latest_version("MyRecord") + assert registered_schema is not None + + +def test_json_associated_name_strategy_fallback_to_topic_record(): + """Test fallback to topic_record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(_JSON_OBJ, ser_ctx) + + deser = JSONDeserializer(None, schema_registry_client=client) + obj2 = deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + # JSON topic-record subject: "topic1-MyRecord" + registered_schema = client.get_latest_version(_TOPIC + "-MyRecord") + assert registered_schema is not None + + +def test_json_associated_name_strategy_fallback_none_raises(): + """Test that NONE fallback raises an error when no association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: "NONE"}, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + ser(_JSON_OBJ, ser_ctx) + + assert "No associated subject found" in str(exc_info.value) + + +def test_json_associated_name_strategy_with_kafka_cluster_id(): + """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="my-cluster-id", + resource_id="json-resource-id-4", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cluster-specific-json-subject", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {KAFKA_CLUSTER_ID: "my-cluster-id"}, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(_JSON_OBJ, ser_ctx) + + deser = JSONDeserializer(None, schema_registry_client=client) + obj2 = deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = client.get_latest_version("cluster-specific-json-subject") + assert registered_schema is not None + + client.delete_associations(resource_id="json-resource-id-4", cascade_lifecycle=True) + + +def test_json_associated_name_strategy_caching(): + """Test that results are cached within a strategy instance and serializer works with caching""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="json-resource-id-5", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="json-cached-subject", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + obj1 = {"name": "Kafka", "id": 1} + obj_bytes1 = ser(obj1, ser_ctx) + + registered_schema = client.get_latest_version("json-cached-subject") + assert registered_schema is not None + + deser = JSONDeserializer(None, schema_registry_client=client) + result1 = deser(obj_bytes1, ser_ctx) + assert obj1 == result1 + + # Delete associations (but serializer should still work due to caching) + client.delete_associations(resource_id="json-resource-id-5", cascade_lifecycle=True) + + obj2 = {"name": "Kafka", "id": 2} + obj_bytes2 = ser(obj2, ser_ctx) + + result2 = deser(obj_bytes2, ser_ctx) + assert obj2 == result2 diff --git a/tests/schema_registry/_sync/test_proto_serdes.py b/tests/schema_registry/_sync/test_proto_serdes.py index ea756a680..806899c5e 100644 --- a/tests/schema_registry/_sync/test_proto_serdes.py +++ b/tests/schema_registry/_sync/test_proto_serdes.py @@ -24,6 +24,15 @@ from confluent_kafka.schema_registry import Metadata, MetadataProperties, Schema, header_schema_id_serializer from confluent_kafka.schema_registry._sync.protobuf import ProtobufDeserializer, ProtobufSerializer from confluent_kafka.schema_registry._sync.schema_registry_client import SchemaRegistryClient +from confluent_kafka.schema_registry._sync.serde import ( + FALLBACK_TYPE, + KAFKA_CLUSTER_ID, +) +from confluent_kafka.schema_registry.common.schema_registry_client import ( + AssociationCreateOrUpdateInfo, + AssociationCreateOrUpdateRequest, +) +from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.protobuf import _schema_to_str from confluent_kafka.schema_registry.rules.cel.cel_executor import CelExecutor from confluent_kafka.schema_registry.rules.cel.cel_field_executor import CelFieldExecutor @@ -615,3 +624,286 @@ def deserialize_with_all_versions(client, ser_ctx, obj_bytes, obj, obj2, obj3): deser = ProtobufDeserializer(newerwidget_pb2.NewerWidget, deser_conf, client) newobj = deser(obj_bytes, ser_ctx) assert obj3.length == newobj.length + + +def test_associated_name_strategy_with_association(): + """Test that AssociatedNameStrategy returns subject from association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=123, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="proto-resource-id-1", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-custom-subject-value", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + deser = ProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = client.get_latest_version("my-custom-subject-value") + assert registered_schema is not None + + client.delete_associations(resource_id="proto-resource-id-1", cascade_lifecycle=True) + + +def test_associated_name_strategy_with_key_association(): + """Test that AssociatedNameStrategy returns subject for key""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author(name='Kafka', id=42) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="proto-resource-id-2", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-key-subject", + association_type="key", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.KEY) + obj_bytes = ser(obj, ser_ctx) + + deser = ProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = client.get_latest_version("my-key-subject") + assert registered_schema is not None + + client.delete_associations(resource_id="proto-resource-id-2", cascade_lifecycle=True) + + +def test_associated_name_strategy_fallback_to_topic(): + """Test fallback to topic_subject_name_strategy when no association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=456, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + deser = ProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = client.get_latest_version(_TOPIC + "-value") + assert registered_schema is not None + + +def test_associated_name_strategy_fallback_to_record(): + """Test fallback to record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=789, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.RECORD}, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + deser = ProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = client.get_latest_version(example_pb2.Author.DESCRIPTOR.full_name) + assert registered_schema is not None + + +def test_associated_name_strategy_fallback_to_topic_record(): + """Test fallback to topic_record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=100, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + deser = ProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = client.get_latest_version(_TOPIC + "-" + example_pb2.Author.DESCRIPTOR.full_name) + assert registered_schema is not None + + +def test_associated_name_strategy_fallback_none_raises(): + """Test that NONE fallback raises an error when no association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author(name='Kafka', id=1) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_TYPE: "NONE"}, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + ser(obj, ser_ctx) + + assert "No associated subject found" in str(exc_info.value) + + +def test_associated_name_strategy_with_kafka_cluster_id(): + """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=100, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="my-cluster-id", + resource_id="proto-resource-id-4", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cluster-specific-proto-subject", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {KAFKA_CLUSTER_ID: "my-cluster-id"}, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + deser = ProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = client.get_latest_version("cluster-specific-proto-subject") + assert registered_schema is not None + + client.delete_associations(resource_id="proto-resource-id-4", cascade_lifecycle=True) + + +def test_associated_name_strategy_caching(): + """Test that results are cached within a strategy instance and serializer works with caching""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="proto-resource-id-5", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="proto-cached-subject", + association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + obj1 = example_pb2.Author(name='Kafka', id=1) + obj_bytes1 = ser(obj1, ser_ctx) + + registered_schema = client.get_latest_version("proto-cached-subject") + assert registered_schema is not None + + deser = ProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + result1 = deser(obj_bytes1, ser_ctx) + assert obj1 == result1 + + # Delete associations (but serializer should still work due to caching) + client.delete_associations(resource_id="proto-resource-id-5", cascade_lifecycle=True) + + obj2 = example_pb2.Author(name='Kafka', id=2) + obj_bytes2 = ser(obj2, ser_ctx) + + result2 = deser(obj_bytes2, ser_ctx) + assert obj2 == result2 diff --git a/tests/schema_registry/conftest.py b/tests/schema_registry/conftest.py index 9a93ae4d9..5e6bf76d5 100644 --- a/tests/schema_registry/conftest.py +++ b/tests/schema_registry/conftest.py @@ -148,6 +148,8 @@ def mock_schema_registry(): respx_mock.get(CONTEXTS_RE).mock(side_effect=get_contexts_callback) + respx_mock.get(ASSOCIATIONS_RESOURCES_RE).mock(side_effect=get_associations_resources_callback) + respx_mock.get(MODE_GLOBAL_RE).mock(side_effect=get_global_mode_callback) respx_mock.put(MODE_GLOBAL_RE).mock(side_effect=put_global_mode_callback) respx_mock.get(MODE_RE).mock(side_effect=get_mode_callback) @@ -192,6 +194,8 @@ def mock_schema_registry(): CONTEXTS_RE = re.compile(r"/contexts(\?.*)?$") +ASSOCIATIONS_RESOURCES_RE = re.compile(r"/associations/resources/(.*)/(.*?)(\?.*)?$") + # constants SCHEMA_ID = 47 VERSION = 3 @@ -268,6 +272,11 @@ def get_contexts_callback(request, route): return Response(200, json=['context1', 'context2']) +def get_associations_resources_callback(request, route): + COUNTER['GET'][request.url.path] += 1 + return Response(200, json=[]) + + def delete_subject_callback(request, route): COUNTER['DELETE'][request.url.path] += 1