From 63bbe97c76d1f8fe86c936430d8770606bbe4063 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Mon, 26 Jan 2026 11:46:34 -0800 Subject: [PATCH 01/20] First cut at AssociatedNameStrategy refactor --- .../schema_registry/_async/avro.py | 88 +++- .../schema_registry/_async/json_schema.py | 82 +++- .../_async/mock_schema_registry_client.py | 165 ++++++- .../schema_registry/_async/protobuf.py | 93 +++- .../_async/schema_registry_client.py | 104 +++++ .../schema_registry/_async/serde.py | 270 +++++++++++- .../schema_registry/_sync/avro.py | 87 +++- .../schema_registry/_sync/json_schema.py | 81 +++- .../_sync/mock_schema_registry_client.py | 161 ++++++- .../schema_registry/_sync/protobuf.py | 91 +++- .../_sync/schema_registry_client.py | 99 +++++ .../schema_registry/_sync/serde.py | 272 +++++++++++- .../common/schema_registry_client.py | 248 +++++++++++ .../schema_registry/common/serde.py | 28 +- .../_async/test_avro_serdes.py | 402 ++++++++++++++++++ .../schema_registry/_sync/test_avro_serdes.py | 402 ++++++++++++++++++ 16 files changed, 2538 insertions(+), 135 deletions(-) diff --git a/src/confluent_kafka/schema_registry/_async/avro.py b/src/confluent_kafka/schema_registry/_async/avro.py index 991e8ec99..a5d0503c9 100644 --- a/src/confluent_kafka/schema_registry/_async/avro.py +++ b/src/confluent_kafka/schema_registry/_async/avro.py @@ -26,7 +26,6 @@ Schema, dual_schema_id_deserializer, prefix_schema_id_serializer, - topic_subject_name_strategy, ) from confluent_kafka.schema_registry.common import asyncinit from confluent_kafka.schema_registry.common.avro import ( @@ -125,14 +124,28 @@ class AsyncAvroSerializer(AsyncBaseSerializer): | | | | | | | Defaults to None. | +-----------------------------------+----------+--------------------------------------------------+ + | ``subject.name.strategy.type`` | str | The type of subject name strategy to use. | + | | | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to TOPIC if neither this nor | + | | | subject.name.strategy is specified. | + +-----------------------------------+----------+--------------------------------------------------+ + | ``subject.name.strategy.conf`` | dict | Configuration dictionary passed to strategies | + | | | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-----------------------------------+----------+--------------------------------------------------+ | ``subject.name.strategy`` | callable | Callable(SerializationContext, str) -> str | | | | | | | | Defines how Schema Registry subject names are | | | | constructed. Standard naming strategies are | | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-----------------------------------+----------+--------------------------------------------------+ | ``schema.id.serializer`` | callable | Callable(bytes, SerializationContext, schema_id) | | | | -> bytes | @@ -224,7 +237,9 @@ class AsyncAvroSerializer(AsyncBaseSerializer): 'use.schema.id': None, 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.serializer': prefix_schema_id_serializer, 'validate.strict': False, 'validate.strict.allow.default': False, @@ -286,12 +301,11 @@ async def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), + subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), + subject_name_strategy=conf_copy.pop('subject.name.strategy'), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_serializer = cast( Callable[[bytes, Optional[SerializationContext], Any], bytes], conf_copy.pop('schema.id.serializer') @@ -373,7 +387,11 @@ async def __serialize(self, obj: object, ctx: Optional[SerializationContext] = N if obj is None: return None - subject = self._subject_name_func(ctx, self._schema_name) + subject = ( + await self._subject_name_func(ctx, self._schema_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, self._schema_name) + ) latest_schema = await self._get_reader_schema(subject) if subject else None if latest_schema is not None: self._schema_id = SchemaId(AVRO_TYPE, latest_schema.schema_id, latest_schema.guid) @@ -481,14 +499,28 @@ class AsyncAvroDeserializer(AsyncBaseDeserializer): | | | | | | | Defaults to None. | +-----------------------------+----------+--------------------------------------------------+ + | | | The type of subject name strategy to use. | + |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to TOPIC if neither this nor | + | | | subject.name.strategy is specified. | + +-----------------------------+----------+--------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + |``subject.name.strategy.conf``| dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-----------------------------+----------+--------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | | | | constructed. Standard naming strategies are | | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-----------------------------+----------+--------------------------------------------------+ | | | Callable(bytes, SerializationContext, schema_id) | | | | -> io.BytesIO | @@ -533,7 +565,9 @@ class AsyncAvroDeserializer(AsyncBaseDeserializer): _default_conf = { 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.deserializer': dual_schema_id_deserializer, } @@ -575,12 +609,11 @@ async def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), + subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), + subject_name_strategy=conf_copy.pop('subject.name.strategy'), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_deserializer = cast( Callable[[bytes, Optional[SerializationContext], Any], io.BytesIO], conf_copy.pop('schema.id.deserializer') @@ -649,7 +682,15 @@ async def __deserialize( "Schema Registry serializer".format(len(data)) ) - subject = self._subject_name_func(ctx, None) if ctx else None + subject = ( + ( + await self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, None) + ) + if ctx + else None + ) latest_schema = None if subject is not None: latest_schema = await self._get_reader_schema(subject) @@ -661,7 +702,14 @@ async def __deserialize( writer_schema = await self._get_parsed_schema(writer_schema_raw) if subject is None: subject = ( - self._subject_name_func(ctx, writer_schema.get("name")) if ctx else None # type: ignore[union-attr] + ( + await self._subject_name_func( + ctx, writer_schema.get("name"), self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, writer_schema.get("name")) + ) + if ctx + else None ) if subject is not None: latest_schema = await self._get_reader_schema(subject) diff --git a/src/confluent_kafka/schema_registry/_async/json_schema.py b/src/confluent_kafka/schema_registry/_async/json_schema.py index 76e5d5e97..bff82f812 100644 --- a/src/confluent_kafka/schema_registry/_async/json_schema.py +++ b/src/confluent_kafka/schema_registry/_async/json_schema.py @@ -32,7 +32,6 @@ Schema, dual_schema_id_deserializer, prefix_schema_id_serializer, - topic_subject_name_strategy, ) from confluent_kafka.schema_registry.common import asyncinit from confluent_kafka.schema_registry.common.json_schema import ( @@ -134,14 +133,28 @@ class AsyncJSONSerializer(AsyncBaseSerializer): | | | | | | | Defaults to None. | +-----------------------------+----------+----------------------------------------------------+ + | | | The type of subject name strategy to use. | + |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to TOPIC if neither this nor | + | | | subject.name.strategy is specified. | + +-----------------------------+----------+----------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + |``subject.name.strategy.conf``| dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-----------------------------+----------+----------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | | | | constructed. Standard naming strategies are | | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-----------------------------+----------+----------------------------------------------------+ | | | Whether to validate the payload against the | | ``validate`` | bool | the given schema. | @@ -226,7 +239,9 @@ class AsyncJSONSerializer(AsyncBaseSerializer): 'use.schema.id': None, 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.serializer': prefix_schema_id_serializer, 'validate': True, } @@ -292,12 +307,11 @@ async def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), + subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), + subject_name_strategy=conf_copy.pop('subject.name.strategy'), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_serializer = cast( Callable[[bytes, Optional[SerializationContext], Any], bytes], conf_copy.pop('schema.id.serializer') @@ -354,7 +368,11 @@ async def __serialize(self, obj: object, ctx: Optional[SerializationContext] = N if obj is None: return None - subject = self._subject_name_func(ctx, self._schema_name) + subject = ( + await self._subject_name_func(ctx, self._schema_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, self._schema_name) + ) latest_schema = await self._get_reader_schema(subject) if subject else None if latest_schema is not None: self._schema_id = SchemaId(JSON_TYPE, latest_schema.schema_id, latest_schema.guid) @@ -481,14 +499,28 @@ class AsyncJSONDeserializer(AsyncBaseDeserializer): | | | | | | | Defaults to None. | +-----------------------------+----------+----------------------------------------------------+ + | | | The type of subject name strategy to use. | + |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to TOPIC if neither this nor | + | | | subject.name.strategy is specified. | + +-----------------------------+----------+----------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + |``subject.name.strategy.conf``| dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-----------------------------+----------+----------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | | | | constructed. Standard naming strategies are | | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-----------------------------+----------+----------------------------------------------------+ | | | Whether to validate the payload against the | | ``validate`` | bool | the given schema. | @@ -531,7 +563,9 @@ class AsyncJSONDeserializer(AsyncBaseDeserializer): _default_conf = { 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.deserializer': dual_schema_id_deserializer, 'validate': True, } @@ -584,12 +618,11 @@ async def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), + subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), + subject_name_strategy=conf_copy.pop('subject.name.strategy'), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_deserializer = cast( Callable[[bytes, Optional[SerializationContext], Any], io.BytesIO], conf_copy.pop('schema.id.deserializer') @@ -647,7 +680,11 @@ async def __deserialize(self, data: Optional[bytes], ctx: Optional[Serialization if data is None: return None - subject = self._subject_name_func(ctx, None) + subject = ( + await self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, None) + ) latest_schema = None if subject is not None and self._registry is not None: latest_schema = await self._get_reader_schema(subject) @@ -660,7 +697,12 @@ async def __deserialize(self, data: Optional[bytes], ctx: Optional[Serialization writer_schema_raw = await self._get_writer_schema(schema_id, subject) writer_schema, writer_ref_registry = await self._get_parsed_schema(writer_schema_raw) if subject is None and isinstance(writer_schema, dict): - subject = self._subject_name_func(ctx, writer_schema.get("title")) + subject = ( + await self._subject_name_func( + ctx, writer_schema.get("title"), self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, writer_schema.get("title")) + ) if subject is not None: latest_schema = await self._get_reader_schema(subject) else: diff --git a/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py b/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py index f782822be..e1c1bdd09 100644 --- a/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py @@ -20,7 +20,15 @@ from threading import Lock from typing import Dict, List, Literal, Optional, Union -from ..common.schema_registry_client import RegisteredSchema, Schema, ServerConfig +from ..common.schema_registry_client import ( + Association, + AssociationCreateOrUpdateRequest, + AssociationResponse, + AssociationInfo, + RegisteredSchema, + Schema, + ServerConfig, +) from ..error import SchemaRegistryError from .schema_registry_client import AsyncSchemaRegistryClient @@ -142,11 +150,120 @@ def clear(self): self.subject_schemas.clear() +class _AssociationStore(object): + + def __init__(self): + self.lock = Lock() + # Key: resource_id -> List[Association] + self.associations_by_resource_id: Dict[str, List[Association]] = defaultdict(list) + # Key: (resource_namespace, resource_name) -> resource_id + self.resource_id_index: Dict[tuple, str] = {} + + def create_association( + self, + request: AssociationCreateOrUpdateRequest + ) -> AssociationResponse: + with self.lock: + resource_id = request.resource_id + resource_name = request.resource_name + resource_namespace = request.resource_namespace + resource_type = request.resource_type + + # Index resource_id by (namespace, name) + if resource_name and resource_namespace: + self.resource_id_index[(resource_namespace, resource_name)] = resource_id + + created_associations = [] + if request.associations: + for assoc_info in request.associations: + association = Association( + subject=assoc_info.subject, + guid=None, + resource_name=resource_name, + resource_namespace=resource_namespace, + resource_id=resource_id, + resource_type=resource_type, + association_type=assoc_info.association_type, + frozen=assoc_info.frozen if assoc_info.frozen is not None else False, + ) + self.associations_by_resource_id[resource_id].append(association) + created_associations.append(AssociationInfo( + subject=assoc_info.subject, + association_type=assoc_info.association_type, + lifecycle=assoc_info.lifecycle, + frozen=assoc_info.frozen if assoc_info.frozen is not None else False, + schema=assoc_info.schema, + )) + + return AssociationResponse( + resource_name=resource_name, + resource_namespace=resource_namespace, + resource_id=resource_id, + resource_type=resource_type, + associations=created_associations, + ) + + def delete_associations( + self, + resource_id: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + ) -> None: + with self.lock: + if resource_id not in self.associations_by_resource_id: + return + + if association_types is None and resource_type is None: + # Delete all associations for this resource + del self.associations_by_resource_id[resource_id] + else: + # Filter and keep only non-matching associations + remaining = [] + for assoc in self.associations_by_resource_id[resource_id]: + keep = False + if resource_type is not None and assoc.resource_type != resource_type: + keep = True + if association_types is not None and assoc.association_type not in association_types: + keep = True + if keep: + remaining.append(assoc) + self.associations_by_resource_id[resource_id] = remaining + + def get_associations_by_resource_name( + self, + resource_name: str, + resource_namespace: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None + ) -> List[Association]: + with self.lock: + result = [] + for resource_id, associations in self.associations_by_resource_id.items(): + for assoc in associations: + # Check if namespace matches (or is wildcard) + if resource_namespace != "-" and assoc.resource_namespace != resource_namespace: + continue + if assoc.resource_name != resource_name: + continue + if resource_type is not None and assoc.resource_type != resource_type: + continue + if association_types is not None and assoc.association_type not in association_types: + continue + result.append(assoc) + return result + + def clear(self): + with self.lock: + self.associations_by_resource_id.clear() + self.resource_id_index.clear() + + class AsyncMockSchemaRegistryClient(AsyncSchemaRegistryClient): def __init__(self, conf: dict): super().__init__(conf) self._store = _SchemaStore() + self._association_store = _AssociationStore() async def register_schema(self, subject_name: str, schema: 'Schema', normalize_schemas: bool = False) -> int: registered_schema = await self.register_schema_full_response( @@ -300,3 +417,49 @@ async def set_config( async def get_config(self, subject_name: Optional[str] = None) -> 'ServerConfig': # noqa F821 return None # type: ignore[return-value] + + async def get_associations_by_resource_name( + self, + resource_name: str, + resource_namespace: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + offset: int = 0, + limit: int = -1 + ) -> List['Association']: + return self._association_store.get_associations_by_resource_name( + resource_name, resource_namespace, resource_type, association_types + ) + + async def create_association( + self, + request: 'AssociationCreateOrUpdateRequest' + ) -> 'AssociationResponse': + """ + Creates an association between a subject and a resource. + + Args: + request (AssociationCreateOrUpdateRequest): The association create or update request. + + Returns: + AssociationResponse: The response containing the created associations. + """ + return self._association_store.create_association(request) + + async def delete_associations( + self, + resource_id: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + cascade_lifecycle: bool = False + ) -> None: + """ + Deletes associations for a resource. + + Args: + resource_id (str): The resource identifier. + resource_type (str, optional): The type of resource (e.g., "topic"). + association_types (List[str], optional): The types of associations to delete. + cascade_lifecycle (bool): Whether to cascade the lifecycle policy to dependent schemas. + """ + self._association_store.delete_associations(resource_id, resource_type, association_types) diff --git a/src/confluent_kafka/schema_registry/_async/protobuf.py b/src/confluent_kafka/schema_registry/_async/protobuf.py index 33c316e81..108f857fc 100644 --- a/src/confluent_kafka/schema_registry/_async/protobuf.py +++ b/src/confluent_kafka/schema_registry/_async/protobuf.py @@ -31,7 +31,6 @@ dual_schema_id_deserializer, prefix_schema_id_serializer, reference_subject_name_strategy, - topic_subject_name_strategy, ) from confluent_kafka.schema_registry.common import asyncinit from confluent_kafka.schema_registry.common.protobuf import ( @@ -151,14 +150,28 @@ class AsyncProtobufSerializer(AsyncBaseSerializer): | | | | | | | Defaults to True. | +-------------------------------------+----------+------------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to TOPIC if neither this nor | + | | | subject.name.strategy is specified. | + +-------------------------------------+----------+------------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | | | | constructed. Standard naming strategies are | | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | @@ -229,7 +242,9 @@ class AsyncProtobufSerializer(AsyncBaseSerializer): 'use.latest.version': False, 'use.latest.with.metadata': None, 'skip.known.types': True, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'reference.subject.name.strategy': reference_subject_name_strategy, 'schema.id.serializer': prefix_schema_id_serializer, 'use.deprecated.format': False, @@ -281,19 +296,18 @@ async def __init_impl( if self._use_deprecated_format: raise ValueError("use.deprecated.format is no longer supported") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), + subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), + subject_name_strategy=conf_copy.pop('subject.name.strategy'), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._ref_reference_subject_func = cast( Callable[[Optional[SerializationContext], Any], Optional[str]], conf_copy.pop('reference.subject.name.strategy'), ) if not callable(self._ref_reference_subject_func): - raise ValueError("subject.name.strategy must be callable") + raise ValueError("reference.subject.name.strategy must be callable") self._schema_id_serializer = cast( Callable[[bytes, Optional[SerializationContext], Any], bytes], conf_copy.pop('schema.id.serializer') @@ -418,7 +432,16 @@ async def __serialize(self, message: Message, ctx: Optional[SerializationContext if not isinstance(message, self._msg_class): raise ValueError("message must be of type {} not {}".format(self._msg_class, type(message))) - subject = self._subject_name_func(ctx, message.DESCRIPTOR.full_name) if ctx else None + subject = ( + ( + await self._subject_name_func( + ctx, message.DESCRIPTOR.full_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, message.DESCRIPTOR.full_name) + ) + if ctx + else None + ) latest_schema = None if subject is not None: latest_schema = await self._get_reader_schema(subject, fmt='serialized') @@ -513,14 +536,28 @@ class AsyncProtobufDeserializer(AsyncBaseDeserializer): | | | | | | | Defaults to None. | +-------------------------------------+----------+------------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to TOPIC if neither this nor | + | | | subject.name.strategy is specified. | + +-------------------------------------+----------+------------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka. schema_registry | - | | | namespace . | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(bytes, SerializationContext, schema_id) | | | | -> io.BytesIO | @@ -539,7 +576,9 @@ class AsyncProtobufDeserializer(AsyncBaseDeserializer): _default_conf = { 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.deserializer': dual_schema_id_deserializer, 'use.deprecated.format': False, } @@ -571,12 +610,11 @@ async def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), + subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), + subject_name_strategy=conf_copy.pop('subject.name.strategy'), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_deserializer = cast( Callable[[bytes, Optional[SerializationContext], Any], io.BytesIO], conf_copy.pop('schema.id.deserializer') @@ -626,7 +664,11 @@ async def __deserialize(self, data: Optional[bytes], ctx: Optional[Serialization if data is None: return None - subject = self._subject_name_func(ctx, None) + subject = ( + await self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, None) + ) latest_schema = None if subject is not None and self._registry is not None: latest_schema = await self._get_reader_schema(subject, fmt='serialized') @@ -641,7 +683,12 @@ async def __deserialize(self, data: Optional[bytes], ctx: Optional[Serialization writer_schema = pool.FindFileByName(fd_proto.name) writer_desc = self._get_message_desc(pool, writer_schema, msg_index if msg_index is not None else []) if subject is None: - subject = self._subject_name_func(ctx, writer_desc.full_name) + subject = ( + await self._subject_name_func( + ctx, writer_desc.full_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, writer_desc.full_name) + ) if subject is not None: latest_schema = await self._get_reader_schema(subject, fmt='serialized') else: diff --git a/src/confluent_kafka/schema_registry/_async/schema_registry_client.py b/src/confluent_kafka/schema_registry/_async/schema_registry_client.py index 768e00940..aa3ffc990 100644 --- a/src/confluent_kafka/schema_registry/_async/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_async/schema_registry_client.py @@ -42,6 +42,9 @@ _StaticOAuthBearerFieldProviderBuilder, ) from confluent_kafka.schema_registry.common.schema_registry_client import ( + Association, + AssociationCreateOrUpdateRequest, + AssociationResponse, RegisteredSchema, Schema, SchemaVersion, @@ -1579,6 +1582,107 @@ async def clear_caches(self): self._latest_with_metadata_cache.clear() self._cache.clear() + async def get_associations_by_resource_name( + self, + resource_name: str, + resource_namespace: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + offset: int = 0, + limit: int = -1 + ) -> List['Association']: + """ + Retrieves associations for a given resource name and namespace. + + Args: + resource_name (str): The name of the resource (e.g., topic name). + resource_namespace (str): The namespace of the resource (e.g., kafka cluster ID). + Use "-" as a wildcard. + resource_type (str, optional): The type of resource (e.g., "topic"). + association_types (List[str], optional): The types of associations to filter by + (e.g., ["key", "value"]). + offset (int): Pagination offset for results. + limit (int): Pagination size for results. Ignored if negative. + + Returns: + List[Association]: List of associations matching the criteria. + + Raises: + SchemaRegistryError: if the request was unsuccessful. + """ + query: Dict[str, Any] = {} + if resource_type is not None: + query['resourceType'] = resource_type + if association_types is not None: + query['associationType'] = association_types + if offset > 0: + query['offset'] = offset + if limit >= 1: + query['limit'] = limit + + response = await self._rest_client.get( + 'associations/resources/{}/{}'.format( + _urlencode(resource_namespace), + _urlencode(resource_name) + ), + query + ) + + return [Association.from_dict(a) for a in response] + + async def create_association( + self, + request: 'AssociationCreateOrUpdateRequest' + ) -> 'AssociationResponse': + """ + Creates an association between a subject and a resource. + + Args: + request (AssociationCreateOrUpdateRequest): The association create or update request. + + Returns: + AssociationResponse: The response containing the created associations. + + Raises: + SchemaRegistryError: if the request was unsuccessful. + """ + response = await self._rest_client.post('associations', body=request.to_dict()) + return AssociationResponse.from_dict(response) + + async def delete_associations( + self, + resource_id: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + cascade_lifecycle: bool = False + ) -> None: + """ + Deletes associations for a resource. + + Args: + resource_id (str): The resource identifier. + resource_type (str, optional): The type of resource (e.g., "topic"). + association_types (List[str], optional): The types of associations to delete + (e.g., ["key", "value"]). If not specified, all associations are deleted. + cascade_lifecycle (bool): Whether to cascade the lifecycle policy to dependent schemas. + + Raises: + SchemaRegistryError: if the request was unsuccessful. + """ + query: Dict[str, Any] = {'cascadeLifecycle': str(cascade_lifecycle).lower()} + if resource_type is not None: + query['resourceType'] = resource_type + if association_types is not None: + query['associationType'] = association_types + + await self._rest_client.delete( + 'associations/resources/{}?{}'.format( + _urlencode(resource_id), + '&'.join(f"{k}={v}" if not isinstance(v, list) + else '&'.join(f"{k}={item}" for item in v) for k, v in query.items()) + ) + ) + @staticmethod def new_client(conf: dict) -> 'AsyncSchemaRegistryClient': from .mock_schema_registry_client import AsyncMockSchemaRegistryClient diff --git a/src/confluent_kafka/schema_registry/_async/serde.py b/src/confluent_kafka/schema_registry/_async/serde.py index 143855a7b..54e46ab98 100644 --- a/src/confluent_kafka/schema_registry/_async/serde.py +++ b/src/confluent_kafka/schema_registry/_async/serde.py @@ -16,12 +16,21 @@ # limitations under the License. # +import asyncio as _locks import logging -from typing import Any, Callable, Dict, List, Optional, Set +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union -from confluent_kafka.schema_registry import RegisteredSchema +from cachetools import LRUCache + +from confluent_kafka.schema_registry import ( + RegisteredSchema, + topic_subject_name_strategy, + SchemaRegistryClient, +) +from confluent_kafka.schema_registry.error import SchemaRegistryError from confluent_kafka.schema_registry.common.schema_registry_client import RulePhase from confluent_kafka.schema_registry.common.serde import ( + STRATEGY_TYPE_MAP, ErrorAction, FieldTransformer, Migration, @@ -31,19 +40,199 @@ RuleContext, RuleError, SchemaId, + SubjectNameStrategyType, ) from confluent_kafka.schema_registry.schema_registry_client import Rule, RuleKind, RuleMode, RuleSet, Schema -from confluent_kafka.serialization import Deserializer, SerializationContext, SerializationError, Serializer +from confluent_kafka.serialization import (Deserializer, MessageField, + SerializationContext, SerializationError, Serializer) __all__ = [ + 'AsyncAssociatedNameStrategy', 'AsyncBaseSerde', 'AsyncBaseSerializer', 'AsyncBaseDeserializer', + 'KAFKA_CLUSTER_ID', + 'FALLBACK_SUBJECT_NAME_STRATEGY_TYPE', ] log = logging.getLogger(__name__) +KAFKA_CLUSTER_ID = "kafka.cluster.id" +NAMESPACE_WILDCARD = "-" +FALLBACK_SUBJECT_NAME_STRATEGY_TYPE = "fallback.subject.name.strategy.type" +DEFAULT_CACHE_CAPACITY = 1000 + + +class AsyncAssociatedNameStrategy: + """ + A subject name strategy that retrieves the associated subject name from schema registry + by querying associations for the topic. + + This class encapsulates a cache for subject name lookups to avoid repeated API calls. + + Args: + cache_capacity (int): Maximum number of entries to cache. Defaults to 1000. + """ + + def __init__(self, cache_capacity: int = DEFAULT_CACHE_CAPACITY): + self._cache: LRUCache = LRUCache(maxsize=cache_capacity) + self._lock: _locks.Lock = _locks.Lock() + + def _get_cache_key( + self, topic: str, is_key: bool, record_name: Optional[str] + ) -> Tuple[str, bool, Optional[str]]: + """Create a cache key from topic, is_key, and record_name.""" + return (topic, is_key, record_name) + + async def _load_subject_name( + self, + topic: str, + is_key: bool, + record_name: Optional[str], + ctx: SerializationContext, + schema_registry_client: SchemaRegistryClient, + conf: Optional[dict] + ) -> Optional[str]: + """Load the subject name from schema registry (not cached).""" + # Determine resource namespace from config + kafka_cluster_id = None + fallback_strategy = SubjectNameStrategyType.TOPIC # default fallback + + if conf is not None: + kafka_cluster_id = conf.get(KAFKA_CLUSTER_ID) + fallback_config = conf.get(FALLBACK_SUBJECT_NAME_STRATEGY_TYPE) + if fallback_config is not None: + if isinstance(fallback_config, SubjectNameStrategyType): + fallback_strategy = fallback_config + else: + try: + fallback_strategy = SubjectNameStrategyType(str(fallback_config).upper()) + except ValueError: + valid_fallbacks = [e.value for e in SubjectNameStrategyType + if e != SubjectNameStrategyType.ASSOCIATED] + raise ValueError( + f"Invalid value for {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE}: {fallback_config}. " + f"Valid values are: {', '.join(valid_fallbacks)}" + ) + + resource_namespace = kafka_cluster_id if kafka_cluster_id is not None else NAMESPACE_WILDCARD + + # Determine association type based on whether this is key or value + association_type = "key" if is_key else "value" + + # Query schema registry for associations + try: + associations = await schema_registry_client.get_associations_by_resource_name( + resource_name=topic, + resource_namespace=resource_namespace, + resource_type="topic", + association_types=[association_type], + offset=0, + limit=-1 + ) + except SchemaRegistryError as e: + if e.http_status_code == 404: + return STRATEGY_TYPE_MAP[fallback_strategy](ctx, record_name) + else: + raise + + if len(associations) > 1: + raise SerializationError(f"Multiple associated subjects found for topic {topic}") + elif len(associations) == 1: + return associations[0].subject + else: + # No associations found, use fallback strategy + if fallback_strategy == SubjectNameStrategyType.NONE: + raise SerializationError(f"No associated subject found for topic {topic}") + elif fallback_strategy == SubjectNameStrategyType.ASSOCIATED: + raise ValueError( + f"Invalid value for {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE}: {fallback_strategy.value}. " + f"ASSOCIATED cannot be used as a fallback strategy." + ) + + return STRATEGY_TYPE_MAP[fallback_strategy](ctx, record_name) + + async def __call__( + self, + ctx: Optional[SerializationContext], + record_name: Optional[str], + schema_registry_client: SchemaRegistryClient, + conf: Optional[dict] = None + ) -> Optional[str]: + """ + Retrieves the associated subject name from schema registry by querying + associations for the topic. + + The topic is passed as the resource name to schema registry. If there is a + configuration property named "kafka.cluster.id", then its value will be passed + as the resource namespace; otherwise the value "-" will be passed as the + resource namespace. + + If more than one subject is returned from the query, a SerializationError + will be raised. If no subjects are returned from the query, then the behavior + will fall back to topic_subject_name_strategy, unless the configuration property + "fallback.subject.name.strategy.type" is set to "RECORD", "TOPIC_RECORD", or "NONE". + + Results are cached using an LRU cache to avoid repeated API calls. + + Args: + ctx (SerializationContext): Metadata pertaining to the serialization + operation. **Required** - must contain topic and field information. + + record_name (Optional[str]): Record name (used for fallback strategies). + + schema_registry_client (SchemaRegistryClient): SchemaRegistryClient instance. + + conf (Optional[dict]): Configuration dictionary. Supports: + - "kafka.cluster.id": Kafka cluster ID to use as resource namespace. + - "fallback.subject.name.strategy.type": Fallback strategy when no + associations are found. One of "TOPIC", "RECORD", "TOPIC_RECORD", or "NONE". + Defaults to "TOPIC". + + Returns: + Optional[str]: The subject name from the association, or from the fallback strategy. + + Raises: + SerializationError: If multiple associated subjects are found for the topic, + or if no subjects are found and fallback is set to "NONE". + ValueError: If ctx is None. + """ + if ctx is None: + raise ValueError( + "SerializationContext is required for AsyncAssociatedNameStrategy. " + "Either provide a SerializationContext or use a different strategy." + ) + + topic = ctx.topic + if topic is None: + return None + + is_key = ctx.field == MessageField.KEY + cache_key = self._get_cache_key(topic, is_key, record_name) + + # Check cache first + async with self._lock: + cached_result = self._cache.get(cache_key) + if cached_result is not None: + return cached_result + + # Not in cache, load from schema registry + result = await self._load_subject_name(topic, is_key, record_name, ctx, schema_registry_client, conf) + + # Cache the result + if result is not None: + async with self._lock: + self._cache[cache_key] = result + + return result + + async def clear_cache(self) -> None: + """Clear the association subject name cache.""" + async with self._lock: + self._cache.clear() + + class AsyncBaseSerde(object): __slots__ = [ '_use_schema_id', @@ -51,6 +240,8 @@ class AsyncBaseSerde(object): '_use_latest_with_metadata', '_registry', '_rule_registry', + '_strategy_accepts_client', + '_subject_name_conf', '_subject_name_func', '_field_transformer', ] @@ -60,9 +251,82 @@ class AsyncBaseSerde(object): _use_latest_with_metadata: Optional[Dict[str, str]] _registry: Any # AsyncSchemaRegistryClient _rule_registry: Any # RuleRegistry + _subject_name_conf: Optional[dict] _subject_name_func: Callable[[Optional['SerializationContext'], Optional[str]], Optional[str]] _field_transformer: Optional[FieldTransformer] + def configure_subject_name_strategy( + self, + subject_name_strategy_type: Optional[Union[SubjectNameStrategyType, str]] = None, + subject_name_strategy_conf: Optional[dict] = None, + subject_name_strategy: Optional[Callable] = None, + ) -> None: + """ + Configure the subject name strategy for this serde. + + This method supports both the legacy callable approach and the new type-based approach. + If both `subject_name_strategy` (as a callable) and `subject_name_strategy_type` are + provided, the callable takes precedence. + + Args: + subject_name_strategy: A callable that implements the subject name strategy. + Signature: (SerializationContext, str) -> str or + (SerializationContext, str, SchemaRegistryClient, dict) -> str + + subject_name_strategy_type: The type of subject name strategy to use. + Can be a SubjectNameStrategyType enum value or a string + ("TOPIC", "RECORD", "TOPIC_RECORD", "ASSOCIATED"). + + subject_name_strategy_conf: Configuration dictionary passed to strategies + that accept extra parameters (like ASSOCIATED). + + Raises: + ValueError: If the strategy is not callable or the type is invalid. + """ + self._subject_name_conf = subject_name_strategy_conf + + # If a callable is provided, use it directly (backward compatible) + if subject_name_strategy is not None: + if not callable(subject_name_strategy): + raise ValueError("subject.name.strategy must be callable") + self._subject_name_func = subject_name_strategy + self._strategy_accepts_client = False + return + + # If a type is provided, resolve it to a callable + if subject_name_strategy_type is not None: + # Convert string to enum if needed + if isinstance(subject_name_strategy_type, str): + try: + subject_name_strategy_type = SubjectNameStrategyType(subject_name_strategy_type.upper()) + except ValueError: + raise ValueError( + f"Invalid subject.name.strategy.type: {subject_name_strategy_type}. " + f"Valid values are: {[e.value for e in SubjectNameStrategyType]}" + ) + + # Handle ASSOCIATED specially since it needs schema_registry_client + if subject_name_strategy_type == SubjectNameStrategyType.ASSOCIATED: + self._subject_name_func = AsyncAssociatedNameStrategy() + self._strategy_accepts_client = True + elif subject_name_strategy_type == SubjectNameStrategyType.NONE: + raise ValueError( + f"Invalid subject.name.strategy.type: {subject_name_strategy_type}. " + f"NONE cannot be used as a subject name strategy." + ) + elif subject_name_strategy_type in STRATEGY_TYPE_MAP: + self._subject_name_func = STRATEGY_TYPE_MAP[subject_name_strategy_type] + self._strategy_accepts_client = False + else: + raise ValueError( + f"Unknown subject.name.strategy.type: {subject_name_strategy_type}" + ) + return + + # Default to topic_subject_name_strategy + self._subject_name_func = topic_subject_name_strategy + self._strategy_accepts_client = False + async def _get_reader_schema(self, subject: str, fmt: Optional[str] = None) -> Optional[RegisteredSchema]: if self._use_schema_id is not None: schema = await self._registry.get_schema(self._use_schema_id, subject, fmt) diff --git a/src/confluent_kafka/schema_registry/_sync/avro.py b/src/confluent_kafka/schema_registry/_sync/avro.py index 289367265..354d2bf8a 100644 --- a/src/confluent_kafka/schema_registry/_sync/avro.py +++ b/src/confluent_kafka/schema_registry/_sync/avro.py @@ -26,7 +26,6 @@ SchemaRegistryClient, dual_schema_id_deserializer, prefix_schema_id_serializer, - topic_subject_name_strategy, ) from confluent_kafka.schema_registry.common.avro import ( AVRO_TYPE, @@ -121,14 +120,28 @@ class AvroSerializer(BaseSerializer): | | | | | | | Defaults to None. | +-----------------------------------+----------+--------------------------------------------------+ + | ``subject.name.strategy.type`` | str | The type of subject name strategy to use. | + | | | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to TOPIC if neither this nor | + | | | subject.name.strategy is specified. | + +-----------------------------------+----------+--------------------------------------------------+ + | ``subject.name.strategy.conf`` | dict | Configuration dictionary passed to strategies | + | | | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-----------------------------------+----------+--------------------------------------------------+ | ``subject.name.strategy`` | callable | Callable(SerializationContext, str) -> str | | | | | | | | Defines how Schema Registry subject names are | | | | constructed. Standard naming strategies are | | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-----------------------------------+----------+--------------------------------------------------+ | ``schema.id.serializer`` | callable | Callable(bytes, SerializationContext, schema_id) | | | | -> bytes | @@ -220,7 +233,9 @@ class AvroSerializer(BaseSerializer): 'use.schema.id': None, 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.serializer': prefix_schema_id_serializer, 'validate.strict': False, 'validate.strict.allow.default': False, @@ -282,12 +297,11 @@ def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), + subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), + subject_name_strategy=conf_copy.pop('subject.name.strategy'), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_serializer = cast( Callable[[bytes, Optional[SerializationContext], Any], bytes], conf_copy.pop('schema.id.serializer') @@ -369,7 +383,11 @@ def __serialize(self, obj: object, ctx: Optional[SerializationContext] = None) - if obj is None: return None - subject = self._subject_name_func(ctx, self._schema_name) + subject = ( + self._subject_name_func(ctx, self._schema_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, self._schema_name) + ) latest_schema = self._get_reader_schema(subject) if subject else None if latest_schema is not None: self._schema_id = SchemaId(AVRO_TYPE, latest_schema.schema_id, latest_schema.guid) @@ -476,14 +494,28 @@ class AvroDeserializer(BaseDeserializer): | | | | | | | Defaults to None. | +-----------------------------+----------+--------------------------------------------------+ + | | | The type of subject name strategy to use. | + |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to TOPIC if neither this nor | + | | | subject.name.strategy is specified. | + +-----------------------------+----------+--------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + |``subject.name.strategy.conf``| dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-----------------------------+----------+--------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | | | | constructed. Standard naming strategies are | | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-----------------------------+----------+--------------------------------------------------+ | | | Callable(bytes, SerializationContext, schema_id) | | | | -> io.BytesIO | @@ -528,7 +560,9 @@ class AvroDeserializer(BaseDeserializer): _default_conf = { 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.deserializer': dual_schema_id_deserializer, } @@ -570,12 +604,11 @@ def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), + subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), + subject_name_strategy=conf_copy.pop('subject.name.strategy'), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_deserializer = cast( Callable[[bytes, Optional[SerializationContext], Any], io.BytesIO], conf_copy.pop('schema.id.deserializer') @@ -642,7 +675,15 @@ def __deserialize( "Schema Registry serializer".format(len(data)) ) - subject = self._subject_name_func(ctx, None) if ctx else None + subject = ( + ( + self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, None) + ) + if ctx + else None + ) latest_schema = None if subject is not None: latest_schema = self._get_reader_schema(subject) @@ -654,7 +695,13 @@ def __deserialize( writer_schema = self._get_parsed_schema(writer_schema_raw) if subject is None: subject = ( - self._subject_name_func(ctx, writer_schema.get("name")) if ctx else None # type: ignore[union-attr] + ( + self._subject_name_func(ctx, writer_schema.get("name"), self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, writer_schema.get("name")) + ) + if ctx + else None ) if subject is not None: latest_schema = self._get_reader_schema(subject) diff --git a/src/confluent_kafka/schema_registry/_sync/json_schema.py b/src/confluent_kafka/schema_registry/_sync/json_schema.py index bc6fd4d4a..b331b844c 100644 --- a/src/confluent_kafka/schema_registry/_sync/json_schema.py +++ b/src/confluent_kafka/schema_registry/_sync/json_schema.py @@ -32,7 +32,6 @@ SchemaRegistryClient, dual_schema_id_deserializer, prefix_schema_id_serializer, - topic_subject_name_strategy, ) from confluent_kafka.schema_registry.common.json_schema import ( DEFAULT_SPEC, @@ -132,14 +131,28 @@ class JSONSerializer(BaseSerializer): | | | | | | | Defaults to None. | +-----------------------------+----------+----------------------------------------------------+ + | | | The type of subject name strategy to use. | + |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to TOPIC if neither this nor | + | | | subject.name.strategy is specified. | + +-----------------------------+----------+----------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + |``subject.name.strategy.conf``| dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-----------------------------+----------+----------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | | | | constructed. Standard naming strategies are | | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-----------------------------+----------+----------------------------------------------------+ | | | Whether to validate the payload against the | | ``validate`` | bool | the given schema. | @@ -224,7 +237,9 @@ class JSONSerializer(BaseSerializer): 'use.schema.id': None, 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.serializer': prefix_schema_id_serializer, 'validate': True, } @@ -290,12 +305,11 @@ def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), + subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), + subject_name_strategy=conf_copy.pop('subject.name.strategy'), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_serializer = cast( Callable[[bytes, Optional[SerializationContext], Any], bytes], conf_copy.pop('schema.id.serializer') @@ -352,7 +366,11 @@ def __serialize(self, obj: object, ctx: Optional[SerializationContext] = None) - if obj is None: return None - subject = self._subject_name_func(ctx, self._schema_name) + subject = ( + self._subject_name_func(ctx, self._schema_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, self._schema_name) + ) latest_schema = self._get_reader_schema(subject) if subject else None if latest_schema is not None: self._schema_id = SchemaId(JSON_TYPE, latest_schema.schema_id, latest_schema.guid) @@ -478,14 +496,28 @@ class JSONDeserializer(BaseDeserializer): | | | | | | | Defaults to None. | +-----------------------------+----------+----------------------------------------------------+ + | | | The type of subject name strategy to use. | + |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to TOPIC if neither this nor | + | | | subject.name.strategy is specified. | + +-----------------------------+----------+----------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + |``subject.name.strategy.conf``| dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-----------------------------+----------+----------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | | | | constructed. Standard naming strategies are | | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-----------------------------+----------+----------------------------------------------------+ | | | Whether to validate the payload against the | | ``validate`` | bool | the given schema. | @@ -528,7 +560,9 @@ class JSONDeserializer(BaseDeserializer): _default_conf = { 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.deserializer': dual_schema_id_deserializer, 'validate': True, } @@ -581,12 +615,11 @@ def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), + subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), + subject_name_strategy=conf_copy.pop('subject.name.strategy'), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_deserializer = cast( Callable[[bytes, Optional[SerializationContext], Any], io.BytesIO], conf_copy.pop('schema.id.deserializer') @@ -642,7 +675,11 @@ def __deserialize(self, data: Optional[bytes], ctx: Optional[SerializationContex if data is None: return None - subject = self._subject_name_func(ctx, None) + subject = ( + self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, None) + ) latest_schema = None if subject is not None and self._registry is not None: latest_schema = self._get_reader_schema(subject) @@ -655,7 +692,11 @@ def __deserialize(self, data: Optional[bytes], ctx: Optional[SerializationContex writer_schema_raw = self._get_writer_schema(schema_id, subject) writer_schema, writer_ref_registry = self._get_parsed_schema(writer_schema_raw) if subject is None and isinstance(writer_schema, dict): - subject = self._subject_name_func(ctx, writer_schema.get("title")) + subject = ( + self._subject_name_func(ctx, writer_schema.get("title"), self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, writer_schema.get("title")) + ) if subject is not None: latest_schema = self._get_reader_schema(subject) else: diff --git a/src/confluent_kafka/schema_registry/_sync/mock_schema_registry_client.py b/src/confluent_kafka/schema_registry/_sync/mock_schema_registry_client.py index 539070bae..352579afe 100644 --- a/src/confluent_kafka/schema_registry/_sync/mock_schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_sync/mock_schema_registry_client.py @@ -20,7 +20,15 @@ from threading import Lock from typing import Dict, List, Literal, Optional, Union -from ..common.schema_registry_client import RegisteredSchema, Schema, ServerConfig +from ..common.schema_registry_client import ( + Association, + AssociationCreateOrUpdateRequest, + AssociationInfo, + AssociationResponse, + RegisteredSchema, + Schema, + ServerConfig, +) from ..error import SchemaRegistryError from .schema_registry_client import SchemaRegistryClient @@ -142,11 +150,119 @@ def clear(self): self.subject_schemas.clear() +class _AssociationStore(object): + + def __init__(self): + self.lock = Lock() + # Key: resource_id -> List[Association] + self.associations_by_resource_id: Dict[str, List[Association]] = defaultdict(list) + # Key: (resource_namespace, resource_name) -> resource_id + self.resource_id_index: Dict[tuple, str] = {} + + def create_association(self, request: AssociationCreateOrUpdateRequest) -> AssociationResponse: + with self.lock: + resource_id = request.resource_id + resource_name = request.resource_name + resource_namespace = request.resource_namespace + resource_type = request.resource_type + + # Index resource_id by (namespace, name) + if resource_name and resource_namespace: + self.resource_id_index[(resource_namespace, resource_name)] = resource_id + + created_associations = [] + if request.associations: + for assoc_info in request.associations: + association = Association( + subject=assoc_info.subject, + guid=None, + resource_name=resource_name, + resource_namespace=resource_namespace, + resource_id=resource_id, + resource_type=resource_type, + association_type=assoc_info.association_type, + frozen=assoc_info.frozen if assoc_info.frozen is not None else False, + ) + self.associations_by_resource_id[resource_id].append(association) + created_associations.append( + AssociationInfo( + subject=assoc_info.subject, + association_type=assoc_info.association_type, + lifecycle=assoc_info.lifecycle, + frozen=assoc_info.frozen if assoc_info.frozen is not None else False, + schema=assoc_info.schema, + ) + ) + + return AssociationResponse( + resource_name=resource_name, + resource_namespace=resource_namespace, + resource_id=resource_id, + resource_type=resource_type, + associations=created_associations, + ) + + def delete_associations( + self, + resource_id: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + ) -> None: + with self.lock: + if resource_id not in self.associations_by_resource_id: + return + + if association_types is None and resource_type is None: + # Delete all associations for this resource + del self.associations_by_resource_id[resource_id] + else: + # Filter and keep only non-matching associations + remaining = [] + for assoc in self.associations_by_resource_id[resource_id]: + keep = False + if resource_type is not None and assoc.resource_type != resource_type: + keep = True + if association_types is not None and assoc.association_type not in association_types: + keep = True + if keep: + remaining.append(assoc) + self.associations_by_resource_id[resource_id] = remaining + + def get_associations_by_resource_name( + self, + resource_name: str, + resource_namespace: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + ) -> List[Association]: + with self.lock: + result = [] + for resource_id, associations in self.associations_by_resource_id.items(): + for assoc in associations: + # Check if namespace matches (or is wildcard) + if resource_namespace != "-" and assoc.resource_namespace != resource_namespace: + continue + if assoc.resource_name != resource_name: + continue + if resource_type is not None and assoc.resource_type != resource_type: + continue + if association_types is not None and assoc.association_type not in association_types: + continue + result.append(assoc) + return result + + def clear(self): + with self.lock: + self.associations_by_resource_id.clear() + self.resource_id_index.clear() + + class MockSchemaRegistryClient(SchemaRegistryClient): def __init__(self, conf: dict): super().__init__(conf) self._store = _SchemaStore() + self._association_store = _AssociationStore() def register_schema(self, subject_name: str, schema: 'Schema', normalize_schemas: bool = False) -> int: registered_schema = self.register_schema_full_response( @@ -300,3 +416,46 @@ def set_config( def get_config(self, subject_name: Optional[str] = None) -> 'ServerConfig': # noqa F821 return None # type: ignore[return-value] + + def get_associations_by_resource_name( + self, + resource_name: str, + resource_namespace: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + offset: int = 0, + limit: int = -1, + ) -> List['Association']: + return self._association_store.get_associations_by_resource_name( + resource_name, resource_namespace, resource_type, association_types + ) + + def create_association(self, request: 'AssociationCreateOrUpdateRequest') -> 'AssociationResponse': + """ + Creates an association between a subject and a resource. + + Args: + request (AssociationCreateOrUpdateRequest): The association create or update request. + + Returns: + AssociationResponse: The response containing the created associations. + """ + return self._association_store.create_association(request) + + def delete_associations( + self, + resource_id: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + cascade_lifecycle: bool = False, + ) -> None: + """ + Deletes associations for a resource. + + Args: + resource_id (str): The resource identifier. + resource_type (str, optional): The type of resource (e.g., "topic"). + association_types (List[str], optional): The types of associations to delete. + cascade_lifecycle (bool): Whether to cascade the lifecycle policy to dependent schemas. + """ + self._association_store.delete_associations(resource_id, resource_type, association_types) diff --git a/src/confluent_kafka/schema_registry/_sync/protobuf.py b/src/confluent_kafka/schema_registry/_sync/protobuf.py index 5e1620276..7f0900bfe 100644 --- a/src/confluent_kafka/schema_registry/_sync/protobuf.py +++ b/src/confluent_kafka/schema_registry/_sync/protobuf.py @@ -31,7 +31,6 @@ dual_schema_id_deserializer, prefix_schema_id_serializer, reference_subject_name_strategy, - topic_subject_name_strategy, ) from confluent_kafka.schema_registry.common.protobuf import ( PROTOBUF_TYPE, @@ -149,14 +148,28 @@ class ProtobufSerializer(BaseSerializer): | | | | | | | Defaults to True. | +-------------------------------------+----------+------------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to TOPIC if neither this nor | + | | | subject.name.strategy is specified. | + +-------------------------------------+----------+------------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | | | | constructed. Standard naming strategies are | | | | defined in the confluent_kafka.schema_registry | - | | | namespace. | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | @@ -227,7 +240,9 @@ class ProtobufSerializer(BaseSerializer): 'use.latest.version': False, 'use.latest.with.metadata': None, 'skip.known.types': True, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'reference.subject.name.strategy': reference_subject_name_strategy, 'schema.id.serializer': prefix_schema_id_serializer, 'use.deprecated.format': False, @@ -279,19 +294,18 @@ def __init_impl( if self._use_deprecated_format: raise ValueError("use.deprecated.format is no longer supported") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), + subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), + subject_name_strategy=conf_copy.pop('subject.name.strategy'), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._ref_reference_subject_func = cast( Callable[[Optional[SerializationContext], Any], Optional[str]], conf_copy.pop('reference.subject.name.strategy'), ) if not callable(self._ref_reference_subject_func): - raise ValueError("subject.name.strategy must be callable") + raise ValueError("reference.subject.name.strategy must be callable") self._schema_id_serializer = cast( Callable[[bytes, Optional[SerializationContext], Any], bytes], conf_copy.pop('schema.id.serializer') @@ -414,7 +428,15 @@ def __serialize(self, message: Message, ctx: Optional[SerializationContext] = No if not isinstance(message, self._msg_class): raise ValueError("message must be of type {} not {}".format(self._msg_class, type(message))) - subject = self._subject_name_func(ctx, message.DESCRIPTOR.full_name) if ctx else None + subject = ( + ( + self._subject_name_func(ctx, message.DESCRIPTOR.full_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, message.DESCRIPTOR.full_name) + ) + if ctx + else None + ) latest_schema = None if subject is not None: latest_schema = self._get_reader_schema(subject, fmt='serialized') @@ -508,14 +530,28 @@ class ProtobufDeserializer(BaseDeserializer): | | | | | | | Defaults to None. | +-------------------------------------+----------+------------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to TOPIC if neither this nor | + | | | subject.name.strategy is specified. | + +-------------------------------------+----------+------------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(SerializationContext, str) -> str | | | | | | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka. schema_registry | - | | | namespace . | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | | | | | - | | | Defaults to topic_subject_name_strategy. | + | | | Defaults to None. | +-------------------------------------+----------+------------------------------------------------------+ | | | Callable(bytes, SerializationContext, schema_id) | | | | -> io.BytesIO | @@ -534,7 +570,9 @@ class ProtobufDeserializer(BaseDeserializer): _default_conf = { 'use.latest.version': False, 'use.latest.with.metadata': None, - 'subject.name.strategy': topic_subject_name_strategy, + 'subject.name.strategy.type': None, + 'subject.name.strategy.conf': None, + 'subject.name.strategy': None, 'schema.id.deserializer': dual_schema_id_deserializer, 'use.deprecated.format': False, } @@ -566,12 +604,11 @@ def __init_impl( if self._use_latest_with_metadata is not None and not isinstance(self._use_latest_with_metadata, dict): raise ValueError("use.latest.with.metadata must be a dict value") - self._subject_name_func = cast( - Callable[[Optional[SerializationContext], Optional[str]], Optional[str]], - conf_copy.pop('subject.name.strategy'), + self.configure_subject_name_strategy( + subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), + subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), + subject_name_strategy=conf_copy.pop('subject.name.strategy'), ) - if not callable(self._subject_name_func): - raise ValueError("subject.name.strategy must be callable") self._schema_id_deserializer = cast( Callable[[bytes, Optional[SerializationContext], Any], io.BytesIO], conf_copy.pop('schema.id.deserializer') @@ -619,7 +656,11 @@ def __deserialize(self, data: Optional[bytes], ctx: Optional[SerializationContex if data is None: return None - subject = self._subject_name_func(ctx, None) + subject = ( + self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, None) + ) latest_schema = None if subject is not None and self._registry is not None: latest_schema = self._get_reader_schema(subject, fmt='serialized') @@ -634,7 +675,11 @@ def __deserialize(self, data: Optional[bytes], ctx: Optional[SerializationContex writer_schema = pool.FindFileByName(fd_proto.name) writer_desc = self._get_message_desc(pool, writer_schema, msg_index if msg_index is not None else []) if subject is None: - subject = self._subject_name_func(ctx, writer_desc.full_name) + subject = ( + self._subject_name_func(ctx, writer_desc.full_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, writer_desc.full_name) + ) if subject is not None: latest_schema = self._get_reader_schema(subject, fmt='serialized') else: diff --git a/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py b/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py index 9f0e56a5a..5f4f4a982 100644 --- a/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py @@ -41,6 +41,9 @@ _StaticOAuthBearerFieldProviderBuilder, ) from confluent_kafka.schema_registry.common.schema_registry_client import ( + Association, + AssociationCreateOrUpdateRequest, + AssociationResponse, RegisteredSchema, Schema, SchemaVersion, @@ -1566,6 +1569,102 @@ def clear_caches(self): self._latest_with_metadata_cache.clear() self._cache.clear() + def get_associations_by_resource_name( + self, + resource_name: str, + resource_namespace: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + offset: int = 0, + limit: int = -1, + ) -> List['Association']: + """ + Retrieves associations for a given resource name and namespace. + + Args: + resource_name (str): The name of the resource (e.g., topic name). + resource_namespace (str): The namespace of the resource (e.g., kafka cluster ID). + Use "-" as a wildcard. + resource_type (str, optional): The type of resource (e.g., "topic"). + association_types (List[str], optional): The types of associations to filter by + (e.g., ["key", "value"]). + offset (int): Pagination offset for results. + limit (int): Pagination size for results. Ignored if negative. + + Returns: + List[Association]: List of associations matching the criteria. + + Raises: + SchemaRegistryError: if the request was unsuccessful. + """ + query: Dict[str, Any] = {} + if resource_type is not None: + query['resourceType'] = resource_type + if association_types is not None: + query['associationType'] = association_types + if offset > 0: + query['offset'] = offset + if limit >= 1: + query['limit'] = limit + + response = self._rest_client.get( + 'associations/resources/{}/{}'.format(_urlencode(resource_namespace), _urlencode(resource_name)), query + ) + + return [Association.from_dict(a) for a in response] + + def create_association(self, request: 'AssociationCreateOrUpdateRequest') -> 'AssociationResponse': + """ + Creates an association between a subject and a resource. + + Args: + request (AssociationCreateOrUpdateRequest): The association create or update request. + + Returns: + AssociationResponse: The response containing the created associations. + + Raises: + SchemaRegistryError: if the request was unsuccessful. + """ + response = self._rest_client.post('associations', body=request.to_dict()) + return AssociationResponse.from_dict(response) + + def delete_associations( + self, + resource_id: str, + resource_type: Optional[str] = None, + association_types: Optional[List[str]] = None, + cascade_lifecycle: bool = False, + ) -> None: + """ + Deletes associations for a resource. + + Args: + resource_id (str): The resource identifier. + resource_type (str, optional): The type of resource (e.g., "topic"). + association_types (List[str], optional): The types of associations to delete + (e.g., ["key", "value"]). If not specified, all associations are deleted. + cascade_lifecycle (bool): Whether to cascade the lifecycle policy to dependent schemas. + + Raises: + SchemaRegistryError: if the request was unsuccessful. + """ + query: Dict[str, Any] = {'cascadeLifecycle': str(cascade_lifecycle).lower()} + if resource_type is not None: + query['resourceType'] = resource_type + if association_types is not None: + query['associationType'] = association_types + + self._rest_client.delete( + 'associations/resources/{}?{}'.format( + _urlencode(resource_id), + '&'.join( + f"{k}={v}" if not isinstance(v, list) else '&'.join(f"{k}={item}" for item in v) + for k, v in query.items() + ), + ) + ) + @staticmethod def new_client(conf: dict) -> 'SchemaRegistryClient': from .mock_schema_registry_client import MockSchemaRegistryClient diff --git a/src/confluent_kafka/schema_registry/_sync/serde.py b/src/confluent_kafka/schema_registry/_sync/serde.py index 56868cad0..272f0ce22 100644 --- a/src/confluent_kafka/schema_registry/_sync/serde.py +++ b/src/confluent_kafka/schema_registry/_sync/serde.py @@ -17,11 +17,19 @@ # import logging -from typing import Any, Callable, Dict, List, Optional, Set +import threading as _locks +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union -from confluent_kafka.schema_registry import RegisteredSchema +from cachetools import LRUCache + +from confluent_kafka.schema_registry import ( + RegisteredSchema, + SchemaRegistryClient, + topic_subject_name_strategy, +) from confluent_kafka.schema_registry.common.schema_registry_client import RulePhase from confluent_kafka.schema_registry.common.serde import ( + STRATEGY_TYPE_MAP, ErrorAction, FieldTransformer, Migration, @@ -31,19 +39,204 @@ RuleContext, RuleError, SchemaId, + SubjectNameStrategyType, ) +from confluent_kafka.schema_registry.error import SchemaRegistryError from confluent_kafka.schema_registry.schema_registry_client import Rule, RuleKind, RuleMode, RuleSet, Schema -from confluent_kafka.serialization import Deserializer, SerializationContext, SerializationError, Serializer +from confluent_kafka.serialization import ( + Deserializer, + MessageField, + SerializationContext, + SerializationError, + Serializer, +) __all__ = [ + 'AssociatedNameStrategy', 'BaseSerde', 'BaseSerializer', 'BaseDeserializer', + 'KAFKA_CLUSTER_ID', + 'FALLBACK_SUBJECT_NAME_STRATEGY_TYPE', ] log = logging.getLogger(__name__) +KAFKA_CLUSTER_ID = "kafka.cluster.id" +NAMESPACE_WILDCARD = "-" +FALLBACK_SUBJECT_NAME_STRATEGY_TYPE = "fallback.subject.name.strategy.type" +DEFAULT_CACHE_CAPACITY = 1000 + + +class AssociatedNameStrategy: + """ + A subject name strategy that retrieves the associated subject name from schema registry + by querying associations for the topic. + + This class encapsulates a cache for subject name lookups to avoid repeated API calls. + + Args: + cache_capacity (int): Maximum number of entries to cache. Defaults to 1000. + """ + + def __init__(self, cache_capacity: int = DEFAULT_CACHE_CAPACITY): + self._cache: LRUCache = LRUCache(maxsize=cache_capacity) + self._lock: _locks.Lock = _locks.Lock() + + def _get_cache_key(self, topic: str, is_key: bool, record_name: Optional[str]) -> Tuple[str, bool, Optional[str]]: + """Create a cache key from topic, is_key, and record_name.""" + return (topic, is_key, record_name) + + def _load_subject_name( + self, + topic: str, + is_key: bool, + record_name: Optional[str], + ctx: SerializationContext, + schema_registry_client: SchemaRegistryClient, + conf: Optional[dict], + ) -> Optional[str]: + """Load the subject name from schema registry (not cached).""" + # Determine resource namespace from config + kafka_cluster_id = None + fallback_strategy = SubjectNameStrategyType.TOPIC # default fallback + + if conf is not None: + kafka_cluster_id = conf.get(KAFKA_CLUSTER_ID) + fallback_config = conf.get(FALLBACK_SUBJECT_NAME_STRATEGY_TYPE) + if fallback_config is not None: + if isinstance(fallback_config, SubjectNameStrategyType): + fallback_strategy = fallback_config + else: + try: + fallback_strategy = SubjectNameStrategyType(str(fallback_config).upper()) + except ValueError: + valid_fallbacks = [ + e.value for e in SubjectNameStrategyType if e != SubjectNameStrategyType.ASSOCIATED + ] + raise ValueError( + f"Invalid value for {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE}: {fallback_config}. " + f"Valid values are: {', '.join(valid_fallbacks)}" + ) + + resource_namespace = kafka_cluster_id if kafka_cluster_id is not None else NAMESPACE_WILDCARD + + # Determine association type based on whether this is key or value + association_type = "key" if is_key else "value" + + # Query schema registry for associations + try: + associations = schema_registry_client.get_associations_by_resource_name( + resource_name=topic, + resource_namespace=resource_namespace, + resource_type="topic", + association_types=[association_type], + offset=0, + limit=-1, + ) + except SchemaRegistryError as e: + if e.http_status_code == 404: + return STRATEGY_TYPE_MAP[fallback_strategy](ctx, record_name) + else: + raise + + if len(associations) > 1: + raise SerializationError(f"Multiple associated subjects found for topic {topic}") + elif len(associations) == 1: + return associations[0].subject + else: + # No associations found, use fallback strategy + if fallback_strategy == SubjectNameStrategyType.NONE: + raise SerializationError(f"No associated subject found for topic {topic}") + elif fallback_strategy == SubjectNameStrategyType.ASSOCIATED: + raise ValueError( + f"Invalid value for {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE}: {fallback_strategy.value}. " + f"ASSOCIATED cannot be used as a fallback strategy." + ) + + return STRATEGY_TYPE_MAP[fallback_strategy](ctx, record_name) + + def __call__( + self, + ctx: Optional[SerializationContext], + record_name: Optional[str], + schema_registry_client: SchemaRegistryClient, + conf: Optional[dict] = None, + ) -> Optional[str]: + """ + Retrieves the associated subject name from schema registry by querying + associations for the topic. + + The topic is passed as the resource name to schema registry. If there is a + configuration property named "kafka.cluster.id", then its value will be passed + as the resource namespace; otherwise the value "-" will be passed as the + resource namespace. + + If more than one subject is returned from the query, a SerializationError + will be raised. If no subjects are returned from the query, then the behavior + will fall back to topic_subject_name_strategy, unless the configuration property + "fallback.subject.name.strategy.type" is set to "RECORD", "TOPIC_RECORD", or "NONE". + + Results are cached using an LRU cache to avoid repeated API calls. + + Args: + ctx (SerializationContext): Metadata pertaining to the serialization + operation. **Required** - must contain topic and field information. + + record_name (Optional[str]): Record name (used for fallback strategies). + + schema_registry_client (SchemaRegistryClient): SchemaRegistryClient instance. + + conf (Optional[dict]): Configuration dictionary. Supports: + - "kafka.cluster.id": Kafka cluster ID to use as resource namespace. + - "fallback.subject.name.strategy.type": Fallback strategy when no + associations are found. One of "TOPIC", "RECORD", "TOPIC_RECORD", or "NONE". + Defaults to "TOPIC". + + Returns: + Optional[str]: The subject name from the association, or from the fallback strategy. + + Raises: + SerializationError: If multiple associated subjects are found for the topic, + or if no subjects are found and fallback is set to "NONE". + ValueError: If ctx is None. + """ + if ctx is None: + raise ValueError( + "SerializationContext is required for AssociatedNameStrategy. " + "Either provide a SerializationContext or use a different strategy." + ) + + topic = ctx.topic + if topic is None: + return None + + is_key = ctx.field == MessageField.KEY + cache_key = self._get_cache_key(topic, is_key, record_name) + + # Check cache first + with self._lock: + cached_result = self._cache.get(cache_key) + if cached_result is not None: + return cached_result + + # Not in cache, load from schema registry + result = self._load_subject_name(topic, is_key, record_name, ctx, schema_registry_client, conf) + + # Cache the result + if result is not None: + with self._lock: + self._cache[cache_key] = result + + return result + + def clear_cache(self) -> None: + """Clear the association subject name cache.""" + with self._lock: + self._cache.clear() + + class BaseSerde(object): __slots__ = [ '_use_schema_id', @@ -51,6 +244,8 @@ class BaseSerde(object): '_use_latest_with_metadata', '_registry', '_rule_registry', + '_strategy_accepts_client', + '_subject_name_conf', '_subject_name_func', '_field_transformer', ] @@ -60,9 +255,80 @@ class BaseSerde(object): _use_latest_with_metadata: Optional[Dict[str, str]] _registry: Any # SchemaRegistryClient _rule_registry: Any # RuleRegistry + _subject_name_conf: Optional[dict] _subject_name_func: Callable[[Optional['SerializationContext'], Optional[str]], Optional[str]] _field_transformer: Optional[FieldTransformer] + def configure_subject_name_strategy( + self, + subject_name_strategy_type: Optional[Union[SubjectNameStrategyType, str]] = None, + subject_name_strategy_conf: Optional[dict] = None, + subject_name_strategy: Optional[Callable] = None, + ) -> None: + """ + Configure the subject name strategy for this serde. + + This method supports both the legacy callable approach and the new type-based approach. + If both `subject_name_strategy` (as a callable) and `subject_name_strategy_type` are + provided, the callable takes precedence. + + Args: + subject_name_strategy: A callable that implements the subject name strategy. + Signature: (SerializationContext, str) -> str or + (SerializationContext, str, SchemaRegistryClient, dict) -> str + + subject_name_strategy_type: The type of subject name strategy to use. + Can be a SubjectNameStrategyType enum value or a string + ("TOPIC", "RECORD", "TOPIC_RECORD", "ASSOCIATED"). + + subject_name_strategy_conf: Configuration dictionary passed to strategies + that accept extra parameters (like ASSOCIATED). + + Raises: + ValueError: If the strategy is not callable or the type is invalid. + """ + self._subject_name_conf = subject_name_strategy_conf + + # If a callable is provided, use it directly (backward compatible) + if subject_name_strategy is not None: + if not callable(subject_name_strategy): + raise ValueError("subject.name.strategy must be callable") + self._subject_name_func = subject_name_strategy + self._strategy_accepts_client = False + return + + # If a type is provided, resolve it to a callable + if subject_name_strategy_type is not None: + # Convert string to enum if needed + if isinstance(subject_name_strategy_type, str): + try: + subject_name_strategy_type = SubjectNameStrategyType(subject_name_strategy_type.upper()) + except ValueError: + raise ValueError( + f"Invalid subject.name.strategy.type: {subject_name_strategy_type}. " + f"Valid values are: {[e.value for e in SubjectNameStrategyType]}" + ) + + # Handle ASSOCIATED specially since it needs schema_registry_client + if subject_name_strategy_type == SubjectNameStrategyType.ASSOCIATED: + self._subject_name_func = AssociatedNameStrategy() + self._strategy_accepts_client = True + elif subject_name_strategy_type == SubjectNameStrategyType.NONE: + raise ValueError( + f"Invalid subject.name.strategy.type: {subject_name_strategy_type}. " + f"NONE cannot be used as a subject name strategy." + ) + elif subject_name_strategy_type in STRATEGY_TYPE_MAP: + self._subject_name_func = STRATEGY_TYPE_MAP[subject_name_strategy_type] + self._strategy_accepts_client = False + else: + raise ValueError(f"Unknown subject.name.strategy.type: {subject_name_strategy_type}") + return + + # Default to topic_subject_name_strategy + self._subject_name_func = topic_subject_name_strategy + self._strategy_accepts_client = False + def _get_reader_schema(self, subject: str, fmt: Optional[str] = None) -> Optional[RegisteredSchema]: if self._use_schema_id is not None: schema = self._registry.get_schema(self._use_schema_id, subject, fmt) diff --git a/src/confluent_kafka/schema_registry/common/schema_registry_client.py b/src/confluent_kafka/schema_registry/common/schema_registry_client.py index eaf2430f6..b7f8a8696 100644 --- a/src/confluent_kafka/schema_registry/common/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/common/schema_registry_client.py @@ -42,6 +42,7 @@ 'ServerConfig', 'Schema', 'RegisteredSchema', + 'Association', ] VALID_AUTH_PROVIDERS = ['URL', 'USER_INFO'] @@ -972,3 +973,250 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: ) return schema # type: ignore[return-value] + + +@_attrs_define(frozen=True) +class Association: + """ + An association between a subject and a resource. + """ + + subject: Optional[str] + guid: Optional[str] + resource_name: Optional[str] + resource_namespace: Optional[str] + resource_id: Optional[str] + resource_type: Optional[str] + association_type: Optional[str] + frozen: bool = False + + def to_dict(self) -> Dict[str, Any]: + field_dict: Dict[str, Any] = {} + if self.subject is not None: + field_dict["subject"] = self.subject + if self.guid is not None: + field_dict["guid"] = self.guid + if self.resource_name is not None: + field_dict["resourceName"] = self.resource_name + if self.resource_namespace is not None: + field_dict["resourceNamespace"] = self.resource_namespace + if self.resource_id is not None: + field_dict["resourceId"] = self.resource_id + if self.resource_type is not None: + field_dict["resourceType"] = self.resource_type + if self.association_type is not None: + field_dict["associationType"] = self.association_type + field_dict["frozen"] = self.frozen + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + + subject = d.pop("subject", None) + guid = d.pop("guid", None) + resource_name = d.pop("resourceName", None) + resource_namespace = d.pop("resourceNamespace", None) + resource_id = d.pop("resourceId", None) + resource_type = d.pop("resourceType", None) + association_type = d.pop("associationType", None) + frozen = d.pop("frozen", False) + + association = cls( # type: ignore[call-arg] + subject=subject, + guid=guid, + resource_name=resource_name, + resource_namespace=resource_namespace, + resource_id=resource_id, + resource_type=resource_type, + association_type=association_type, + frozen=frozen, + ) + + return association + + +@_attrs_define +class AssociationInfo: + """ + Information about an association in a response. + """ + + subject: Optional[str] = None + association_type: Optional[str] = None + lifecycle: Optional[str] = None + frozen: bool = False + schema: Optional['Schema'] = None + + def to_dict(self) -> Dict[str, Any]: + field_dict: Dict[str, Any] = {} + if self.subject is not None: + field_dict["subject"] = self.subject + if self.association_type is not None: + field_dict["associationType"] = self.association_type + if self.lifecycle is not None: + field_dict["lifecycle"] = self.lifecycle + field_dict["frozen"] = self.frozen + if self.schema is not None: + field_dict["schema"] = self.schema.to_dict() + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + subject = d.pop("subject", None) + association_type = d.pop("associationType", None) + lifecycle = d.pop("lifecycle", None) + frozen = d.pop("frozen", False) + schema_dict = d.pop("schema", None) + schema = Schema.from_dict(schema_dict) if schema_dict else None + + return cls( # type: ignore[call-arg] + subject=subject, + association_type=association_type, + lifecycle=lifecycle, + frozen=frozen, + schema=schema, + ) + + +@_attrs_define +class AssociationCreateOrUpdateInfo: + """ + Information about an association to create or update. + """ + + subject: Optional[str] = None + association_type: Optional[str] = None + lifecycle: Optional[str] = None + frozen: Optional[bool] = None + schema: Optional['Schema'] = None + normalize: Optional[bool] = None + + def to_dict(self) -> Dict[str, Any]: + field_dict: Dict[str, Any] = {} + if self.subject is not None: + field_dict["subject"] = self.subject + if self.association_type is not None: + field_dict["associationType"] = self.association_type + if self.lifecycle is not None: + field_dict["lifecycle"] = self.lifecycle + if self.frozen is not None: + field_dict["frozen"] = self.frozen + if self.schema is not None: + field_dict["schema"] = self.schema.to_dict() + if self.normalize is not None: + field_dict["normalize"] = self.normalize + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + subject = d.pop("subject", None) + association_type = d.pop("associationType", None) + lifecycle = d.pop("lifecycle", None) + frozen = d.pop("frozen", None) + schema_dict = d.pop("schema", None) + schema = Schema.from_dict(schema_dict) if schema_dict else None + normalize = d.pop("normalize", None) + + return cls( # type: ignore[call-arg] + subject=subject, + association_type=association_type, + lifecycle=lifecycle, + frozen=frozen, + schema=schema, + normalize=normalize, + ) + + +@_attrs_define +class AssociationCreateOrUpdateRequest: + """ + Request to create or update associations. + """ + + resource_name: Optional[str] = None + resource_namespace: Optional[str] = None + resource_id: Optional[str] = None + resource_type: Optional[str] = None + associations: Optional[List[AssociationCreateOrUpdateInfo]] = None + + def to_dict(self) -> Dict[str, Any]: + field_dict: Dict[str, Any] = {} + if self.resource_name is not None: + field_dict["resourceName"] = self.resource_name + if self.resource_namespace is not None: + field_dict["resourceNamespace"] = self.resource_namespace + if self.resource_id is not None: + field_dict["resourceId"] = self.resource_id + if self.resource_type is not None: + field_dict["resourceType"] = self.resource_type + if self.associations is not None: + field_dict["associations"] = [a.to_dict() for a in self.associations] + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + resource_name = d.pop("resourceName", None) + resource_namespace = d.pop("resourceNamespace", None) + resource_id = d.pop("resourceId", None) + resource_type = d.pop("resourceType", None) + associations_list = d.pop("associations", None) + associations = [AssociationCreateOrUpdateInfo.from_dict(a) + for a in associations_list] if associations_list else None + + return cls( # type: ignore[call-arg] + resource_name=resource_name, + resource_namespace=resource_namespace, + resource_id=resource_id, + resource_type=resource_type, + associations=associations, + ) + + +@_attrs_define +class AssociationResponse: + """ + Response from creating or updating associations. + """ + + resource_name: Optional[str] = None + resource_namespace: Optional[str] = None + resource_id: Optional[str] = None + resource_type: Optional[str] = None + associations: Optional[List[AssociationInfo]] = None + + def to_dict(self) -> Dict[str, Any]: + field_dict: Dict[str, Any] = {} + if self.resource_name is not None: + field_dict["resourceName"] = self.resource_name + if self.resource_namespace is not None: + field_dict["resourceNamespace"] = self.resource_namespace + if self.resource_id is not None: + field_dict["resourceId"] = self.resource_id + if self.resource_type is not None: + field_dict["resourceType"] = self.resource_type + if self.associations is not None: + field_dict["associations"] = [a.to_dict() for a in self.associations] + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + resource_name = d.pop("resourceName", None) + resource_namespace = d.pop("resourceNamespace", None) + resource_id = d.pop("resourceId", None) + resource_type = d.pop("resourceType", None) + associations_list = d.pop("associations", None) + associations = [AssociationInfo.from_dict(a) for a in associations_list] if associations_list else None + + return cls( # type: ignore[call-arg] + resource_name=resource_name, + resource_namespace=resource_namespace, + resource_id=resource_id, + resource_type=resource_type, + associations=associations, + ) diff --git a/src/confluent_kafka/schema_registry/common/serde.py b/src/confluent_kafka/schema_registry/common/serde.py index c41de51fe..decb6e45c 100644 --- a/src/confluent_kafka/schema_registry/common/serde.py +++ b/src/confluent_kafka/schema_registry/common/serde.py @@ -25,12 +25,21 @@ from threading import Lock from typing import Any, Callable, Dict, List, Optional, Set, TypeVar -from confluent_kafka.schema_registry import _MAGIC_BYTE_V0, _MAGIC_BYTE_V1, RegisteredSchema +from confluent_kafka.schema_registry import ( + _MAGIC_BYTE_V0, + _MAGIC_BYTE_V1, + RegisteredSchema, + record_subject_name_strategy, + topic_record_subject_name_strategy, + topic_subject_name_strategy, +) from confluent_kafka.schema_registry.schema_registry_client import Rule, RuleKind, RuleMode, Schema from confluent_kafka.schema_registry.wildcard_matcher import wildcard_match from confluent_kafka.serialization import SerializationContext, SerializationError __all__ = [ + 'STRATEGY_TYPE_MAP', + 'SubjectNameStrategyType', 'FieldType', 'FieldContext', 'RuleContext', @@ -52,6 +61,23 @@ log = logging.getLogger(__name__) +class SubjectNameStrategyType(str, Enum): + NONE = "NONE" + TOPIC = "TOPIC" + RECORD = "RECORD" + TOPIC_RECORD = "TOPIC_RECORD" + ASSOCIATED = "ASSOCIATED" + + +# Mapping from SubjectNameStrategyType to strategy functions. +# NONE and ASSOCIATED are handled specially and not included here. +STRATEGY_TYPE_MAP = { + SubjectNameStrategyType.TOPIC: topic_subject_name_strategy, + SubjectNameStrategyType.RECORD: record_subject_name_strategy, + SubjectNameStrategyType.TOPIC_RECORD: topic_record_subject_name_strategy, +} + + class FieldType(str, Enum): RECORD = "RECORD" ENUM = "ENUM" diff --git a/tests/schema_registry/_async/test_avro_serdes.py b/tests/schema_registry/_async/test_avro_serdes.py index ce168bdfa..623ae2c9f 100644 --- a/tests/schema_registry/_async/test_avro_serdes.py +++ b/tests/schema_registry/_async/test_avro_serdes.py @@ -29,6 +29,15 @@ Schema, header_schema_id_serializer, ) +from confluent_kafka.schema_registry.common.schema_registry_client import ( + AssociationCreateOrUpdateInfo, + AssociationCreateOrUpdateRequest, +) +from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType +from confluent_kafka.schema_registry._async.serde import ( + KAFKA_CLUSTER_ID, + FALLBACK_SUBJECT_NAME_STRATEGY_TYPE, +) from confluent_kafka.schema_registry.avro import AsyncAvroDeserializer, AsyncAvroSerializer from confluent_kafka.schema_registry.rule_registry import RuleOverride, RuleRegistry from confluent_kafka.schema_registry.rules.cel.cel_executor import CelExecutor @@ -2658,3 +2667,396 @@ def __init__(self, award, user): def __eq__(self, other): return all([self.award == other.award, self.user == other.user]) + + +async def test_associated_name_strategy_with_association(): + """Test that AsyncAssociatedNameStrategy returns subject from association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema and test object + schema = { + 'type': 'record', + 'name': 'TestRecord', + 'fields': [ + {'name': 'intField', 'type': 'int'}, + {'name': 'stringField', 'type': 'string'}, + ], + } + obj = {'intField': 123, 'stringField': 'hello'} + + # Add an association for the custom subject + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="mock-resource-id-1", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-custom-subject-value", + association_type="value", + ) + ], + ) + await client.create_association(request) + + # Create serializer with associated name strategy + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + # Deserialize and verify + deser = await AsyncAvroDeserializer(client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Verify the schema was registered with the custom subject + registered_schema = await client.get_latest_version("my-custom-subject-value") + assert registered_schema is not None + + +async def test_associated_name_strategy_with_key_association(): + """Test that AsyncAssociatedNameStrategy returns subject for key""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema and test object + schema = { + 'type': 'record', + 'name': 'KeyRecord', + 'fields': [ + {'name': 'id', 'type': 'int'}, + ], + } + obj = {'id': 42} + + # Add an association for key + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="mock-resource-id-2", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-key-subject", + association_type="key", + ) + ], + ) + await client.create_association(request) + + # Create serializer with associated name strategy for KEY + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.KEY) + obj_bytes = await ser(obj, ser_ctx) + + # Deserialize and verify + deser = await AsyncAvroDeserializer(client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Verify the schema was registered with the key subject + registered_schema = await client.get_latest_version("my-key-subject") + assert registered_schema is not None + + +async def test_associated_name_strategy_fallback_to_topic(): + """Test fallback to topic_subject_name_strategy when no association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema and test object + schema = { + 'type': 'record', + 'name': 'TestRecord', + 'fields': [ + {'name': 'intField', 'type': 'int'}, + {'name': 'stringField', 'type': 'string'}, + ], + } + obj = {'intField': 456, 'stringField': 'world'} + + # No associations added, should fall back to topic strategy + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + # Deserialize and verify + deser = await AsyncAvroDeserializer(client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Default fallback is topic_subject_name_strategy which returns topic-value + registered_schema = await client.get_latest_version(_TOPIC + "-value") + assert registered_schema is not None + + +async def test_associated_name_strategy_fallback_to_record(): + """Test fallback to record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema with a specific record name + schema = { + 'type': 'record', + 'name': 'MyRecord', + 'fields': [ + {'name': 'value', 'type': 'string'}, + ], + } + obj = {'value': 'test'} + + # No associations, configure fallback to RECORD + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.RECORD}, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + # Deserialize and verify + deser = await AsyncAvroDeserializer(client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Should have registered under the record name + registered_schema = await client.get_latest_version("MyRecord") + assert registered_schema is not None + + +async def test_associated_name_strategy_fallback_to_topic_record(): + """Test fallback to topic_record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema with a specific record name + schema = { + 'type': 'record', + 'name': 'MyRecord', + 'fields': [ + {'name': 'data', 'type': 'int'}, + ], + } + obj = {'data': 789} + + # No associations, configure fallback to TOPIC_RECORD + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + # Deserialize and verify + deser = await AsyncAvroDeserializer(client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Should have registered under topic-record_name + registered_schema = await client.get_latest_version(_TOPIC + "-MyRecord") + assert registered_schema is not None + + +async def test_associated_name_strategy_fallback_none_raises(): + """Test that NONE fallback raises an error when no association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema + schema = { + 'type': 'record', + 'name': 'MyRecord', + 'fields': [ + {'name': 'value', 'type': 'string'}, + ], + } + obj = {'value': 'test'} + + # No associations, configure fallback to NONE + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: "NONE"}, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + await ser(obj, ser_ctx) + + assert "No associated subject found" in str(exc_info.value) + + +async def test_associated_name_strategy_multiple_associations_raises(): + """Test that multiple associations raise an error""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema + schema = { + 'type': 'record', + 'name': 'TestRecord', + 'fields': [ + {'name': 'value', 'type': 'string'}, + ], + } + obj = {'value': 'test'} + + # Add multiple associations for the same topic/value + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="mock-resource-id-3", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="subject1", + association_type="value", + ), + AssociationCreateOrUpdateInfo( + subject="subject2", + association_type="value", + ), + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + await ser(obj, ser_ctx) + + assert "Multiple associated subjects found" in str(exc_info.value) + + +async def test_associated_name_strategy_with_kafka_cluster_id(): + """Test that kafka.cluster.id config is used as resource namespace""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema + schema = { + 'type': 'record', + 'name': 'TestRecord', + 'fields': [ + {'name': 'intField', 'type': 'int'}, + ], + } + obj = {'intField': 100} + + # Add an association with specific namespace + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="my-cluster-id", + resource_id="mock-resource-id-4", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cluster-specific-subject", + association_type="value", + ) + ], + ) + await client.create_association(request) + + # Create serializer with matching cluster ID + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {KAFKA_CLUSTER_ID: "my-cluster-id"}, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + # Deserialize and verify + deser = await AsyncAvroDeserializer(client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Verify the schema was registered with the cluster-specific subject + registered_schema = await client.get_latest_version("cluster-specific-subject") + assert registered_schema is not None + + +async def test_associated_name_strategy_caching(): + """Test that results are cached within a strategy instance and serializer works with caching""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + # Define schema + schema = { + 'type': 'record', + 'name': 'CacheTestRecord', + 'fields': [ + {'name': 'count', 'type': 'int'}, + ], + } + + # Add an association + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="mock-resource-id-5", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cached-subject", + association_type="value", + ) + ], + ) + await client.create_association(request) + + # Create serializer with associated name strategy + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + # First serialization + obj1 = {'count': 1} + obj_bytes1 = await ser(obj1, ser_ctx) + + # Verify it was registered under cached-subject + registered_schema = await client.get_latest_version("cached-subject") + assert registered_schema is not None + + # Deserialize first message + deser = await AsyncAvroDeserializer(client) + result1 = await deser(obj_bytes1, ser_ctx) + assert obj1 == result1 + + # Delete associations (but serializer should still work due to caching) + await client.delete_associations("mock-resource-id-5") + + # Second serialization should still work (schema already registered) + obj2 = {'count': 2} + obj_bytes2 = await ser(obj2, ser_ctx) + + # Deserialize second message + result2 = await deser(obj_bytes2, ser_ctx) + assert obj2 == result2 diff --git a/tests/schema_registry/_sync/test_avro_serdes.py b/tests/schema_registry/_sync/test_avro_serdes.py index 0c8ca7977..f57a69102 100644 --- a/tests/schema_registry/_sync/test_avro_serdes.py +++ b/tests/schema_registry/_sync/test_avro_serdes.py @@ -29,7 +29,16 @@ SchemaRegistryClient, header_schema_id_serializer, ) +from confluent_kafka.schema_registry._sync.serde import ( + FALLBACK_SUBJECT_NAME_STRATEGY_TYPE, + KAFKA_CLUSTER_ID, +) from confluent_kafka.schema_registry.avro import AvroDeserializer, AvroSerializer +from confluent_kafka.schema_registry.common.schema_registry_client import ( + AssociationCreateOrUpdateInfo, + AssociationCreateOrUpdateRequest, +) +from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.rule_registry import RuleOverride, RuleRegistry from confluent_kafka.schema_registry.rules.cel.cel_executor import CelExecutor from confluent_kafka.schema_registry.rules.cel.cel_field_executor import CelFieldExecutor @@ -2658,3 +2667,396 @@ def __init__(self, award, user): def __eq__(self, other): return all([self.award == other.award, self.user == other.user]) + + +def test_associated_name_strategy_with_association(): + """Test that AssociatedNameStrategy returns subject from association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema and test object + schema = { + 'type': 'record', + 'name': 'TestRecord', + 'fields': [ + {'name': 'intField', 'type': 'int'}, + {'name': 'stringField', 'type': 'string'}, + ], + } + obj = {'intField': 123, 'stringField': 'hello'} + + # Add an association for the custom subject + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="mock-resource-id-1", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-custom-subject-value", + association_type="value", + ) + ], + ) + client.create_association(request) + + # Create serializer with associated name strategy + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + # Deserialize and verify + deser = AvroDeserializer(client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Verify the schema was registered with the custom subject + registered_schema = client.get_latest_version("my-custom-subject-value") + assert registered_schema is not None + + +def test_associated_name_strategy_with_key_association(): + """Test that AssociatedNameStrategy returns subject for key""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema and test object + schema = { + 'type': 'record', + 'name': 'KeyRecord', + 'fields': [ + {'name': 'id', 'type': 'int'}, + ], + } + obj = {'id': 42} + + # Add an association for key + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="mock-resource-id-2", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-key-subject", + association_type="key", + ) + ], + ) + client.create_association(request) + + # Create serializer with associated name strategy for KEY + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.KEY) + obj_bytes = ser(obj, ser_ctx) + + # Deserialize and verify + deser = AvroDeserializer(client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Verify the schema was registered with the key subject + registered_schema = client.get_latest_version("my-key-subject") + assert registered_schema is not None + + +def test_associated_name_strategy_fallback_to_topic(): + """Test fallback to topic_subject_name_strategy when no association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema and test object + schema = { + 'type': 'record', + 'name': 'TestRecord', + 'fields': [ + {'name': 'intField', 'type': 'int'}, + {'name': 'stringField', 'type': 'string'}, + ], + } + obj = {'intField': 456, 'stringField': 'world'} + + # No associations added, should fall back to topic strategy + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + # Deserialize and verify + deser = AvroDeserializer(client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Default fallback is topic_subject_name_strategy which returns topic-value + registered_schema = client.get_latest_version(_TOPIC + "-value") + assert registered_schema is not None + + +def test_associated_name_strategy_fallback_to_record(): + """Test fallback to record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema with a specific record name + schema = { + 'type': 'record', + 'name': 'MyRecord', + 'fields': [ + {'name': 'value', 'type': 'string'}, + ], + } + obj = {'value': 'test'} + + # No associations, configure fallback to RECORD + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.RECORD}, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + # Deserialize and verify + deser = AvroDeserializer(client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Should have registered under the record name + registered_schema = client.get_latest_version("MyRecord") + assert registered_schema is not None + + +def test_associated_name_strategy_fallback_to_topic_record(): + """Test fallback to topic_record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema with a specific record name + schema = { + 'type': 'record', + 'name': 'MyRecord', + 'fields': [ + {'name': 'data', 'type': 'int'}, + ], + } + obj = {'data': 789} + + # No associations, configure fallback to TOPIC_RECORD + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + # Deserialize and verify + deser = AvroDeserializer(client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Should have registered under topic-record_name + registered_schema = client.get_latest_version(_TOPIC + "-MyRecord") + assert registered_schema is not None + + +def test_associated_name_strategy_fallback_none_raises(): + """Test that NONE fallback raises an error when no association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema + schema = { + 'type': 'record', + 'name': 'MyRecord', + 'fields': [ + {'name': 'value', 'type': 'string'}, + ], + } + obj = {'value': 'test'} + + # No associations, configure fallback to NONE + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: "NONE"}, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + ser(obj, ser_ctx) + + assert "No associated subject found" in str(exc_info.value) + + +def test_associated_name_strategy_multiple_associations_raises(): + """Test that multiple associations raise an error""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema + schema = { + 'type': 'record', + 'name': 'TestRecord', + 'fields': [ + {'name': 'value', 'type': 'string'}, + ], + } + obj = {'value': 'test'} + + # Add multiple associations for the same topic/value + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="mock-resource-id-3", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="subject1", + association_type="value", + ), + AssociationCreateOrUpdateInfo( + subject="subject2", + association_type="value", + ), + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + ser(obj, ser_ctx) + + assert "Multiple associated subjects found" in str(exc_info.value) + + +def test_associated_name_strategy_with_kafka_cluster_id(): + """Test that kafka.cluster.id config is used as resource namespace""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema + schema = { + 'type': 'record', + 'name': 'TestRecord', + 'fields': [ + {'name': 'intField', 'type': 'int'}, + ], + } + obj = {'intField': 100} + + # Add an association with specific namespace + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="my-cluster-id", + resource_id="mock-resource-id-4", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cluster-specific-subject", + association_type="value", + ) + ], + ) + client.create_association(request) + + # Create serializer with matching cluster ID + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {KAFKA_CLUSTER_ID: "my-cluster-id"}, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + # Deserialize and verify + deser = AvroDeserializer(client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + # Verify the schema was registered with the cluster-specific subject + registered_schema = client.get_latest_version("cluster-specific-subject") + assert registered_schema is not None + + +def test_associated_name_strategy_caching(): + """Test that results are cached within a strategy instance and serializer works with caching""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + # Define schema + schema = { + 'type': 'record', + 'name': 'CacheTestRecord', + 'fields': [ + {'name': 'count', 'type': 'int'}, + ], + } + + # Add an association + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="mock-resource-id-5", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cached-subject", + association_type="value", + ) + ], + ) + client.create_association(request) + + # Create serializer with associated name strategy + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + # First serialization + obj1 = {'count': 1} + obj_bytes1 = ser(obj1, ser_ctx) + + # Verify it was registered under cached-subject + registered_schema = client.get_latest_version("cached-subject") + assert registered_schema is not None + + # Deserialize first message + deser = AvroDeserializer(client) + result1 = deser(obj_bytes1, ser_ctx) + assert obj1 == result1 + + # Delete associations (but serializer should still work due to caching) + client.delete_associations("mock-resource-id-5") + + # Second serialization should still work (schema already registered) + obj2 = {'count': 2} + obj_bytes2 = ser(obj2, ser_ctx) + + # Deserialize second message + result2 = deser(obj_bytes2, ser_ctx) + assert obj2 == result2 From 3fb0fef43489bc064361a8c318b7ac0d3c32799f Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Mon, 23 Feb 2026 17:13:11 -0800 Subject: [PATCH 02/20] Incorporate review feedback --- src/confluent_kafka/schema_registry/_async/serde.py | 3 ++- src/confluent_kafka/schema_registry/_sync/serde.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/confluent_kafka/schema_registry/_async/serde.py b/src/confluent_kafka/schema_registry/_async/serde.py index 54e46ab98..89b028edb 100644 --- a/src/confluent_kafka/schema_registry/_async/serde.py +++ b/src/confluent_kafka/schema_registry/_async/serde.py @@ -133,7 +133,8 @@ async def _load_subject_name( ) except SchemaRegistryError as e: if e.http_status_code == 404: - return STRATEGY_TYPE_MAP[fallback_strategy](ctx, record_name) + # Treat 404 as no associations found and fall through to existing fallback logic + associations = [] else: raise diff --git a/src/confluent_kafka/schema_registry/_sync/serde.py b/src/confluent_kafka/schema_registry/_sync/serde.py index 272f0ce22..725152970 100644 --- a/src/confluent_kafka/schema_registry/_sync/serde.py +++ b/src/confluent_kafka/schema_registry/_sync/serde.py @@ -137,7 +137,8 @@ def _load_subject_name( ) except SchemaRegistryError as e: if e.http_status_code == 404: - return STRATEGY_TYPE_MAP[fallback_strategy](ctx, record_name) + # Treat 404 as no associations found and fall through to existing fallback logic + associations = [] else: raise From 0ec11ae04ecfae4e7aa090bd742ac7a340f7afff Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Mon, 23 Feb 2026 17:26:00 -0800 Subject: [PATCH 03/20] Incorporate review feedback --- .../_async/schema_registry_client.py | 13 +++++-------- .../_sync/schema_registry_client.py | 16 ++++------------ 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/src/confluent_kafka/schema_registry/_async/schema_registry_client.py b/src/confluent_kafka/schema_registry/_async/schema_registry_client.py index aa3ffc990..dd10a8209 100644 --- a/src/confluent_kafka/schema_registry/_async/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_async/schema_registry_client.py @@ -403,7 +403,7 @@ async def get(self, url: str, query: Optional[dict] = None) -> Any: async def post(self, url: str, body: Optional[dict], **kwargs) -> Any: raise NotImplementedError() - async def delete(self, url: str) -> Any: + async def delete(self, url: str, query: Optional[dict] = None) -> Any: raise NotImplementedError() async def put(self, url: str, body: Optional[dict] = None) -> Any: @@ -453,8 +453,8 @@ async def get(self, url: str, query: Optional[dict] = None) -> Any: async def post(self, url: str, body: Optional[dict], **kwargs) -> Any: return await self.send_request(url, method='POST', body=body) - async def delete(self, url: str) -> Any: - return await self.send_request(url, method='DELETE') + async def delete(self, url: str, query: Optional[dict] = None) -> Any: + return await self.send_request(url, method='DELETE', query=query) async def put(self, url: str, body: Optional[dict] = None) -> Any: return await self.send_request(url, method='PUT', body=body) @@ -1676,11 +1676,8 @@ async def delete_associations( query['associationType'] = association_types await self._rest_client.delete( - 'associations/resources/{}?{}'.format( - _urlencode(resource_id), - '&'.join(f"{k}={v}" if not isinstance(v, list) - else '&'.join(f"{k}={item}" for item in v) for k, v in query.items()) - ) + 'associations/resources/{}'.format(_urlencode(resource_id)), + query=query ) @staticmethod diff --git a/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py b/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py index 5f4f4a982..e2bd2554b 100644 --- a/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py @@ -402,7 +402,7 @@ def get(self, url: str, query: Optional[dict] = None) -> Any: def post(self, url: str, body: Optional[dict], **kwargs) -> Any: raise NotImplementedError() - def delete(self, url: str) -> Any: + def delete(self, url: str, query: Optional[dict] = None) -> Any: raise NotImplementedError() def put(self, url: str, body: Optional[dict] = None) -> Any: @@ -452,8 +452,8 @@ def get(self, url: str, query: Optional[dict] = None) -> Any: def post(self, url: str, body: Optional[dict], **kwargs) -> Any: return self.send_request(url, method='POST', body=body) - def delete(self, url: str) -> Any: - return self.send_request(url, method='DELETE') + def delete(self, url: str, query: Optional[dict] = None) -> Any: + return self.send_request(url, method='DELETE', query=query) def put(self, url: str, body: Optional[dict] = None) -> Any: return self.send_request(url, method='PUT', body=body) @@ -1655,15 +1655,7 @@ def delete_associations( if association_types is not None: query['associationType'] = association_types - self._rest_client.delete( - 'associations/resources/{}?{}'.format( - _urlencode(resource_id), - '&'.join( - f"{k}={v}" if not isinstance(v, list) else '&'.join(f"{k}={item}" for item in v) - for k, v in query.items() - ), - ) - ) + self._rest_client.delete('associations/resources/{}'.format(_urlencode(resource_id)), query=query) @staticmethod def new_client(conf: dict) -> 'SchemaRegistryClient': From e3c07d4d9dd820a2153c9eebf9114bb2ea9fa333 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Mon, 23 Feb 2026 17:27:19 -0800 Subject: [PATCH 04/20] Incorporate review feedback --- .../schema_registry/common/schema_registry_client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/confluent_kafka/schema_registry/common/schema_registry_client.py b/src/confluent_kafka/schema_registry/common/schema_registry_client.py index b7f8a8696..fa9f8c482 100644 --- a/src/confluent_kafka/schema_registry/common/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/common/schema_registry_client.py @@ -43,6 +43,10 @@ 'Schema', 'RegisteredSchema', 'Association', + 'AssociationInfo', + 'AssociationCreateOrUpdateInfo', + 'AssociationCreateOrUpdateRequest', + 'AssociationResponse', ] VALID_AUTH_PROVIDERS = ['URL', 'USER_INFO'] From eec9287fd63b50ed147a2467801623f4c0a3b06d Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Mon, 23 Feb 2026 17:31:21 -0800 Subject: [PATCH 05/20] Incorporate review feedback --- src/confluent_kafka/schema_registry/_async/serde.py | 2 +- src/confluent_kafka/schema_registry/_sync/serde.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/confluent_kafka/schema_registry/_async/serde.py b/src/confluent_kafka/schema_registry/_async/serde.py index 89b028edb..d92737154 100644 --- a/src/confluent_kafka/schema_registry/_async/serde.py +++ b/src/confluent_kafka/schema_registry/_async/serde.py @@ -291,7 +291,7 @@ def configure_subject_name_strategy( if not callable(subject_name_strategy): raise ValueError("subject.name.strategy must be callable") self._subject_name_func = subject_name_strategy - self._strategy_accepts_client = False + self._strategy_accepts_client = isinstance(subject_name_strategy, AsyncAssociatedNameStrategy) return # If a type is provided, resolve it to a callable diff --git a/src/confluent_kafka/schema_registry/_sync/serde.py b/src/confluent_kafka/schema_registry/_sync/serde.py index 725152970..2a96c5bd6 100644 --- a/src/confluent_kafka/schema_registry/_sync/serde.py +++ b/src/confluent_kafka/schema_registry/_sync/serde.py @@ -295,7 +295,7 @@ def configure_subject_name_strategy( if not callable(subject_name_strategy): raise ValueError("subject.name.strategy must be callable") self._subject_name_func = subject_name_strategy - self._strategy_accepts_client = False + self._strategy_accepts_client = isinstance(subject_name_strategy, AssociatedNameStrategy) return # If a type is provided, resolve it to a callable From bc587166a44798c6f0c2ca3c7fc080bd12157ed1 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Mon, 23 Feb 2026 17:32:14 -0800 Subject: [PATCH 06/20] Incorporate review feedback --- .../schema_registry/_async/protobuf.py | 22 +++++++++++++------ .../schema_registry/_sync/protobuf.py | 21 +++++++++++++----- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/confluent_kafka/schema_registry/_async/protobuf.py b/src/confluent_kafka/schema_registry/_async/protobuf.py index 108f857fc..4a054c9ae 100644 --- a/src/confluent_kafka/schema_registry/_async/protobuf.py +++ b/src/confluent_kafka/schema_registry/_async/protobuf.py @@ -665,9 +665,13 @@ async def __deserialize(self, data: Optional[bytes], ctx: Optional[Serialization return None subject = ( - await self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) - if self._strategy_accepts_client - else self._subject_name_func(ctx, None) + ( + await self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, None) + ) + if ctx + else None ) latest_schema = None if subject is not None and self._registry is not None: @@ -684,10 +688,14 @@ async def __deserialize(self, data: Optional[bytes], ctx: Optional[Serialization writer_desc = self._get_message_desc(pool, writer_schema, msg_index if msg_index is not None else []) if subject is None: subject = ( - await self._subject_name_func( - ctx, writer_desc.full_name, self._registry, self._subject_name_conf) - if self._strategy_accepts_client - else self._subject_name_func(ctx, writer_desc.full_name) + ( + await self._subject_name_func( + ctx, writer_desc.full_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, writer_desc.full_name) + ) + if ctx + else None ) if subject is not None: latest_schema = await self._get_reader_schema(subject, fmt='serialized') diff --git a/src/confluent_kafka/schema_registry/_sync/protobuf.py b/src/confluent_kafka/schema_registry/_sync/protobuf.py index 7f0900bfe..2751facac 100644 --- a/src/confluent_kafka/schema_registry/_sync/protobuf.py +++ b/src/confluent_kafka/schema_registry/_sync/protobuf.py @@ -657,9 +657,13 @@ def __deserialize(self, data: Optional[bytes], ctx: Optional[SerializationContex return None subject = ( - self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) - if self._strategy_accepts_client - else self._subject_name_func(ctx, None) + ( + self._subject_name_func(ctx, None, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, None) + ) + if ctx + else None ) latest_schema = None if subject is not None and self._registry is not None: @@ -676,9 +680,14 @@ def __deserialize(self, data: Optional[bytes], ctx: Optional[SerializationContex writer_desc = self._get_message_desc(pool, writer_schema, msg_index if msg_index is not None else []) if subject is None: subject = ( - self._subject_name_func(ctx, writer_desc.full_name, self._registry, self._subject_name_conf) - if self._strategy_accepts_client - else self._subject_name_func(ctx, writer_desc.full_name) + ( + self._subject_name_func( + ctx, writer_desc.full_name, self._registry, self._subject_name_conf) + if self._strategy_accepts_client + else self._subject_name_func(ctx, writer_desc.full_name) + ) + if ctx + else None ) if subject is not None: latest_schema = self._get_reader_schema(subject, fmt='serialized') From c5733edfd5c4efee57a9094d3abd315014ee8630 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Mon, 23 Feb 2026 17:37:44 -0800 Subject: [PATCH 07/20] Incorporate review feedback --- src/confluent_kafka/schema_registry/_async/serde.py | 10 +++++----- src/confluent_kafka/schema_registry/_sync/protobuf.py | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/confluent_kafka/schema_registry/_async/serde.py b/src/confluent_kafka/schema_registry/_async/serde.py index d92737154..2d0185d1e 100644 --- a/src/confluent_kafka/schema_registry/_async/serde.py +++ b/src/confluent_kafka/schema_registry/_async/serde.py @@ -23,9 +23,9 @@ from cachetools import LRUCache from confluent_kafka.schema_registry import ( + AsyncSchemaRegistryClient, RegisteredSchema, topic_subject_name_strategy, - SchemaRegistryClient, ) from confluent_kafka.schema_registry.error import SchemaRegistryError from confluent_kafka.schema_registry.common.schema_registry_client import RulePhase @@ -91,7 +91,7 @@ async def _load_subject_name( is_key: bool, record_name: Optional[str], ctx: SerializationContext, - schema_registry_client: SchemaRegistryClient, + schema_registry_client: AsyncSchemaRegistryClient, conf: Optional[dict] ) -> Optional[str]: """Load the subject name from schema registry (not cached).""" @@ -158,7 +158,7 @@ async def __call__( self, ctx: Optional[SerializationContext], record_name: Optional[str], - schema_registry_client: SchemaRegistryClient, + schema_registry_client: AsyncSchemaRegistryClient, conf: Optional[dict] = None ) -> Optional[str]: """ @@ -183,7 +183,7 @@ async def __call__( record_name (Optional[str]): Record name (used for fallback strategies). - schema_registry_client (SchemaRegistryClient): SchemaRegistryClient instance. + schema_registry_client (AsyncSchemaRegistryClient): AsyncSchemaRegistryClient instance. conf (Optional[dict]): Configuration dictionary. Supports: - "kafka.cluster.id": Kafka cluster ID to use as resource namespace. @@ -272,7 +272,7 @@ def configure_subject_name_strategy( Args: subject_name_strategy: A callable that implements the subject name strategy. Signature: (SerializationContext, str) -> str or - (SerializationContext, str, SchemaRegistryClient, dict) -> str + (SerializationContext, str, AsyncSchemaRegistryClient, dict) -> str subject_name_strategy_type: The type of subject name strategy to use. Can be a SubjectNameStrategyType enum value or a string diff --git a/src/confluent_kafka/schema_registry/_sync/protobuf.py b/src/confluent_kafka/schema_registry/_sync/protobuf.py index 2751facac..edc48d2ba 100644 --- a/src/confluent_kafka/schema_registry/_sync/protobuf.py +++ b/src/confluent_kafka/schema_registry/_sync/protobuf.py @@ -681,8 +681,7 @@ def __deserialize(self, data: Optional[bytes], ctx: Optional[SerializationContex if subject is None: subject = ( ( - self._subject_name_func( - ctx, writer_desc.full_name, self._registry, self._subject_name_conf) + self._subject_name_func(ctx, writer_desc.full_name, self._registry, self._subject_name_conf) if self._strategy_accepts_client else self._subject_name_func(ctx, writer_desc.full_name) ) From 428cdc8f86f70111f19aae97b70f98f9c7a4f36c Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Mon, 23 Feb 2026 19:41:32 -0800 Subject: [PATCH 08/20] Incorporate review feedback --- src/confluent_kafka/schema_registry/_async/serde.py | 1 + src/confluent_kafka/schema_registry/_sync/serde.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/confluent_kafka/schema_registry/_async/serde.py b/src/confluent_kafka/schema_registry/_async/serde.py index 2d0185d1e..2bf615c90 100644 --- a/src/confluent_kafka/schema_registry/_async/serde.py +++ b/src/confluent_kafka/schema_registry/_async/serde.py @@ -252,6 +252,7 @@ class AsyncBaseSerde(object): _use_latest_with_metadata: Optional[Dict[str, str]] _registry: Any # AsyncSchemaRegistryClient _rule_registry: Any # RuleRegistry + _strategy_accepts_client: bool _subject_name_conf: Optional[dict] _subject_name_func: Callable[[Optional['SerializationContext'], Optional[str]], Optional[str]] _field_transformer: Optional[FieldTransformer] diff --git a/src/confluent_kafka/schema_registry/_sync/serde.py b/src/confluent_kafka/schema_registry/_sync/serde.py index 2a96c5bd6..4cad46d02 100644 --- a/src/confluent_kafka/schema_registry/_sync/serde.py +++ b/src/confluent_kafka/schema_registry/_sync/serde.py @@ -256,6 +256,7 @@ class BaseSerde(object): _use_latest_with_metadata: Optional[Dict[str, str]] _registry: Any # SchemaRegistryClient _rule_registry: Any # RuleRegistry + _strategy_accepts_client: bool _subject_name_conf: Optional[dict] _subject_name_func: Callable[[Optional['SerializationContext'], Optional[str]], Optional[str]] _field_transformer: Optional[FieldTransformer] From c4ae106243760e0d2b552368b68fbd6d1a030022 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Mon, 23 Feb 2026 20:32:41 -0800 Subject: [PATCH 09/20] Incorporate review feedback --- .../schema_registry/_async/schema_registry_client.py | 2 +- .../schema_registry/_sync/schema_registry_client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/confluent_kafka/schema_registry/_async/schema_registry_client.py b/src/confluent_kafka/schema_registry/_async/schema_registry_client.py index dd10a8209..91b148ea4 100644 --- a/src/confluent_kafka/schema_registry/_async/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_async/schema_registry_client.py @@ -1669,7 +1669,7 @@ async def delete_associations( Raises: SchemaRegistryError: if the request was unsuccessful. """ - query: Dict[str, Any] = {'cascadeLifecycle': str(cascade_lifecycle).lower()} + query: Dict[str, Any] = {'cascadeLifecycle': cascade_lifecycle} if resource_type is not None: query['resourceType'] = resource_type if association_types is not None: diff --git a/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py b/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py index e2bd2554b..c440d439e 100644 --- a/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py @@ -1649,7 +1649,7 @@ def delete_associations( Raises: SchemaRegistryError: if the request was unsuccessful. """ - query: Dict[str, Any] = {'cascadeLifecycle': str(cascade_lifecycle).lower()} + query: Dict[str, Any] = {'cascadeLifecycle': cascade_lifecycle} if resource_type is not None: query['resourceType'] = resource_type if association_types is not None: From e3eb783e36c16905b133c5a54a46c5add4c25385 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Tue, 24 Feb 2026 16:45:35 -0800 Subject: [PATCH 10/20] Add tests --- .../_async/test_json_serdes.py | 294 +++++++++++++++++ .../_async/test_proto_serdes.py | 308 ++++++++++++++++++ .../schema_registry/_sync/test_json_serdes.py | 296 +++++++++++++++++ .../_sync/test_proto_serdes.py | 308 ++++++++++++++++++ 4 files changed, 1206 insertions(+) diff --git a/tests/schema_registry/_async/test_json_serdes.py b/tests/schema_registry/_async/test_json_serdes.py index 6059cb55c..b89157da5 100644 --- a/tests/schema_registry/_async/test_json_serdes.py +++ b/tests/schema_registry/_async/test_json_serdes.py @@ -49,6 +49,15 @@ SchemaReference, ServerConfig, ) +from confluent_kafka.schema_registry._async.serde import ( + FALLBACK_SUBJECT_NAME_STRATEGY_TYPE, + KAFKA_CLUSTER_ID, +) +from confluent_kafka.schema_registry.common.schema_registry_client import ( + AssociationCreateOrUpdateInfo, + AssociationCreateOrUpdateRequest, +) +from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.serde import RuleConditionError from confluent_kafka.serialization import MessageField, SerializationContext, SerializationError from tests.schema_registry._async.test_avro_serdes import FakeClock @@ -1449,3 +1458,288 @@ async def test_json_deeply_nested_refs(): executor.executor.client = dek_client obj2 = await deser(obj_bytes, ser_ctx) assert obj == obj2 + + +_JSON_SCHEMA = json.dumps({ + "type": "object", + "title": "MyRecord", + "properties": { + "name": {"type": "string"}, + "id": {"type": "integer"}, + }, +}) +_JSON_OBJ = {"name": "Kafka", "id": 123} + + +async def test_json_associated_name_strategy_with_association(): + """Test that AssociatedNameStrategy returns subject from association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="json-resource-id-1", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-custom-subject-value", + association_type="value", + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(_JSON_OBJ, ser_ctx) + + deser = await AsyncJSONDeserializer(None, schema_registry_client=client) + obj2 = await deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = await client.get_latest_version("my-custom-subject-value") + assert registered_schema is not None + + +async def test_json_associated_name_strategy_with_key_association(): + """Test that AssociatedNameStrategy returns subject for key""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="json-resource-id-2", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-key-subject", + association_type="key", + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.KEY) + obj_bytes = await ser(_JSON_OBJ, ser_ctx) + + deser = await AsyncJSONDeserializer(None, schema_registry_client=client) + obj2 = await deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = await client.get_latest_version("my-key-subject") + assert registered_schema is not None + + +async def test_json_associated_name_strategy_fallback_to_topic(): + """Test fallback to topic_subject_name_strategy when no association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(_JSON_OBJ, ser_ctx) + + deser = await AsyncJSONDeserializer(None, schema_registry_client=client) + obj2 = await deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = await client.get_latest_version(_TOPIC + "-value") + assert registered_schema is not None + + +async def test_json_associated_name_strategy_fallback_to_record(): + """Test fallback to record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.RECORD}, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(_JSON_OBJ, ser_ctx) + + deser = await AsyncJSONDeserializer(None, schema_registry_client=client) + obj2 = await deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + # JSON record name comes from schema "title" + registered_schema = await client.get_latest_version("MyRecord") + assert registered_schema is not None + + +async def test_json_associated_name_strategy_fallback_to_topic_record(): + """Test fallback to topic_record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(_JSON_OBJ, ser_ctx) + + deser = await AsyncJSONDeserializer(None, schema_registry_client=client) + obj2 = await deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + # JSON topic-record subject: "topic1-MyRecord" + registered_schema = await client.get_latest_version(_TOPIC + "-MyRecord") + assert registered_schema is not None + + +async def test_json_associated_name_strategy_fallback_none_raises(): + """Test that NONE fallback raises an error when no association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: "NONE"}, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + await ser(_JSON_OBJ, ser_ctx) + + assert "No associated subject found" in str(exc_info.value) + + +async def test_json_associated_name_strategy_multiple_associations_raises(): + """Test that multiple associations raise an error""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="json-resource-id-3", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="json-subject-1", + association_type="value", + ), + AssociationCreateOrUpdateInfo( + subject="json-subject-2", + association_type="value", + ), + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + await ser(_JSON_OBJ, ser_ctx) + + assert "Multiple associated subjects found" in str(exc_info.value) + + +async def test_json_associated_name_strategy_with_kafka_cluster_id(): + """Test that kafka.cluster.id config is used as resource namespace""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="my-cluster-id", + resource_id="json-resource-id-4", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cluster-specific-json-subject", + association_type="value", + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {KAFKA_CLUSTER_ID: "my-cluster-id"}, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(_JSON_OBJ, ser_ctx) + + deser = await AsyncJSONDeserializer(None, schema_registry_client=client) + obj2 = await deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = await client.get_latest_version("cluster-specific-json-subject") + assert registered_schema is not None + + +async def test_json_associated_name_strategy_caching(): + """Test that results are cached within a strategy instance and serializer works with caching""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="json-resource-id-5", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="json-cached-subject", + association_type="value", + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + obj1 = {"name": "Kafka", "id": 1} + obj_bytes1 = await ser(obj1, ser_ctx) + + registered_schema = await client.get_latest_version("json-cached-subject") + assert registered_schema is not None + + deser = await AsyncJSONDeserializer(None, schema_registry_client=client) + result1 = await deser(obj_bytes1, ser_ctx) + assert obj1 == result1 + + # Delete associations (but serializer should still work due to caching) + await client.delete_associations("json-resource-id-5") + + obj2 = {"name": "Kafka", "id": 2} + obj_bytes2 = await ser(obj2, ser_ctx) + + result2 = await deser(obj_bytes2, ser_ctx) + assert obj2 == result2 diff --git a/tests/schema_registry/_async/test_proto_serdes.py b/tests/schema_registry/_async/test_proto_serdes.py index e08e62501..8536e9f92 100644 --- a/tests/schema_registry/_async/test_proto_serdes.py +++ b/tests/schema_registry/_async/test_proto_serdes.py @@ -46,6 +46,15 @@ RuleSet, ServerConfig, ) +from confluent_kafka.schema_registry._async.serde import ( + FALLBACK_SUBJECT_NAME_STRATEGY_TYPE, + KAFKA_CLUSTER_ID, +) +from confluent_kafka.schema_registry.common.schema_registry_client import ( + AssociationCreateOrUpdateInfo, + AssociationCreateOrUpdateRequest, +) +from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.serde import RuleConditionError from confluent_kafka.serialization import MessageField, SerializationContext, SerializationError @@ -615,3 +624,302 @@ async def deserialize_with_all_versions(client, ser_ctx, obj_bytes, obj, obj2, o deser = await AsyncProtobufDeserializer(newerwidget_pb2.NewerWidget, deser_conf, client) newobj = await deser(obj_bytes, ser_ctx) assert obj3.length == newobj.length + + +async def test_associated_name_strategy_with_association(): + """Test that AssociatedNameStrategy returns subject from association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=123, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="proto-resource-id-1", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-custom-subject-value", + association_type="value", + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + deser = await AsyncProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = await client.get_latest_version("my-custom-subject-value") + assert registered_schema is not None + + +async def test_associated_name_strategy_with_key_association(): + """Test that AssociatedNameStrategy returns subject for key""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author(name='Kafka', id=42) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="proto-resource-id-2", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-key-subject", + association_type="key", + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.KEY) + obj_bytes = await ser(obj, ser_ctx) + + deser = await AsyncProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = await client.get_latest_version("my-key-subject") + assert registered_schema is not None + + +async def test_associated_name_strategy_fallback_to_topic(): + """Test fallback to topic_subject_name_strategy when no association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=456, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + deser = await AsyncProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = await client.get_latest_version(_TOPIC + "-value") + assert registered_schema is not None + + +async def test_associated_name_strategy_fallback_to_record(): + """Test fallback to record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=789, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.RECORD}, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + deser = await AsyncProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = await client.get_latest_version(example_pb2.Author.DESCRIPTOR.full_name) + assert registered_schema is not None + + +async def test_associated_name_strategy_fallback_to_topic_record(): + """Test fallback to topic_record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=100, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + deser = await AsyncProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = await client.get_latest_version(_TOPIC + "-" + example_pb2.Author.DESCRIPTOR.full_name) + assert registered_schema is not None + + +async def test_associated_name_strategy_fallback_none_raises(): + """Test that NONE fallback raises an error when no association""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author(name='Kafka', id=1) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: "NONE"}, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + await ser(obj, ser_ctx) + + assert "No associated subject found" in str(exc_info.value) + + +async def test_associated_name_strategy_multiple_associations_raises(): + """Test that multiple associations raise an error""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author(name='Kafka', id=2) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="proto-resource-id-3", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="proto-subject-1", + association_type="value", + ), + AssociationCreateOrUpdateInfo( + subject="proto-subject-2", + association_type="value", + ), + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + await ser(obj, ser_ctx) + + assert "Multiple associated subjects found" in str(exc_info.value) + + +async def test_associated_name_strategy_with_kafka_cluster_id(): + """Test that kafka.cluster.id config is used as resource namespace""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=100, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="my-cluster-id", + resource_id="proto-resource-id-4", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cluster-specific-proto-subject", + association_type="value", + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {KAFKA_CLUSTER_ID: "my-cluster-id"}, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = await ser(obj, ser_ctx) + + deser = await AsyncProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = await deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = await client.get_latest_version("cluster-specific-proto-subject") + assert registered_schema is not None + + +async def test_associated_name_strategy_caching(): + """Test that results are cached within a strategy instance and serializer works with caching""" + conf = {'url': _BASE_URL} + client = AsyncSchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="proto-resource-id-5", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="proto-cached-subject", + association_type="value", + ) + ], + ) + await client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + obj1 = example_pb2.Author(name='Kafka', id=1) + obj_bytes1 = await ser(obj1, ser_ctx) + + registered_schema = await client.get_latest_version("proto-cached-subject") + assert registered_schema is not None + + deser = await AsyncProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + result1 = await deser(obj_bytes1, ser_ctx) + assert obj1 == result1 + + # Delete associations (but serializer should still work due to caching) + await client.delete_associations("proto-resource-id-5") + + obj2 = example_pb2.Author(name='Kafka', id=2) + obj_bytes2 = await ser(obj2, ser_ctx) + + result2 = await deser(obj_bytes2, ser_ctx) + assert obj2 == result2 diff --git a/tests/schema_registry/_sync/test_json_serdes.py b/tests/schema_registry/_sync/test_json_serdes.py index 43f10b20f..e95e2842e 100644 --- a/tests/schema_registry/_sync/test_json_serdes.py +++ b/tests/schema_registry/_sync/test_json_serdes.py @@ -27,6 +27,15 @@ SchemaRegistryClient, header_schema_id_serializer, ) +from confluent_kafka.schema_registry._sync.serde import ( + FALLBACK_SUBJECT_NAME_STRATEGY_TYPE, + KAFKA_CLUSTER_ID, +) +from confluent_kafka.schema_registry.common.schema_registry_client import ( + AssociationCreateOrUpdateInfo, + AssociationCreateOrUpdateRequest, +) +from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.json_schema import JSONDeserializer, JSONSerializer from confluent_kafka.schema_registry.rules.cel.cel_executor import CelExecutor from confluent_kafka.schema_registry.rules.cel.cel_field_executor import CelFieldExecutor @@ -1449,3 +1458,290 @@ def test_json_deeply_nested_refs(): executor.executor.client = dek_client obj2 = deser(obj_bytes, ser_ctx) assert obj == obj2 + + +_JSON_SCHEMA = json.dumps( + { + "type": "object", + "title": "MyRecord", + "properties": { + "name": {"type": "string"}, + "id": {"type": "integer"}, + }, + } +) +_JSON_OBJ = {"name": "Kafka", "id": 123} + + +def test_json_associated_name_strategy_with_association(): + """Test that AssociatedNameStrategy returns subject from association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="json-resource-id-1", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-custom-subject-value", + association_type="value", + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(_JSON_OBJ, ser_ctx) + + deser = JSONDeserializer(None, schema_registry_client=client) + obj2 = deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = client.get_latest_version("my-custom-subject-value") + assert registered_schema is not None + + +def test_json_associated_name_strategy_with_key_association(): + """Test that AssociatedNameStrategy returns subject for key""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="json-resource-id-2", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-key-subject", + association_type="key", + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.KEY) + obj_bytes = ser(_JSON_OBJ, ser_ctx) + + deser = JSONDeserializer(None, schema_registry_client=client) + obj2 = deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = client.get_latest_version("my-key-subject") + assert registered_schema is not None + + +def test_json_associated_name_strategy_fallback_to_topic(): + """Test fallback to topic_subject_name_strategy when no association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(_JSON_OBJ, ser_ctx) + + deser = JSONDeserializer(None, schema_registry_client=client) + obj2 = deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = client.get_latest_version(_TOPIC + "-value") + assert registered_schema is not None + + +def test_json_associated_name_strategy_fallback_to_record(): + """Test fallback to record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.RECORD}, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(_JSON_OBJ, ser_ctx) + + deser = JSONDeserializer(None, schema_registry_client=client) + obj2 = deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + # JSON record name comes from schema "title" + registered_schema = client.get_latest_version("MyRecord") + assert registered_schema is not None + + +def test_json_associated_name_strategy_fallback_to_topic_record(): + """Test fallback to topic_record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(_JSON_OBJ, ser_ctx) + + deser = JSONDeserializer(None, schema_registry_client=client) + obj2 = deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + # JSON topic-record subject: "topic1-MyRecord" + registered_schema = client.get_latest_version(_TOPIC + "-MyRecord") + assert registered_schema is not None + + +def test_json_associated_name_strategy_fallback_none_raises(): + """Test that NONE fallback raises an error when no association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: "NONE"}, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + ser(_JSON_OBJ, ser_ctx) + + assert "No associated subject found" in str(exc_info.value) + + +def test_json_associated_name_strategy_multiple_associations_raises(): + """Test that multiple associations raise an error""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="json-resource-id-3", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="json-subject-1", + association_type="value", + ), + AssociationCreateOrUpdateInfo( + subject="json-subject-2", + association_type="value", + ), + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + ser(_JSON_OBJ, ser_ctx) + + assert "Multiple associated subjects found" in str(exc_info.value) + + +def test_json_associated_name_strategy_with_kafka_cluster_id(): + """Test that kafka.cluster.id config is used as resource namespace""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="my-cluster-id", + resource_id="json-resource-id-4", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cluster-specific-json-subject", + association_type="value", + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {KAFKA_CLUSTER_ID: "my-cluster-id"}, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(_JSON_OBJ, ser_ctx) + + deser = JSONDeserializer(None, schema_registry_client=client) + obj2 = deser(obj_bytes, ser_ctx) + assert _JSON_OBJ == obj2 + + registered_schema = client.get_latest_version("cluster-specific-json-subject") + assert registered_schema is not None + + +def test_json_associated_name_strategy_caching(): + """Test that results are cached within a strategy instance and serializer works with caching""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="json-resource-id-5", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="json-cached-subject", + association_type="value", + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + obj1 = {"name": "Kafka", "id": 1} + obj_bytes1 = ser(obj1, ser_ctx) + + registered_schema = client.get_latest_version("json-cached-subject") + assert registered_schema is not None + + deser = JSONDeserializer(None, schema_registry_client=client) + result1 = deser(obj_bytes1, ser_ctx) + assert obj1 == result1 + + # Delete associations (but serializer should still work due to caching) + client.delete_associations("json-resource-id-5") + + obj2 = {"name": "Kafka", "id": 2} + obj_bytes2 = ser(obj2, ser_ctx) + + result2 = deser(obj_bytes2, ser_ctx) + assert obj2 == result2 diff --git a/tests/schema_registry/_sync/test_proto_serdes.py b/tests/schema_registry/_sync/test_proto_serdes.py index ea756a680..f1968770b 100644 --- a/tests/schema_registry/_sync/test_proto_serdes.py +++ b/tests/schema_registry/_sync/test_proto_serdes.py @@ -24,6 +24,15 @@ from confluent_kafka.schema_registry import Metadata, MetadataProperties, Schema, header_schema_id_serializer from confluent_kafka.schema_registry._sync.protobuf import ProtobufDeserializer, ProtobufSerializer from confluent_kafka.schema_registry._sync.schema_registry_client import SchemaRegistryClient +from confluent_kafka.schema_registry._sync.serde import ( + FALLBACK_SUBJECT_NAME_STRATEGY_TYPE, + KAFKA_CLUSTER_ID, +) +from confluent_kafka.schema_registry.common.schema_registry_client import ( + AssociationCreateOrUpdateInfo, + AssociationCreateOrUpdateRequest, +) +from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.protobuf import _schema_to_str from confluent_kafka.schema_registry.rules.cel.cel_executor import CelExecutor from confluent_kafka.schema_registry.rules.cel.cel_field_executor import CelFieldExecutor @@ -615,3 +624,302 @@ def deserialize_with_all_versions(client, ser_ctx, obj_bytes, obj, obj2, obj3): deser = ProtobufDeserializer(newerwidget_pb2.NewerWidget, deser_conf, client) newobj = deser(obj_bytes, ser_ctx) assert obj3.length == newobj.length + + +def test_associated_name_strategy_with_association(): + """Test that AssociatedNameStrategy returns subject from association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=123, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="proto-resource-id-1", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-custom-subject-value", + association_type="value", + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + deser = ProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = client.get_latest_version("my-custom-subject-value") + assert registered_schema is not None + + +def test_associated_name_strategy_with_key_association(): + """Test that AssociatedNameStrategy returns subject for key""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author(name='Kafka', id=42) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="proto-resource-id-2", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="my-key-subject", + association_type="key", + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.KEY) + obj_bytes = ser(obj, ser_ctx) + + deser = ProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = client.get_latest_version("my-key-subject") + assert registered_schema is not None + + +def test_associated_name_strategy_fallback_to_topic(): + """Test fallback to topic_subject_name_strategy when no association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=456, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + deser = ProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = client.get_latest_version(_TOPIC + "-value") + assert registered_schema is not None + + +def test_associated_name_strategy_fallback_to_record(): + """Test fallback to record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=789, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.RECORD}, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + deser = ProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = client.get_latest_version(example_pb2.Author.DESCRIPTOR.full_name) + assert registered_schema is not None + + +def test_associated_name_strategy_fallback_to_topic_record(): + """Test fallback to topic_record_subject_name_strategy when configured""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=100, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + deser = ProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = client.get_latest_version(_TOPIC + "-" + example_pb2.Author.DESCRIPTOR.full_name) + assert registered_schema is not None + + +def test_associated_name_strategy_fallback_none_raises(): + """Test that NONE fallback raises an error when no association""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author(name='Kafka', id=1) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: "NONE"}, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + ser(obj, ser_ctx) + + assert "No associated subject found" in str(exc_info.value) + + +def test_associated_name_strategy_multiple_associations_raises(): + """Test that multiple associations raise an error""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author(name='Kafka', id=2) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="proto-resource-id-3", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="proto-subject-1", + association_type="value", + ), + AssociationCreateOrUpdateInfo( + subject="proto-subject-2", + association_type="value", + ), + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + with pytest.raises(SerializationError) as exc_info: + ser(obj, ser_ctx) + + assert "Multiple associated subjects found" in str(exc_info.value) + + +def test_associated_name_strategy_with_kafka_cluster_id(): + """Test that kafka.cluster.id config is used as resource namespace""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + obj = example_pb2.Author( + name='Kafka', id=100, picture=b'foobar', works=['The Castle', 'TheTrial'], oneof_string='oneof' + ) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="my-cluster-id", + resource_id="proto-resource-id-4", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="cluster-specific-proto-subject", + association_type="value", + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + 'subject.name.strategy.conf': {KAFKA_CLUSTER_ID: "my-cluster-id"}, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + obj_bytes = ser(obj, ser_ctx) + + deser = ProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + obj2 = deser(obj_bytes, ser_ctx) + assert obj == obj2 + + registered_schema = client.get_latest_version("cluster-specific-proto-subject") + assert registered_schema is not None + + +def test_associated_name_strategy_caching(): + """Test that results are cached within a strategy instance and serializer works with caching""" + conf = {'url': _BASE_URL} + client = SchemaRegistryClient.new_client(conf) + + request = AssociationCreateOrUpdateRequest( + resource_name=_TOPIC, + resource_namespace="-", + resource_id="proto-resource-id-5", + resource_type="topic", + associations=[ + AssociationCreateOrUpdateInfo( + subject="proto-cached-subject", + association_type="value", + ) + ], + ) + client.create_association(request) + + ser_conf = { + 'auto.register.schemas': True, + 'use.deprecated.format': False, + 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, + } + ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) + ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) + + obj1 = example_pb2.Author(name='Kafka', id=1) + obj_bytes1 = ser(obj1, ser_ctx) + + registered_schema = client.get_latest_version("proto-cached-subject") + assert registered_schema is not None + + deser = ProtobufDeserializer(example_pb2.Author, {'use.deprecated.format': False}, client) + result1 = deser(obj_bytes1, ser_ctx) + assert obj1 == result1 + + # Delete associations (but serializer should still work due to caching) + client.delete_associations("proto-resource-id-5") + + obj2 = example_pb2.Author(name='Kafka', id=2) + obj_bytes2 = ser(obj2, ser_ctx) + + result2 = deser(obj_bytes2, ser_ctx) + assert obj2 == result2 From d4fe9ef7743dead69dd998d2142be67ef43289bb Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 27 Feb 2026 08:06:27 -0800 Subject: [PATCH 11/20] Make Associated the default strategy --- src/confluent_kafka/schema_registry/_async/serde.py | 6 +++--- src/confluent_kafka/schema_registry/_sync/serde.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/confluent_kafka/schema_registry/_async/serde.py b/src/confluent_kafka/schema_registry/_async/serde.py index 2bf615c90..69c72de5c 100644 --- a/src/confluent_kafka/schema_registry/_async/serde.py +++ b/src/confluent_kafka/schema_registry/_async/serde.py @@ -325,9 +325,9 @@ def configure_subject_name_strategy( ) return - # Default to topic_subject_name_strategy - self._subject_name_func = topic_subject_name_strategy - self._strategy_accepts_client = False + # Default to AsyncAssociatedNameStrategy (falls back to TOPIC when no associations found) + self._subject_name_func = AsyncAssociatedNameStrategy() + self._strategy_accepts_client = True async def _get_reader_schema(self, subject: str, fmt: Optional[str] = None) -> Optional[RegisteredSchema]: if self._use_schema_id is not None: diff --git a/src/confluent_kafka/schema_registry/_sync/serde.py b/src/confluent_kafka/schema_registry/_sync/serde.py index 4cad46d02..44eedeb6a 100644 --- a/src/confluent_kafka/schema_registry/_sync/serde.py +++ b/src/confluent_kafka/schema_registry/_sync/serde.py @@ -327,9 +327,9 @@ def configure_subject_name_strategy( raise ValueError(f"Unknown subject.name.strategy.type: {subject_name_strategy_type}") return - # Default to topic_subject_name_strategy - self._subject_name_func = topic_subject_name_strategy - self._strategy_accepts_client = False + # Default to AssociatedNameStrategy (falls back to TOPIC when no associations found) + self._subject_name_func = AssociatedNameStrategy() + self._strategy_accepts_client = True def _get_reader_schema(self, subject: str, fmt: Optional[str] = None) -> Optional[RegisteredSchema]: if self._use_schema_id is not None: From 508714560b8f7c70cd800b673078444b2b3c4346 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 27 Feb 2026 08:19:28 -0800 Subject: [PATCH 12/20] Fallback if sr client is null --- src/confluent_kafka/schema_registry/_async/serde.py | 4 ++++ src/confluent_kafka/schema_registry/_sync/serde.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/confluent_kafka/schema_registry/_async/serde.py b/src/confluent_kafka/schema_registry/_async/serde.py index 69c72de5c..54af8056d 100644 --- a/src/confluent_kafka/schema_registry/_async/serde.py +++ b/src/confluent_kafka/schema_registry/_async/serde.py @@ -99,6 +99,10 @@ async def _load_subject_name( kafka_cluster_id = None fallback_strategy = SubjectNameStrategyType.TOPIC # default fallback + # If no client is available, skip association lookup and use fallback directly + if schema_registry_client is None: + return topic_subject_name_strategy(ctx, record_name) + if conf is not None: kafka_cluster_id = conf.get(KAFKA_CLUSTER_ID) fallback_config = conf.get(FALLBACK_SUBJECT_NAME_STRATEGY_TYPE) diff --git a/src/confluent_kafka/schema_registry/_sync/serde.py b/src/confluent_kafka/schema_registry/_sync/serde.py index 44eedeb6a..26574215a 100644 --- a/src/confluent_kafka/schema_registry/_sync/serde.py +++ b/src/confluent_kafka/schema_registry/_sync/serde.py @@ -102,6 +102,10 @@ def _load_subject_name( kafka_cluster_id = None fallback_strategy = SubjectNameStrategyType.TOPIC # default fallback + # If no client is available, skip association lookup and use fallback directly + if schema_registry_client is None: + return topic_subject_name_strategy(ctx, record_name) + if conf is not None: kafka_cluster_id = conf.get(KAFKA_CLUSTER_ID) fallback_config = conf.get(FALLBACK_SUBJECT_NAME_STRATEGY_TYPE) From 18001a9520c36910a82d6fb888fae5b910cc1895 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 27 Feb 2026 08:24:25 -0800 Subject: [PATCH 13/20] fix tests --- tests/schema_registry/_async/test_avro.py | 11 ++++++++--- tests/schema_registry/_async/test_json.py | 3 +++ tests/schema_registry/_sync/test_avro.py | 12 +++++++++--- tests/schema_registry/_sync/test_json.py | 3 +++ tests/schema_registry/conftest.py | 9 +++++++++ 5 files changed, 32 insertions(+), 6 deletions(-) diff --git a/tests/schema_registry/_async/test_avro.py b/tests/schema_registry/_async/test_avro.py index 24b64d355..8e92bc2a1 100644 --- a/tests/schema_registry/_async/test_avro.py +++ b/tests/schema_registry/_async/test_avro.py @@ -193,16 +193,21 @@ async def test_avro_serializer_topic_record_subject_name_strategy_primitive(load assert ctx.headers is None -async def test_avro_serializer_subject_name_strategy_default(load_avsc): +async def test_avro_serializer_subject_name_strategy_default(mock_schema_registry, load_avsc): """ - Ensures record_subject_name_strategy returns the correct record name + Ensures the default subject name strategy returns the correct subject name """ conf = {'url': TEST_URL} test_client = AsyncSchemaRegistryClient(conf) test_serializer = await AsyncAvroSerializer(test_client, load_avsc('basic_schema.avsc')) ctx = SerializationContext('test_subj', MessageField.VALUE) - assert test_serializer._subject_name_func(ctx, test_serializer._schema_name) == 'test_subj-value' + if test_serializer._strategy_accepts_client: + result = await test_serializer._subject_name_func( + ctx, test_serializer._schema_name, test_client, test_serializer._subject_name_conf) + else: + result = test_serializer._subject_name_func(ctx, test_serializer._schema_name) + assert result == 'test_subj-value' async def test_avro_serializer_schema_loads_union(load_avsc): diff --git a/tests/schema_registry/_async/test_json.py b/tests/schema_registry/_async/test_json.py index 59c826c61..ae610c607 100644 --- a/tests/schema_registry/_async/test_json.py +++ b/tests/schema_registry/_async/test_json.py @@ -77,6 +77,7 @@ async def test_custom_json_encoder(): mock_schema_registry_client.register_schema_full_response.return_value = RegisteredSchema( schema_id=1, guid=None, schema=Schema(schema_str), subject="topic-name-value", version=1 ) + mock_schema_registry_client.get_associations_by_resource_name.return_value = [] # Use orjson.dumps as the custom encoder serializer = await AsyncJSONSerializer( @@ -136,6 +137,7 @@ async def test_custom_encoder_decoder_chain(): mock_schema_registry_client.register_schema_full_response.return_value = RegisteredSchema( schema_id=1, guid=None, schema=Schema(schema_str), subject="topic-name-value", version=1 ) + mock_schema_registry_client.get_associations_by_resource_name.return_value = [] def custom_encoder(obj): return orjson.dumps(obj, option=orjson.OPT_SORT_KEYS) @@ -176,6 +178,7 @@ async def test_custom_encoding_with_complex_data(): mock_schema_registry_client.register_schema_full_response.return_value = RegisteredSchema( schema_id=1, guid=None, schema=Schema(schema_str), subject="topic-name-value", version=1 ) + mock_schema_registry_client.get_associations_by_resource_name.return_value = [] def custom_encoder(obj): return json.dumps(obj, indent=2) diff --git a/tests/schema_registry/_sync/test_avro.py b/tests/schema_registry/_sync/test_avro.py index 7a9872285..006734157 100644 --- a/tests/schema_registry/_sync/test_avro.py +++ b/tests/schema_registry/_sync/test_avro.py @@ -189,16 +189,22 @@ def test_avro_serializer_topic_record_subject_name_strategy_primitive(load_avsc) assert ctx.headers is None -def test_avro_serializer_subject_name_strategy_default(load_avsc): +def test_avro_serializer_subject_name_strategy_default(mock_schema_registry, load_avsc): """ - Ensures record_subject_name_strategy returns the correct record name + Ensures the default subject name strategy returns the correct subject name """ conf = {'url': TEST_URL} test_client = SchemaRegistryClient(conf) test_serializer = AvroSerializer(test_client, load_avsc('basic_schema.avsc')) ctx = SerializationContext('test_subj', MessageField.VALUE) - assert test_serializer._subject_name_func(ctx, test_serializer._schema_name) == 'test_subj-value' + if test_serializer._strategy_accepts_client: + result = test_serializer._subject_name_func( + ctx, test_serializer._schema_name, test_client, test_serializer._subject_name_conf + ) + else: + result = test_serializer._subject_name_func(ctx, test_serializer._schema_name) + assert result == 'test_subj-value' def test_avro_serializer_schema_loads_union(load_avsc): diff --git a/tests/schema_registry/_sync/test_json.py b/tests/schema_registry/_sync/test_json.py index 7d3d912e1..b94622957 100644 --- a/tests/schema_registry/_sync/test_json.py +++ b/tests/schema_registry/_sync/test_json.py @@ -77,6 +77,7 @@ def test_custom_json_encoder(): mock_schema_registry_client.register_schema_full_response.return_value = RegisteredSchema( schema_id=1, guid=None, schema=Schema(schema_str), subject="topic-name-value", version=1 ) + mock_schema_registry_client.get_associations_by_resource_name.return_value = [] # Use orjson.dumps as the custom encoder serializer = JSONSerializer( @@ -136,6 +137,7 @@ def test_custom_encoder_decoder_chain(): mock_schema_registry_client.register_schema_full_response.return_value = RegisteredSchema( schema_id=1, guid=None, schema=Schema(schema_str), subject="topic-name-value", version=1 ) + mock_schema_registry_client.get_associations_by_resource_name.return_value = [] def custom_encoder(obj): return orjson.dumps(obj, option=orjson.OPT_SORT_KEYS) @@ -176,6 +178,7 @@ def test_custom_encoding_with_complex_data(): mock_schema_registry_client.register_schema_full_response.return_value = RegisteredSchema( schema_id=1, guid=None, schema=Schema(schema_str), subject="topic-name-value", version=1 ) + mock_schema_registry_client.get_associations_by_resource_name.return_value = [] def custom_encoder(obj): return json.dumps(obj, indent=2) diff --git a/tests/schema_registry/conftest.py b/tests/schema_registry/conftest.py index 9a93ae4d9..5e6bf76d5 100644 --- a/tests/schema_registry/conftest.py +++ b/tests/schema_registry/conftest.py @@ -148,6 +148,8 @@ def mock_schema_registry(): respx_mock.get(CONTEXTS_RE).mock(side_effect=get_contexts_callback) + respx_mock.get(ASSOCIATIONS_RESOURCES_RE).mock(side_effect=get_associations_resources_callback) + respx_mock.get(MODE_GLOBAL_RE).mock(side_effect=get_global_mode_callback) respx_mock.put(MODE_GLOBAL_RE).mock(side_effect=put_global_mode_callback) respx_mock.get(MODE_RE).mock(side_effect=get_mode_callback) @@ -192,6 +194,8 @@ def mock_schema_registry(): CONTEXTS_RE = re.compile(r"/contexts(\?.*)?$") +ASSOCIATIONS_RESOURCES_RE = re.compile(r"/associations/resources/(.*)/(.*?)(\?.*)?$") + # constants SCHEMA_ID = 47 VERSION = 3 @@ -268,6 +272,11 @@ def get_contexts_callback(request, route): return Response(200, json=['context1', 'context2']) +def get_associations_resources_callback(request, route): + COUNTER['GET'][request.url.path] += 1 + return Response(200, json=[]) + + def delete_subject_callback(request, route): COUNTER['DELETE'][request.url.path] += 1 From 72021342351cc099c4abffaa825a0711d0997a51 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 27 Feb 2026 11:56:54 -0800 Subject: [PATCH 14/20] Update docs --- src/confluent_kafka/schema_registry/_async/avro.py | 4 ++-- src/confluent_kafka/schema_registry/_async/json_schema.py | 4 ++-- src/confluent_kafka/schema_registry/_async/protobuf.py | 4 ++-- src/confluent_kafka/schema_registry/_sync/avro.py | 4 ++-- src/confluent_kafka/schema_registry/_sync/json_schema.py | 4 ++-- src/confluent_kafka/schema_registry/_sync/protobuf.py | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/confluent_kafka/schema_registry/_async/avro.py b/src/confluent_kafka/schema_registry/_async/avro.py index a5d0503c9..73b967dc9 100644 --- a/src/confluent_kafka/schema_registry/_async/avro.py +++ b/src/confluent_kafka/schema_registry/_async/avro.py @@ -128,7 +128,7 @@ class AsyncAvroSerializer(AsyncBaseSerializer): | | | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to TOPIC if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-----------------------------------+----------+--------------------------------------------------+ | ``subject.name.strategy.conf`` | dict | Configuration dictionary passed to strategies | @@ -503,7 +503,7 @@ class AsyncAvroDeserializer(AsyncBaseDeserializer): |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to TOPIC if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-----------------------------+----------+--------------------------------------------------+ | | | Configuration dictionary passed to strategies | diff --git a/src/confluent_kafka/schema_registry/_async/json_schema.py b/src/confluent_kafka/schema_registry/_async/json_schema.py index bff82f812..2a079e549 100644 --- a/src/confluent_kafka/schema_registry/_async/json_schema.py +++ b/src/confluent_kafka/schema_registry/_async/json_schema.py @@ -137,7 +137,7 @@ class AsyncJSONSerializer(AsyncBaseSerializer): |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to TOPIC if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-----------------------------+----------+----------------------------------------------------+ | | | Configuration dictionary passed to strategies | @@ -503,7 +503,7 @@ class AsyncJSONDeserializer(AsyncBaseDeserializer): |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to TOPIC if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-----------------------------+----------+----------------------------------------------------+ | | | Configuration dictionary passed to strategies | diff --git a/src/confluent_kafka/schema_registry/_async/protobuf.py b/src/confluent_kafka/schema_registry/_async/protobuf.py index 4a054c9ae..2125945bc 100644 --- a/src/confluent_kafka/schema_registry/_async/protobuf.py +++ b/src/confluent_kafka/schema_registry/_async/protobuf.py @@ -154,7 +154,7 @@ class AsyncProtobufSerializer(AsyncBaseSerializer): | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to TOPIC if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-------------------------------------+----------+------------------------------------------------------+ | | | Configuration dictionary passed to strategies | @@ -540,7 +540,7 @@ class AsyncProtobufDeserializer(AsyncBaseDeserializer): | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to TOPIC if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-------------------------------------+----------+------------------------------------------------------+ | | | Configuration dictionary passed to strategies | diff --git a/src/confluent_kafka/schema_registry/_sync/avro.py b/src/confluent_kafka/schema_registry/_sync/avro.py index 354d2bf8a..4525b13e6 100644 --- a/src/confluent_kafka/schema_registry/_sync/avro.py +++ b/src/confluent_kafka/schema_registry/_sync/avro.py @@ -124,7 +124,7 @@ class AvroSerializer(BaseSerializer): | | | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to TOPIC if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-----------------------------------+----------+--------------------------------------------------+ | ``subject.name.strategy.conf`` | dict | Configuration dictionary passed to strategies | @@ -498,7 +498,7 @@ class AvroDeserializer(BaseDeserializer): |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to TOPIC if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-----------------------------+----------+--------------------------------------------------+ | | | Configuration dictionary passed to strategies | diff --git a/src/confluent_kafka/schema_registry/_sync/json_schema.py b/src/confluent_kafka/schema_registry/_sync/json_schema.py index b331b844c..93da54919 100644 --- a/src/confluent_kafka/schema_registry/_sync/json_schema.py +++ b/src/confluent_kafka/schema_registry/_sync/json_schema.py @@ -135,7 +135,7 @@ class JSONSerializer(BaseSerializer): |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to TOPIC if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-----------------------------+----------+----------------------------------------------------+ | | | Configuration dictionary passed to strategies | @@ -500,7 +500,7 @@ class JSONDeserializer(BaseDeserializer): |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to TOPIC if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-----------------------------+----------+----------------------------------------------------+ | | | Configuration dictionary passed to strategies | diff --git a/src/confluent_kafka/schema_registry/_sync/protobuf.py b/src/confluent_kafka/schema_registry/_sync/protobuf.py index edc48d2ba..990e3e98c 100644 --- a/src/confluent_kafka/schema_registry/_sync/protobuf.py +++ b/src/confluent_kafka/schema_registry/_sync/protobuf.py @@ -152,7 +152,7 @@ class ProtobufSerializer(BaseSerializer): | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to TOPIC if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-------------------------------------+----------+------------------------------------------------------+ | | | Configuration dictionary passed to strategies | @@ -534,7 +534,7 @@ class ProtobufDeserializer(BaseDeserializer): | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to TOPIC if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-------------------------------------+----------+------------------------------------------------------+ | | | Configuration dictionary passed to strategies | From 069cbefc0548df822ea7992126db288ad6203378 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 27 Feb 2026 13:44:17 -0800 Subject: [PATCH 15/20] Minor renaming --- .../schema_registry/_async/serde.py | 18 +++++++++--------- .../schema_registry/_sync/serde.py | 18 +++++++++--------- .../schema_registry/_async/test_avro_serdes.py | 10 +++++----- .../schema_registry/_async/test_json_serdes.py | 10 +++++----- .../_async/test_proto_serdes.py | 10 +++++----- .../schema_registry/_sync/test_avro_serdes.py | 10 +++++----- .../schema_registry/_sync/test_json_serdes.py | 10 +++++----- .../schema_registry/_sync/test_proto_serdes.py | 10 +++++----- 8 files changed, 48 insertions(+), 48 deletions(-) diff --git a/src/confluent_kafka/schema_registry/_async/serde.py b/src/confluent_kafka/schema_registry/_async/serde.py index 54af8056d..35ed68e23 100644 --- a/src/confluent_kafka/schema_registry/_async/serde.py +++ b/src/confluent_kafka/schema_registry/_async/serde.py @@ -52,15 +52,15 @@ 'AsyncBaseSerializer', 'AsyncBaseDeserializer', 'KAFKA_CLUSTER_ID', - 'FALLBACK_SUBJECT_NAME_STRATEGY_TYPE', + 'FALLBACK_TYPE', ] log = logging.getLogger(__name__) -KAFKA_CLUSTER_ID = "kafka.cluster.id" +KAFKA_CLUSTER_ID = "subject.name.strategy.kafka.cluster.id" NAMESPACE_WILDCARD = "-" -FALLBACK_SUBJECT_NAME_STRATEGY_TYPE = "fallback.subject.name.strategy.type" +FALLBACK_TYPE = "subject.name.strategy.fallback.type" DEFAULT_CACHE_CAPACITY = 1000 @@ -105,7 +105,7 @@ async def _load_subject_name( if conf is not None: kafka_cluster_id = conf.get(KAFKA_CLUSTER_ID) - fallback_config = conf.get(FALLBACK_SUBJECT_NAME_STRATEGY_TYPE) + fallback_config = conf.get(FALLBACK_TYPE) if fallback_config is not None: if isinstance(fallback_config, SubjectNameStrategyType): fallback_strategy = fallback_config @@ -116,7 +116,7 @@ async def _load_subject_name( valid_fallbacks = [e.value for e in SubjectNameStrategyType if e != SubjectNameStrategyType.ASSOCIATED] raise ValueError( - f"Invalid value for {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE}: {fallback_config}. " + f"Invalid value for {FALLBACK_TYPE}: {fallback_config}. " f"Valid values are: {', '.join(valid_fallbacks)}" ) @@ -152,7 +152,7 @@ async def _load_subject_name( raise SerializationError(f"No associated subject found for topic {topic}") elif fallback_strategy == SubjectNameStrategyType.ASSOCIATED: raise ValueError( - f"Invalid value for {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE}: {fallback_strategy.value}. " + f"Invalid value for {FALLBACK_TYPE}: {fallback_strategy.value}. " f"ASSOCIATED cannot be used as a fallback strategy." ) @@ -177,7 +177,7 @@ async def __call__( If more than one subject is returned from the query, a SerializationError will be raised. If no subjects are returned from the query, then the behavior will fall back to topic_subject_name_strategy, unless the configuration property - "fallback.subject.name.strategy.type" is set to "RECORD", "TOPIC_RECORD", or "NONE". + "subject.name.strategy.fallback.type" is set to "RECORD", "TOPIC_RECORD", or "NONE". Results are cached using an LRU cache to avoid repeated API calls. @@ -190,8 +190,8 @@ async def __call__( schema_registry_client (AsyncSchemaRegistryClient): AsyncSchemaRegistryClient instance. conf (Optional[dict]): Configuration dictionary. Supports: - - "kafka.cluster.id": Kafka cluster ID to use as resource namespace. - - "fallback.subject.name.strategy.type": Fallback strategy when no + - "subject.name.strategy.kafka.cluster.id": Kafka cluster ID to use as resource namespace. + - "subject.name.strategy.fallback.type": Fallback strategy when no associations are found. One of "TOPIC", "RECORD", "TOPIC_RECORD", or "NONE". Defaults to "TOPIC". diff --git a/src/confluent_kafka/schema_registry/_sync/serde.py b/src/confluent_kafka/schema_registry/_sync/serde.py index 26574215a..b511cc0e3 100644 --- a/src/confluent_kafka/schema_registry/_sync/serde.py +++ b/src/confluent_kafka/schema_registry/_sync/serde.py @@ -57,15 +57,15 @@ 'BaseSerializer', 'BaseDeserializer', 'KAFKA_CLUSTER_ID', - 'FALLBACK_SUBJECT_NAME_STRATEGY_TYPE', + 'FALLBACK_TYPE', ] log = logging.getLogger(__name__) -KAFKA_CLUSTER_ID = "kafka.cluster.id" +KAFKA_CLUSTER_ID = "subject.name.strategy.kafka.cluster.id" NAMESPACE_WILDCARD = "-" -FALLBACK_SUBJECT_NAME_STRATEGY_TYPE = "fallback.subject.name.strategy.type" +FALLBACK_TYPE = "subject.name.strategy.fallback.type" DEFAULT_CACHE_CAPACITY = 1000 @@ -108,7 +108,7 @@ def _load_subject_name( if conf is not None: kafka_cluster_id = conf.get(KAFKA_CLUSTER_ID) - fallback_config = conf.get(FALLBACK_SUBJECT_NAME_STRATEGY_TYPE) + fallback_config = conf.get(FALLBACK_TYPE) if fallback_config is not None: if isinstance(fallback_config, SubjectNameStrategyType): fallback_strategy = fallback_config @@ -120,7 +120,7 @@ def _load_subject_name( e.value for e in SubjectNameStrategyType if e != SubjectNameStrategyType.ASSOCIATED ] raise ValueError( - f"Invalid value for {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE}: {fallback_config}. " + f"Invalid value for {FALLBACK_TYPE}: {fallback_config}. " f"Valid values are: {', '.join(valid_fallbacks)}" ) @@ -156,7 +156,7 @@ def _load_subject_name( raise SerializationError(f"No associated subject found for topic {topic}") elif fallback_strategy == SubjectNameStrategyType.ASSOCIATED: raise ValueError( - f"Invalid value for {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE}: {fallback_strategy.value}. " + f"Invalid value for {FALLBACK_TYPE}: {fallback_strategy.value}. " f"ASSOCIATED cannot be used as a fallback strategy." ) @@ -181,7 +181,7 @@ def __call__( If more than one subject is returned from the query, a SerializationError will be raised. If no subjects are returned from the query, then the behavior will fall back to topic_subject_name_strategy, unless the configuration property - "fallback.subject.name.strategy.type" is set to "RECORD", "TOPIC_RECORD", or "NONE". + "subject.name.strategy.fallback.type" is set to "RECORD", "TOPIC_RECORD", or "NONE". Results are cached using an LRU cache to avoid repeated API calls. @@ -194,8 +194,8 @@ def __call__( schema_registry_client (SchemaRegistryClient): SchemaRegistryClient instance. conf (Optional[dict]): Configuration dictionary. Supports: - - "kafka.cluster.id": Kafka cluster ID to use as resource namespace. - - "fallback.subject.name.strategy.type": Fallback strategy when no + - "subject.name.strategy.kafka.cluster.id": Kafka cluster ID to use as resource namespace. + - "subject.name.strategy.fallback.type": Fallback strategy when no associations are found. One of "TOPIC", "RECORD", "TOPIC_RECORD", or "NONE". Defaults to "TOPIC". diff --git a/tests/schema_registry/_async/test_avro_serdes.py b/tests/schema_registry/_async/test_avro_serdes.py index 623ae2c9f..4085bc979 100644 --- a/tests/schema_registry/_async/test_avro_serdes.py +++ b/tests/schema_registry/_async/test_avro_serdes.py @@ -36,7 +36,7 @@ from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry._async.serde import ( KAFKA_CLUSTER_ID, - FALLBACK_SUBJECT_NAME_STRATEGY_TYPE, + FALLBACK_TYPE, ) from confluent_kafka.schema_registry.avro import AsyncAvroDeserializer, AsyncAvroSerializer from confluent_kafka.schema_registry.rule_registry import RuleOverride, RuleRegistry @@ -2822,7 +2822,7 @@ async def test_associated_name_strategy_fallback_to_record(): ser_conf = { 'auto.register.schemas': True, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.RECORD}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.RECORD}, } ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -2857,7 +2857,7 @@ async def test_associated_name_strategy_fallback_to_topic_record(): ser_conf = { 'auto.register.schemas': True, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, } ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -2892,7 +2892,7 @@ async def test_associated_name_strategy_fallback_none_raises(): ser_conf = { 'auto.register.schemas': True, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: "NONE"}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: "NONE"}, } ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -2951,7 +2951,7 @@ async def test_associated_name_strategy_multiple_associations_raises(): async def test_associated_name_strategy_with_kafka_cluster_id(): - """Test that kafka.cluster.id config is used as resource namespace""" + """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" conf = {'url': _BASE_URL} client = AsyncSchemaRegistryClient.new_client(conf) diff --git a/tests/schema_registry/_async/test_json_serdes.py b/tests/schema_registry/_async/test_json_serdes.py index b89157da5..4826b74c9 100644 --- a/tests/schema_registry/_async/test_json_serdes.py +++ b/tests/schema_registry/_async/test_json_serdes.py @@ -50,7 +50,7 @@ ServerConfig, ) from confluent_kafka.schema_registry._async.serde import ( - FALLBACK_SUBJECT_NAME_STRATEGY_TYPE, + FALLBACK_TYPE, KAFKA_CLUSTER_ID, ) from confluent_kafka.schema_registry.common.schema_registry_client import ( @@ -1570,7 +1570,7 @@ async def test_json_associated_name_strategy_fallback_to_record(): ser_conf = { 'auto.register.schemas': True, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.RECORD}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.RECORD}, } ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -1593,7 +1593,7 @@ async def test_json_associated_name_strategy_fallback_to_topic_record(): ser_conf = { 'auto.register.schemas': True, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, } ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -1616,7 +1616,7 @@ async def test_json_associated_name_strategy_fallback_none_raises(): ser_conf = { 'auto.register.schemas': True, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: "NONE"}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: "NONE"}, } ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -1664,7 +1664,7 @@ async def test_json_associated_name_strategy_multiple_associations_raises(): async def test_json_associated_name_strategy_with_kafka_cluster_id(): - """Test that kafka.cluster.id config is used as resource namespace""" + """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" conf = {'url': _BASE_URL} client = AsyncSchemaRegistryClient.new_client(conf) diff --git a/tests/schema_registry/_async/test_proto_serdes.py b/tests/schema_registry/_async/test_proto_serdes.py index 8536e9f92..e96f42db7 100644 --- a/tests/schema_registry/_async/test_proto_serdes.py +++ b/tests/schema_registry/_async/test_proto_serdes.py @@ -47,7 +47,7 @@ ServerConfig, ) from confluent_kafka.schema_registry._async.serde import ( - FALLBACK_SUBJECT_NAME_STRATEGY_TYPE, + FALLBACK_TYPE, KAFKA_CLUSTER_ID, ) from confluent_kafka.schema_registry.common.schema_registry_client import ( @@ -739,7 +739,7 @@ async def test_associated_name_strategy_fallback_to_record(): 'auto.register.schemas': True, 'use.deprecated.format': False, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.RECORD}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.RECORD}, } ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -765,7 +765,7 @@ async def test_associated_name_strategy_fallback_to_topic_record(): 'auto.register.schemas': True, 'use.deprecated.format': False, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, } ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -789,7 +789,7 @@ async def test_associated_name_strategy_fallback_none_raises(): 'auto.register.schemas': True, 'use.deprecated.format': False, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: "NONE"}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: "NONE"}, } ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -839,7 +839,7 @@ async def test_associated_name_strategy_multiple_associations_raises(): async def test_associated_name_strategy_with_kafka_cluster_id(): - """Test that kafka.cluster.id config is used as resource namespace""" + """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" conf = {'url': _BASE_URL} client = AsyncSchemaRegistryClient.new_client(conf) obj = example_pb2.Author( diff --git a/tests/schema_registry/_sync/test_avro_serdes.py b/tests/schema_registry/_sync/test_avro_serdes.py index f57a69102..a4f93dcb8 100644 --- a/tests/schema_registry/_sync/test_avro_serdes.py +++ b/tests/schema_registry/_sync/test_avro_serdes.py @@ -30,7 +30,7 @@ header_schema_id_serializer, ) from confluent_kafka.schema_registry._sync.serde import ( - FALLBACK_SUBJECT_NAME_STRATEGY_TYPE, + FALLBACK_TYPE, KAFKA_CLUSTER_ID, ) from confluent_kafka.schema_registry.avro import AvroDeserializer, AvroSerializer @@ -2822,7 +2822,7 @@ def test_associated_name_strategy_fallback_to_record(): ser_conf = { 'auto.register.schemas': True, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.RECORD}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.RECORD}, } ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -2857,7 +2857,7 @@ def test_associated_name_strategy_fallback_to_topic_record(): ser_conf = { 'auto.register.schemas': True, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, } ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -2892,7 +2892,7 @@ def test_associated_name_strategy_fallback_none_raises(): ser_conf = { 'auto.register.schemas': True, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: "NONE"}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: "NONE"}, } ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -2951,7 +2951,7 @@ def test_associated_name_strategy_multiple_associations_raises(): def test_associated_name_strategy_with_kafka_cluster_id(): - """Test that kafka.cluster.id config is used as resource namespace""" + """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" conf = {'url': _BASE_URL} client = SchemaRegistryClient.new_client(conf) diff --git a/tests/schema_registry/_sync/test_json_serdes.py b/tests/schema_registry/_sync/test_json_serdes.py index e95e2842e..dcc9dfddb 100644 --- a/tests/schema_registry/_sync/test_json_serdes.py +++ b/tests/schema_registry/_sync/test_json_serdes.py @@ -28,7 +28,7 @@ header_schema_id_serializer, ) from confluent_kafka.schema_registry._sync.serde import ( - FALLBACK_SUBJECT_NAME_STRATEGY_TYPE, + FALLBACK_TYPE, KAFKA_CLUSTER_ID, ) from confluent_kafka.schema_registry.common.schema_registry_client import ( @@ -1572,7 +1572,7 @@ def test_json_associated_name_strategy_fallback_to_record(): ser_conf = { 'auto.register.schemas': True, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.RECORD}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.RECORD}, } ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -1595,7 +1595,7 @@ def test_json_associated_name_strategy_fallback_to_topic_record(): ser_conf = { 'auto.register.schemas': True, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, } ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -1618,7 +1618,7 @@ def test_json_associated_name_strategy_fallback_none_raises(): ser_conf = { 'auto.register.schemas': True, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: "NONE"}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: "NONE"}, } ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -1666,7 +1666,7 @@ def test_json_associated_name_strategy_multiple_associations_raises(): def test_json_associated_name_strategy_with_kafka_cluster_id(): - """Test that kafka.cluster.id config is used as resource namespace""" + """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" conf = {'url': _BASE_URL} client = SchemaRegistryClient.new_client(conf) diff --git a/tests/schema_registry/_sync/test_proto_serdes.py b/tests/schema_registry/_sync/test_proto_serdes.py index f1968770b..5c49b9050 100644 --- a/tests/schema_registry/_sync/test_proto_serdes.py +++ b/tests/schema_registry/_sync/test_proto_serdes.py @@ -25,7 +25,7 @@ from confluent_kafka.schema_registry._sync.protobuf import ProtobufDeserializer, ProtobufSerializer from confluent_kafka.schema_registry._sync.schema_registry_client import SchemaRegistryClient from confluent_kafka.schema_registry._sync.serde import ( - FALLBACK_SUBJECT_NAME_STRATEGY_TYPE, + FALLBACK_TYPE, KAFKA_CLUSTER_ID, ) from confluent_kafka.schema_registry.common.schema_registry_client import ( @@ -739,7 +739,7 @@ def test_associated_name_strategy_fallback_to_record(): 'auto.register.schemas': True, 'use.deprecated.format': False, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.RECORD}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.RECORD}, } ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -765,7 +765,7 @@ def test_associated_name_strategy_fallback_to_topic_record(): 'auto.register.schemas': True, 'use.deprecated.format': False, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: SubjectNameStrategyType.TOPIC_RECORD}, } ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -789,7 +789,7 @@ def test_associated_name_strategy_fallback_none_raises(): 'auto.register.schemas': True, 'use.deprecated.format': False, 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - 'subject.name.strategy.conf': {FALLBACK_SUBJECT_NAME_STRATEGY_TYPE: "NONE"}, + 'subject.name.strategy.conf': {FALLBACK_TYPE: "NONE"}, } ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) @@ -839,7 +839,7 @@ def test_associated_name_strategy_multiple_associations_raises(): def test_associated_name_strategy_with_kafka_cluster_id(): - """Test that kafka.cluster.id config is used as resource namespace""" + """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" conf = {'url': _BASE_URL} client = SchemaRegistryClient.new_client(conf) obj = example_pb2.Author( From 4dd466b5d617bd15669d08a128358f15d9be751e Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Tue, 3 Mar 2026 16:04:56 -0800 Subject: [PATCH 16/20] Minor test cleanup --- .../_async/schema_registry_client.py | 2 + .../_sync/schema_registry_client.py | 2 + .../_async/test_avro_serdes.py | 71 ++++++------------- .../schema_registry/_sync/test_avro_serdes.py | 71 ++++++------------- 4 files changed, 50 insertions(+), 96 deletions(-) diff --git a/src/confluent_kafka/schema_registry/_async/schema_registry_client.py b/src/confluent_kafka/schema_registry/_async/schema_registry_client.py index 91b148ea4..023efb36e 100644 --- a/src/confluent_kafka/schema_registry/_async/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_async/schema_registry_client.py @@ -511,6 +511,8 @@ async def send_request( response = await self.send_http_request(base_url, url, method, headers, body_str, query) if is_success(response.status_code): + if response.status_code == 204 or not response.content: + return None return response.json() if not is_retriable(response.status_code) or i == len(self.base_urls) - 1: diff --git a/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py b/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py index c440d439e..1f01e46c4 100644 --- a/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_sync/schema_registry_client.py @@ -508,6 +508,8 @@ def send_request(self, url: str, method: str, body: Optional[dict] = None, query response = self.send_http_request(base_url, url, method, headers, body_str, query) if is_success(response.status_code): + if response.status_code == 204 or not response.content: + return None return response.json() if not is_retriable(response.status_code) or i == len(self.base_urls) - 1: diff --git a/tests/schema_registry/_async/test_avro_serdes.py b/tests/schema_registry/_async/test_avro_serdes.py index 4085bc979..7464b3d2c 100644 --- a/tests/schema_registry/_async/test_avro_serdes.py +++ b/tests/schema_registry/_async/test_avro_serdes.py @@ -2695,6 +2695,10 @@ async def test_associated_name_strategy_with_association(): AssociationCreateOrUpdateInfo( subject="my-custom-subject-value", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), ) ], ) @@ -2718,6 +2722,8 @@ async def test_associated_name_strategy_with_association(): registered_schema = await client.get_latest_version("my-custom-subject-value") assert registered_schema is not None + await client.delete_associations(resource_id="mock-resource-id-1", cascade_lifecycle=True) + async def test_associated_name_strategy_with_key_association(): """Test that AsyncAssociatedNameStrategy returns subject for key""" @@ -2744,6 +2750,10 @@ async def test_associated_name_strategy_with_key_association(): AssociationCreateOrUpdateInfo( subject="my-key-subject", association_type="key", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), ) ], ) @@ -2767,6 +2777,8 @@ async def test_associated_name_strategy_with_key_association(): registered_schema = await client.get_latest_version("my-key-subject") assert registered_schema is not None + await client.delete_associations(resource_id="mock-resource-id-2", cascade_lifecycle=True) + async def test_associated_name_strategy_fallback_to_topic(): """Test fallback to topic_subject_name_strategy when no association""" @@ -2903,53 +2915,6 @@ async def test_associated_name_strategy_fallback_none_raises(): assert "No associated subject found" in str(exc_info.value) -async def test_associated_name_strategy_multiple_associations_raises(): - """Test that multiple associations raise an error""" - conf = {'url': _BASE_URL} - client = AsyncSchemaRegistryClient.new_client(conf) - - # Define schema - schema = { - 'type': 'record', - 'name': 'TestRecord', - 'fields': [ - {'name': 'value', 'type': 'string'}, - ], - } - obj = {'value': 'test'} - - # Add multiple associations for the same topic/value - request = AssociationCreateOrUpdateRequest( - resource_name=_TOPIC, - resource_namespace="-", - resource_id="mock-resource-id-3", - resource_type="topic", - associations=[ - AssociationCreateOrUpdateInfo( - subject="subject1", - association_type="value", - ), - AssociationCreateOrUpdateInfo( - subject="subject2", - association_type="value", - ), - ], - ) - await client.create_association(request) - - ser_conf = { - 'auto.register.schemas': True, - 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - } - ser = await AsyncAvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) - ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) - - with pytest.raises(SerializationError) as exc_info: - await ser(obj, ser_ctx) - - assert "Multiple associated subjects found" in str(exc_info.value) - - async def test_associated_name_strategy_with_kafka_cluster_id(): """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" conf = {'url': _BASE_URL} @@ -2975,6 +2940,10 @@ async def test_associated_name_strategy_with_kafka_cluster_id(): AssociationCreateOrUpdateInfo( subject="cluster-specific-subject", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), ) ], ) @@ -2999,6 +2968,8 @@ async def test_associated_name_strategy_with_kafka_cluster_id(): registered_schema = await client.get_latest_version("cluster-specific-subject") assert registered_schema is not None + await client.delete_associations(resource_id="mock-resource-id-4", cascade_lifecycle=True) + async def test_associated_name_strategy_caching(): """Test that results are cached within a strategy instance and serializer works with caching""" @@ -3024,6 +2995,10 @@ async def test_associated_name_strategy_caching(): AssociationCreateOrUpdateInfo( subject="cached-subject", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), ) ], ) @@ -3051,7 +3026,7 @@ async def test_associated_name_strategy_caching(): assert obj1 == result1 # Delete associations (but serializer should still work due to caching) - await client.delete_associations("mock-resource-id-5") + await client.delete_associations(resource_id="mock-resource-id-5", cascade_lifecycle=True) # Second serialization should still work (schema already registered) obj2 = {'count': 2} diff --git a/tests/schema_registry/_sync/test_avro_serdes.py b/tests/schema_registry/_sync/test_avro_serdes.py index a4f93dcb8..c244756bf 100644 --- a/tests/schema_registry/_sync/test_avro_serdes.py +++ b/tests/schema_registry/_sync/test_avro_serdes.py @@ -2695,6 +2695,10 @@ def test_associated_name_strategy_with_association(): AssociationCreateOrUpdateInfo( subject="my-custom-subject-value", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), ) ], ) @@ -2718,6 +2722,8 @@ def test_associated_name_strategy_with_association(): registered_schema = client.get_latest_version("my-custom-subject-value") assert registered_schema is not None + client.delete_associations(resource_id="mock-resource-id-1", cascade_lifecycle=True) + def test_associated_name_strategy_with_key_association(): """Test that AssociatedNameStrategy returns subject for key""" @@ -2744,6 +2750,10 @@ def test_associated_name_strategy_with_key_association(): AssociationCreateOrUpdateInfo( subject="my-key-subject", association_type="key", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), ) ], ) @@ -2767,6 +2777,8 @@ def test_associated_name_strategy_with_key_association(): registered_schema = client.get_latest_version("my-key-subject") assert registered_schema is not None + client.delete_associations(resource_id="mock-resource-id-2", cascade_lifecycle=True) + def test_associated_name_strategy_fallback_to_topic(): """Test fallback to topic_subject_name_strategy when no association""" @@ -2903,53 +2915,6 @@ def test_associated_name_strategy_fallback_none_raises(): assert "No associated subject found" in str(exc_info.value) -def test_associated_name_strategy_multiple_associations_raises(): - """Test that multiple associations raise an error""" - conf = {'url': _BASE_URL} - client = SchemaRegistryClient.new_client(conf) - - # Define schema - schema = { - 'type': 'record', - 'name': 'TestRecord', - 'fields': [ - {'name': 'value', 'type': 'string'}, - ], - } - obj = {'value': 'test'} - - # Add multiple associations for the same topic/value - request = AssociationCreateOrUpdateRequest( - resource_name=_TOPIC, - resource_namespace="-", - resource_id="mock-resource-id-3", - resource_type="topic", - associations=[ - AssociationCreateOrUpdateInfo( - subject="subject1", - association_type="value", - ), - AssociationCreateOrUpdateInfo( - subject="subject2", - association_type="value", - ), - ], - ) - client.create_association(request) - - ser_conf = { - 'auto.register.schemas': True, - 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - } - ser = AvroSerializer(client, schema_str=json.dumps(schema), conf=ser_conf) - ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) - - with pytest.raises(SerializationError) as exc_info: - ser(obj, ser_ctx) - - assert "Multiple associated subjects found" in str(exc_info.value) - - def test_associated_name_strategy_with_kafka_cluster_id(): """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" conf = {'url': _BASE_URL} @@ -2975,6 +2940,10 @@ def test_associated_name_strategy_with_kafka_cluster_id(): AssociationCreateOrUpdateInfo( subject="cluster-specific-subject", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), ) ], ) @@ -2999,6 +2968,8 @@ def test_associated_name_strategy_with_kafka_cluster_id(): registered_schema = client.get_latest_version("cluster-specific-subject") assert registered_schema is not None + client.delete_associations(resource_id="mock-resource-id-4", cascade_lifecycle=True) + def test_associated_name_strategy_caching(): """Test that results are cached within a strategy instance and serializer works with caching""" @@ -3024,6 +2995,10 @@ def test_associated_name_strategy_caching(): AssociationCreateOrUpdateInfo( subject="cached-subject", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=json.dumps(schema), + ), ) ], ) @@ -3051,7 +3026,7 @@ def test_associated_name_strategy_caching(): assert obj1 == result1 # Delete associations (but serializer should still work due to caching) - client.delete_associations("mock-resource-id-5") + client.delete_associations(resource_id="mock-resource-id-5", cascade_lifecycle=True) # Second serialization should still work (schema already registered) obj2 = {'count': 2} From f4d1c79c3eb456164509e79c32eb3ff4b103a06b Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Tue, 3 Mar 2026 16:20:35 -0800 Subject: [PATCH 17/20] Minor test cleanup --- .../_async/test_json_serdes.py | 60 +++++++----------- .../_async/test_proto_serdes.py | 62 +++++++------------ .../schema_registry/_sync/test_json_serdes.py | 60 +++++++----------- .../_sync/test_proto_serdes.py | 62 +++++++------------ 4 files changed, 92 insertions(+), 152 deletions(-) diff --git a/tests/schema_registry/_async/test_json_serdes.py b/tests/schema_registry/_async/test_json_serdes.py index 4826b74c9..bf8e81838 100644 --- a/tests/schema_registry/_async/test_json_serdes.py +++ b/tests/schema_registry/_async/test_json_serdes.py @@ -1485,6 +1485,10 @@ async def test_json_associated_name_strategy_with_association(): AssociationCreateOrUpdateInfo( subject="my-custom-subject-value", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), ) ], ) @@ -1505,6 +1509,8 @@ async def test_json_associated_name_strategy_with_association(): registered_schema = await client.get_latest_version("my-custom-subject-value") assert registered_schema is not None + await client.delete_associations(resource_id="json-resource-id-1", cascade_lifecycle=True) + async def test_json_associated_name_strategy_with_key_association(): """Test that AssociatedNameStrategy returns subject for key""" @@ -1520,6 +1526,10 @@ async def test_json_associated_name_strategy_with_key_association(): AssociationCreateOrUpdateInfo( subject="my-key-subject", association_type="key", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), ) ], ) @@ -1540,6 +1550,8 @@ async def test_json_associated_name_strategy_with_key_association(): registered_schema = await client.get_latest_version("my-key-subject") assert registered_schema is not None + await client.delete_associations(resource_id="json-resource-id-2", cascade_lifecycle=True) + async def test_json_associated_name_strategy_fallback_to_topic(): """Test fallback to topic_subject_name_strategy when no association""" @@ -1627,42 +1639,6 @@ async def test_json_associated_name_strategy_fallback_none_raises(): assert "No associated subject found" in str(exc_info.value) -async def test_json_associated_name_strategy_multiple_associations_raises(): - """Test that multiple associations raise an error""" - conf = {'url': _BASE_URL} - client = AsyncSchemaRegistryClient.new_client(conf) - - request = AssociationCreateOrUpdateRequest( - resource_name=_TOPIC, - resource_namespace="-", - resource_id="json-resource-id-3", - resource_type="topic", - associations=[ - AssociationCreateOrUpdateInfo( - subject="json-subject-1", - association_type="value", - ), - AssociationCreateOrUpdateInfo( - subject="json-subject-2", - association_type="value", - ), - ], - ) - await client.create_association(request) - - ser_conf = { - 'auto.register.schemas': True, - 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - } - ser = await AsyncJSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) - ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) - - with pytest.raises(SerializationError) as exc_info: - await ser(_JSON_OBJ, ser_ctx) - - assert "Multiple associated subjects found" in str(exc_info.value) - - async def test_json_associated_name_strategy_with_kafka_cluster_id(): """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" conf = {'url': _BASE_URL} @@ -1677,6 +1653,10 @@ async def test_json_associated_name_strategy_with_kafka_cluster_id(): AssociationCreateOrUpdateInfo( subject="cluster-specific-json-subject", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), ) ], ) @@ -1698,6 +1678,8 @@ async def test_json_associated_name_strategy_with_kafka_cluster_id(): registered_schema = await client.get_latest_version("cluster-specific-json-subject") assert registered_schema is not None + await client.delete_associations(resource_id="json-resource-id-4", cascade_lifecycle=True) + async def test_json_associated_name_strategy_caching(): """Test that results are cached within a strategy instance and serializer works with caching""" @@ -1713,6 +1695,10 @@ async def test_json_associated_name_strategy_caching(): AssociationCreateOrUpdateInfo( subject="json-cached-subject", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), ) ], ) @@ -1736,7 +1722,7 @@ async def test_json_associated_name_strategy_caching(): assert obj1 == result1 # Delete associations (but serializer should still work due to caching) - await client.delete_associations("json-resource-id-5") + await client.delete_associations(resource_id="json-resource-id-5", cascade_lifecycle=True) obj2 = {"name": "Kafka", "id": 2} obj_bytes2 = await ser(obj2, ser_ctx) diff --git a/tests/schema_registry/_async/test_proto_serdes.py b/tests/schema_registry/_async/test_proto_serdes.py index e96f42db7..f2ca0dd53 100644 --- a/tests/schema_registry/_async/test_proto_serdes.py +++ b/tests/schema_registry/_async/test_proto_serdes.py @@ -643,6 +643,10 @@ async def test_associated_name_strategy_with_association(): AssociationCreateOrUpdateInfo( subject="my-custom-subject-value", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), ) ], ) @@ -664,6 +668,8 @@ async def test_associated_name_strategy_with_association(): registered_schema = await client.get_latest_version("my-custom-subject-value") assert registered_schema is not None + await client.delete_associations(resource_id="proto-resource-id-1", cascade_lifecycle=True) + async def test_associated_name_strategy_with_key_association(): """Test that AssociatedNameStrategy returns subject for key""" @@ -680,6 +686,10 @@ async def test_associated_name_strategy_with_key_association(): AssociationCreateOrUpdateInfo( subject="my-key-subject", association_type="key", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), ) ], ) @@ -701,6 +711,8 @@ async def test_associated_name_strategy_with_key_association(): registered_schema = await client.get_latest_version("my-key-subject") assert registered_schema is not None + await client.delete_associations(resource_id="proto-resource-id-2", cascade_lifecycle=True) + async def test_associated_name_strategy_fallback_to_topic(): """Test fallback to topic_subject_name_strategy when no association""" @@ -800,44 +812,6 @@ async def test_associated_name_strategy_fallback_none_raises(): assert "No associated subject found" in str(exc_info.value) -async def test_associated_name_strategy_multiple_associations_raises(): - """Test that multiple associations raise an error""" - conf = {'url': _BASE_URL} - client = AsyncSchemaRegistryClient.new_client(conf) - obj = example_pb2.Author(name='Kafka', id=2) - - request = AssociationCreateOrUpdateRequest( - resource_name=_TOPIC, - resource_namespace="-", - resource_id="proto-resource-id-3", - resource_type="topic", - associations=[ - AssociationCreateOrUpdateInfo( - subject="proto-subject-1", - association_type="value", - ), - AssociationCreateOrUpdateInfo( - subject="proto-subject-2", - association_type="value", - ), - ], - ) - await client.create_association(request) - - ser_conf = { - 'auto.register.schemas': True, - 'use.deprecated.format': False, - 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - } - ser = await AsyncProtobufSerializer(example_pb2.Author, client, conf=ser_conf) - ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) - - with pytest.raises(SerializationError) as exc_info: - await ser(obj, ser_ctx) - - assert "Multiple associated subjects found" in str(exc_info.value) - - async def test_associated_name_strategy_with_kafka_cluster_id(): """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" conf = {'url': _BASE_URL} @@ -855,6 +829,10 @@ async def test_associated_name_strategy_with_kafka_cluster_id(): AssociationCreateOrUpdateInfo( subject="cluster-specific-proto-subject", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), ) ], ) @@ -877,6 +855,8 @@ async def test_associated_name_strategy_with_kafka_cluster_id(): registered_schema = await client.get_latest_version("cluster-specific-proto-subject") assert registered_schema is not None + await client.delete_associations(resource_id="proto-resource-id-4", cascade_lifecycle=True) + async def test_associated_name_strategy_caching(): """Test that results are cached within a strategy instance and serializer works with caching""" @@ -892,6 +872,10 @@ async def test_associated_name_strategy_caching(): AssociationCreateOrUpdateInfo( subject="proto-cached-subject", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), ) ], ) @@ -916,7 +900,7 @@ async def test_associated_name_strategy_caching(): assert obj1 == result1 # Delete associations (but serializer should still work due to caching) - await client.delete_associations("proto-resource-id-5") + await client.delete_associations(resource_id="proto-resource-id-5", cascade_lifecycle=True) obj2 = example_pb2.Author(name='Kafka', id=2) obj_bytes2 = await ser(obj2, ser_ctx) diff --git a/tests/schema_registry/_sync/test_json_serdes.py b/tests/schema_registry/_sync/test_json_serdes.py index dcc9dfddb..485c7db7f 100644 --- a/tests/schema_registry/_sync/test_json_serdes.py +++ b/tests/schema_registry/_sync/test_json_serdes.py @@ -1487,6 +1487,10 @@ def test_json_associated_name_strategy_with_association(): AssociationCreateOrUpdateInfo( subject="my-custom-subject-value", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), ) ], ) @@ -1507,6 +1511,8 @@ def test_json_associated_name_strategy_with_association(): registered_schema = client.get_latest_version("my-custom-subject-value") assert registered_schema is not None + client.delete_associations(resource_id="json-resource-id-1", cascade_lifecycle=True) + def test_json_associated_name_strategy_with_key_association(): """Test that AssociatedNameStrategy returns subject for key""" @@ -1522,6 +1528,10 @@ def test_json_associated_name_strategy_with_key_association(): AssociationCreateOrUpdateInfo( subject="my-key-subject", association_type="key", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), ) ], ) @@ -1542,6 +1552,8 @@ def test_json_associated_name_strategy_with_key_association(): registered_schema = client.get_latest_version("my-key-subject") assert registered_schema is not None + client.delete_associations(resource_id="json-resource-id-2", cascade_lifecycle=True) + def test_json_associated_name_strategy_fallback_to_topic(): """Test fallback to topic_subject_name_strategy when no association""" @@ -1629,42 +1641,6 @@ def test_json_associated_name_strategy_fallback_none_raises(): assert "No associated subject found" in str(exc_info.value) -def test_json_associated_name_strategy_multiple_associations_raises(): - """Test that multiple associations raise an error""" - conf = {'url': _BASE_URL} - client = SchemaRegistryClient.new_client(conf) - - request = AssociationCreateOrUpdateRequest( - resource_name=_TOPIC, - resource_namespace="-", - resource_id="json-resource-id-3", - resource_type="topic", - associations=[ - AssociationCreateOrUpdateInfo( - subject="json-subject-1", - association_type="value", - ), - AssociationCreateOrUpdateInfo( - subject="json-subject-2", - association_type="value", - ), - ], - ) - client.create_association(request) - - ser_conf = { - 'auto.register.schemas': True, - 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - } - ser = JSONSerializer(_JSON_SCHEMA, client, conf=ser_conf) - ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) - - with pytest.raises(SerializationError) as exc_info: - ser(_JSON_OBJ, ser_ctx) - - assert "Multiple associated subjects found" in str(exc_info.value) - - def test_json_associated_name_strategy_with_kafka_cluster_id(): """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" conf = {'url': _BASE_URL} @@ -1679,6 +1655,10 @@ def test_json_associated_name_strategy_with_kafka_cluster_id(): AssociationCreateOrUpdateInfo( subject="cluster-specific-json-subject", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), ) ], ) @@ -1700,6 +1680,8 @@ def test_json_associated_name_strategy_with_kafka_cluster_id(): registered_schema = client.get_latest_version("cluster-specific-json-subject") assert registered_schema is not None + client.delete_associations(resource_id="json-resource-id-4", cascade_lifecycle=True) + def test_json_associated_name_strategy_caching(): """Test that results are cached within a strategy instance and serializer works with caching""" @@ -1715,6 +1697,10 @@ def test_json_associated_name_strategy_caching(): AssociationCreateOrUpdateInfo( subject="json-cached-subject", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_JSON_SCHEMA, + ), ) ], ) @@ -1738,7 +1724,7 @@ def test_json_associated_name_strategy_caching(): assert obj1 == result1 # Delete associations (but serializer should still work due to caching) - client.delete_associations("json-resource-id-5") + client.delete_associations(resource_id="json-resource-id-5", cascade_lifecycle=True) obj2 = {"name": "Kafka", "id": 2} obj_bytes2 = ser(obj2, ser_ctx) diff --git a/tests/schema_registry/_sync/test_proto_serdes.py b/tests/schema_registry/_sync/test_proto_serdes.py index 5c49b9050..806899c5e 100644 --- a/tests/schema_registry/_sync/test_proto_serdes.py +++ b/tests/schema_registry/_sync/test_proto_serdes.py @@ -643,6 +643,10 @@ def test_associated_name_strategy_with_association(): AssociationCreateOrUpdateInfo( subject="my-custom-subject-value", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), ) ], ) @@ -664,6 +668,8 @@ def test_associated_name_strategy_with_association(): registered_schema = client.get_latest_version("my-custom-subject-value") assert registered_schema is not None + client.delete_associations(resource_id="proto-resource-id-1", cascade_lifecycle=True) + def test_associated_name_strategy_with_key_association(): """Test that AssociatedNameStrategy returns subject for key""" @@ -680,6 +686,10 @@ def test_associated_name_strategy_with_key_association(): AssociationCreateOrUpdateInfo( subject="my-key-subject", association_type="key", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), ) ], ) @@ -701,6 +711,8 @@ def test_associated_name_strategy_with_key_association(): registered_schema = client.get_latest_version("my-key-subject") assert registered_schema is not None + client.delete_associations(resource_id="proto-resource-id-2", cascade_lifecycle=True) + def test_associated_name_strategy_fallback_to_topic(): """Test fallback to topic_subject_name_strategy when no association""" @@ -800,44 +812,6 @@ def test_associated_name_strategy_fallback_none_raises(): assert "No associated subject found" in str(exc_info.value) -def test_associated_name_strategy_multiple_associations_raises(): - """Test that multiple associations raise an error""" - conf = {'url': _BASE_URL} - client = SchemaRegistryClient.new_client(conf) - obj = example_pb2.Author(name='Kafka', id=2) - - request = AssociationCreateOrUpdateRequest( - resource_name=_TOPIC, - resource_namespace="-", - resource_id="proto-resource-id-3", - resource_type="topic", - associations=[ - AssociationCreateOrUpdateInfo( - subject="proto-subject-1", - association_type="value", - ), - AssociationCreateOrUpdateInfo( - subject="proto-subject-2", - association_type="value", - ), - ], - ) - client.create_association(request) - - ser_conf = { - 'auto.register.schemas': True, - 'use.deprecated.format': False, - 'subject.name.strategy.type': SubjectNameStrategyType.ASSOCIATED, - } - ser = ProtobufSerializer(example_pb2.Author, client, conf=ser_conf) - ser_ctx = SerializationContext(_TOPIC, MessageField.VALUE) - - with pytest.raises(SerializationError) as exc_info: - ser(obj, ser_ctx) - - assert "Multiple associated subjects found" in str(exc_info.value) - - def test_associated_name_strategy_with_kafka_cluster_id(): """Test that subject.name.strategy.kafka.cluster.id config is used as resource namespace""" conf = {'url': _BASE_URL} @@ -855,6 +829,10 @@ def test_associated_name_strategy_with_kafka_cluster_id(): AssociationCreateOrUpdateInfo( subject="cluster-specific-proto-subject", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), ) ], ) @@ -877,6 +855,8 @@ def test_associated_name_strategy_with_kafka_cluster_id(): registered_schema = client.get_latest_version("cluster-specific-proto-subject") assert registered_schema is not None + client.delete_associations(resource_id="proto-resource-id-4", cascade_lifecycle=True) + def test_associated_name_strategy_caching(): """Test that results are cached within a strategy instance and serializer works with caching""" @@ -892,6 +872,10 @@ def test_associated_name_strategy_caching(): AssociationCreateOrUpdateInfo( subject="proto-cached-subject", association_type="value", + lifecycle="STRONG", + schema=Schema( + schema_str=_schema_to_str(example_pb2.Author.DESCRIPTOR.file), + ), ) ], ) @@ -916,7 +900,7 @@ def test_associated_name_strategy_caching(): assert obj1 == result1 # Delete associations (but serializer should still work due to caching) - client.delete_associations("proto-resource-id-5") + client.delete_associations(resource_id="proto-resource-id-5", cascade_lifecycle=True) obj2 = example_pb2.Author(name='Kafka', id=2) obj_bytes2 = ser(obj2, ser_ctx) From 63a493ff1bf7724078ee3269f5b131ff9f3e11cd Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Thu, 12 Mar 2026 11:07:44 -0700 Subject: [PATCH 18/20] Fix style --- .../schema_registry/_async/avro.py | 3 +- .../schema_registry/_async/json_schema.py | 3 +- .../_async/mock_schema_registry_client.py | 34 ++++++++---------- .../schema_registry/_async/protobuf.py | 6 ++-- .../_async/schema_registry_client.py | 20 +++-------- .../schema_registry/_async/serde.py | 30 ++++++++-------- .../common/schema_registry_client.py | 5 +-- tests/schema_registry/_async/test_avro.py | 3 +- .../_async/test_avro_serdes.py | 10 +++--- .../_async/test_json_serdes.py | 36 ++++++++++--------- .../_async/test_proto_serdes.py | 18 +++++----- 11 files changed, 82 insertions(+), 86 deletions(-) diff --git a/src/confluent_kafka/schema_registry/_async/avro.py b/src/confluent_kafka/schema_registry/_async/avro.py index 73b967dc9..e68533734 100644 --- a/src/confluent_kafka/schema_registry/_async/avro.py +++ b/src/confluent_kafka/schema_registry/_async/avro.py @@ -704,7 +704,8 @@ async def __deserialize( subject = ( ( await self._subject_name_func( - ctx, writer_schema.get("name"), self._registry, self._subject_name_conf) + ctx, writer_schema.get("name"), self._registry, self._subject_name_conf + ) if self._strategy_accepts_client else self._subject_name_func(ctx, writer_schema.get("name")) ) diff --git a/src/confluent_kafka/schema_registry/_async/json_schema.py b/src/confluent_kafka/schema_registry/_async/json_schema.py index 2a079e549..4bdc8c37c 100644 --- a/src/confluent_kafka/schema_registry/_async/json_schema.py +++ b/src/confluent_kafka/schema_registry/_async/json_schema.py @@ -699,7 +699,8 @@ async def __deserialize(self, data: Optional[bytes], ctx: Optional[Serialization if subject is None and isinstance(writer_schema, dict): subject = ( await self._subject_name_func( - ctx, writer_schema.get("title"), self._registry, self._subject_name_conf) + ctx, writer_schema.get("title"), self._registry, self._subject_name_conf + ) if self._strategy_accepts_client else self._subject_name_func(ctx, writer_schema.get("title")) ) diff --git a/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py b/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py index e1c1bdd09..614fbac05 100644 --- a/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py @@ -23,8 +23,8 @@ from ..common.schema_registry_client import ( Association, AssociationCreateOrUpdateRequest, - AssociationResponse, AssociationInfo, + AssociationResponse, RegisteredSchema, Schema, ServerConfig, @@ -159,10 +159,7 @@ def __init__(self): # Key: (resource_namespace, resource_name) -> resource_id self.resource_id_index: Dict[tuple, str] = {} - def create_association( - self, - request: AssociationCreateOrUpdateRequest - ) -> AssociationResponse: + def create_association(self, request: AssociationCreateOrUpdateRequest) -> AssociationResponse: with self.lock: resource_id = request.resource_id resource_name = request.resource_name @@ -187,13 +184,15 @@ def create_association( frozen=assoc_info.frozen if assoc_info.frozen is not None else False, ) self.associations_by_resource_id[resource_id].append(association) - created_associations.append(AssociationInfo( - subject=assoc_info.subject, - association_type=assoc_info.association_type, - lifecycle=assoc_info.lifecycle, - frozen=assoc_info.frozen if assoc_info.frozen is not None else False, - schema=assoc_info.schema, - )) + created_associations.append( + AssociationInfo( + subject=assoc_info.subject, + association_type=assoc_info.association_type, + lifecycle=assoc_info.lifecycle, + frozen=assoc_info.frozen if assoc_info.frozen is not None else False, + schema=assoc_info.schema, + ) + ) return AssociationResponse( resource_name=resource_name, @@ -234,7 +233,7 @@ def get_associations_by_resource_name( resource_name: str, resource_namespace: str, resource_type: Optional[str] = None, - association_types: Optional[List[str]] = None + association_types: Optional[List[str]] = None, ) -> List[Association]: with self.lock: result = [] @@ -425,16 +424,13 @@ async def get_associations_by_resource_name( resource_type: Optional[str] = None, association_types: Optional[List[str]] = None, offset: int = 0, - limit: int = -1 + limit: int = -1, ) -> List['Association']: return self._association_store.get_associations_by_resource_name( resource_name, resource_namespace, resource_type, association_types ) - async def create_association( - self, - request: 'AssociationCreateOrUpdateRequest' - ) -> 'AssociationResponse': + async def create_association(self, request: 'AssociationCreateOrUpdateRequest') -> 'AssociationResponse': """ Creates an association between a subject and a resource. @@ -451,7 +447,7 @@ async def delete_associations( resource_id: str, resource_type: Optional[str] = None, association_types: Optional[List[str]] = None, - cascade_lifecycle: bool = False + cascade_lifecycle: bool = False, ) -> None: """ Deletes associations for a resource. diff --git a/src/confluent_kafka/schema_registry/_async/protobuf.py b/src/confluent_kafka/schema_registry/_async/protobuf.py index 2125945bc..ecd1e1eb7 100644 --- a/src/confluent_kafka/schema_registry/_async/protobuf.py +++ b/src/confluent_kafka/schema_registry/_async/protobuf.py @@ -435,7 +435,8 @@ async def __serialize(self, message: Message, ctx: Optional[SerializationContext subject = ( ( await self._subject_name_func( - ctx, message.DESCRIPTOR.full_name, self._registry, self._subject_name_conf) + ctx, message.DESCRIPTOR.full_name, self._registry, self._subject_name_conf + ) if self._strategy_accepts_client else self._subject_name_func(ctx, message.DESCRIPTOR.full_name) ) @@ -690,7 +691,8 @@ async def __deserialize(self, data: Optional[bytes], ctx: Optional[Serialization subject = ( ( await self._subject_name_func( - ctx, writer_desc.full_name, self._registry, self._subject_name_conf) + ctx, writer_desc.full_name, self._registry, self._subject_name_conf + ) if self._strategy_accepts_client else self._subject_name_func(ctx, writer_desc.full_name) ) diff --git a/src/confluent_kafka/schema_registry/_async/schema_registry_client.py b/src/confluent_kafka/schema_registry/_async/schema_registry_client.py index 023efb36e..a5da213c2 100644 --- a/src/confluent_kafka/schema_registry/_async/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_async/schema_registry_client.py @@ -1591,7 +1591,7 @@ async def get_associations_by_resource_name( resource_type: Optional[str] = None, association_types: Optional[List[str]] = None, offset: int = 0, - limit: int = -1 + limit: int = -1, ) -> List['Association']: """ Retrieves associations for a given resource name and namespace. @@ -1623,19 +1623,12 @@ async def get_associations_by_resource_name( query['limit'] = limit response = await self._rest_client.get( - 'associations/resources/{}/{}'.format( - _urlencode(resource_namespace), - _urlencode(resource_name) - ), - query + 'associations/resources/{}/{}'.format(_urlencode(resource_namespace), _urlencode(resource_name)), query ) return [Association.from_dict(a) for a in response] - async def create_association( - self, - request: 'AssociationCreateOrUpdateRequest' - ) -> 'AssociationResponse': + async def create_association(self, request: 'AssociationCreateOrUpdateRequest') -> 'AssociationResponse': """ Creates an association between a subject and a resource. @@ -1656,7 +1649,7 @@ async def delete_associations( resource_id: str, resource_type: Optional[str] = None, association_types: Optional[List[str]] = None, - cascade_lifecycle: bool = False + cascade_lifecycle: bool = False, ) -> None: """ Deletes associations for a resource. @@ -1677,10 +1670,7 @@ async def delete_associations( if association_types is not None: query['associationType'] = association_types - await self._rest_client.delete( - 'associations/resources/{}'.format(_urlencode(resource_id)), - query=query - ) + await self._rest_client.delete('associations/resources/{}'.format(_urlencode(resource_id)), query=query) @staticmethod def new_client(conf: dict) -> 'AsyncSchemaRegistryClient': diff --git a/src/confluent_kafka/schema_registry/_async/serde.py b/src/confluent_kafka/schema_registry/_async/serde.py index 35ed68e23..976905f90 100644 --- a/src/confluent_kafka/schema_registry/_async/serde.py +++ b/src/confluent_kafka/schema_registry/_async/serde.py @@ -27,7 +27,6 @@ RegisteredSchema, topic_subject_name_strategy, ) -from confluent_kafka.schema_registry.error import SchemaRegistryError from confluent_kafka.schema_registry.common.schema_registry_client import RulePhase from confluent_kafka.schema_registry.common.serde import ( STRATEGY_TYPE_MAP, @@ -42,9 +41,15 @@ SchemaId, SubjectNameStrategyType, ) +from confluent_kafka.schema_registry.error import SchemaRegistryError from confluent_kafka.schema_registry.schema_registry_client import Rule, RuleKind, RuleMode, RuleSet, Schema -from confluent_kafka.serialization import (Deserializer, MessageField, - SerializationContext, SerializationError, Serializer) +from confluent_kafka.serialization import ( + Deserializer, + MessageField, + SerializationContext, + SerializationError, + Serializer, +) __all__ = [ 'AsyncAssociatedNameStrategy', @@ -79,9 +84,7 @@ def __init__(self, cache_capacity: int = DEFAULT_CACHE_CAPACITY): self._cache: LRUCache = LRUCache(maxsize=cache_capacity) self._lock: _locks.Lock = _locks.Lock() - def _get_cache_key( - self, topic: str, is_key: bool, record_name: Optional[str] - ) -> Tuple[str, bool, Optional[str]]: + def _get_cache_key(self, topic: str, is_key: bool, record_name: Optional[str]) -> Tuple[str, bool, Optional[str]]: """Create a cache key from topic, is_key, and record_name.""" return (topic, is_key, record_name) @@ -92,7 +95,7 @@ async def _load_subject_name( record_name: Optional[str], ctx: SerializationContext, schema_registry_client: AsyncSchemaRegistryClient, - conf: Optional[dict] + conf: Optional[dict], ) -> Optional[str]: """Load the subject name from schema registry (not cached).""" # Determine resource namespace from config @@ -113,8 +116,9 @@ async def _load_subject_name( try: fallback_strategy = SubjectNameStrategyType(str(fallback_config).upper()) except ValueError: - valid_fallbacks = [e.value for e in SubjectNameStrategyType - if e != SubjectNameStrategyType.ASSOCIATED] + valid_fallbacks = [ + e.value for e in SubjectNameStrategyType if e != SubjectNameStrategyType.ASSOCIATED + ] raise ValueError( f"Invalid value for {FALLBACK_TYPE}: {fallback_config}. " f"Valid values are: {', '.join(valid_fallbacks)}" @@ -133,7 +137,7 @@ async def _load_subject_name( resource_type="topic", association_types=[association_type], offset=0, - limit=-1 + limit=-1, ) except SchemaRegistryError as e: if e.http_status_code == 404: @@ -163,7 +167,7 @@ async def __call__( ctx: Optional[SerializationContext], record_name: Optional[str], schema_registry_client: AsyncSchemaRegistryClient, - conf: Optional[dict] = None + conf: Optional[dict] = None, ) -> Optional[str]: """ Retrieves the associated subject name from schema registry by querying @@ -324,9 +328,7 @@ def configure_subject_name_strategy( self._subject_name_func = STRATEGY_TYPE_MAP[subject_name_strategy_type] self._strategy_accepts_client = False else: - raise ValueError( - f"Unknown subject.name.strategy.type: {subject_name_strategy_type}" - ) + raise ValueError(f"Unknown subject.name.strategy.type: {subject_name_strategy_type}") return # Default to AsyncAssociatedNameStrategy (falls back to TOPIC when no associations found) diff --git a/src/confluent_kafka/schema_registry/common/schema_registry_client.py b/src/confluent_kafka/schema_registry/common/schema_registry_client.py index fa9f8c482..88819d085 100644 --- a/src/confluent_kafka/schema_registry/common/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/common/schema_registry_client.py @@ -1169,8 +1169,9 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: resource_id = d.pop("resourceId", None) resource_type = d.pop("resourceType", None) associations_list = d.pop("associations", None) - associations = [AssociationCreateOrUpdateInfo.from_dict(a) - for a in associations_list] if associations_list else None + associations = ( + [AssociationCreateOrUpdateInfo.from_dict(a) for a in associations_list] if associations_list else None + ) return cls( # type: ignore[call-arg] resource_name=resource_name, diff --git a/tests/schema_registry/_async/test_avro.py b/tests/schema_registry/_async/test_avro.py index 8e92bc2a1..06b346321 100644 --- a/tests/schema_registry/_async/test_avro.py +++ b/tests/schema_registry/_async/test_avro.py @@ -204,7 +204,8 @@ async def test_avro_serializer_subject_name_strategy_default(mock_schema_registr ctx = SerializationContext('test_subj', MessageField.VALUE) if test_serializer._strategy_accepts_client: result = await test_serializer._subject_name_func( - ctx, test_serializer._schema_name, test_client, test_serializer._subject_name_conf) + ctx, test_serializer._schema_name, test_client, test_serializer._subject_name_conf + ) else: result = test_serializer._subject_name_func(ctx, test_serializer._schema_name) assert result == 'test_subj-value' diff --git a/tests/schema_registry/_async/test_avro_serdes.py b/tests/schema_registry/_async/test_avro_serdes.py index 7464b3d2c..cd6f5e54b 100644 --- a/tests/schema_registry/_async/test_avro_serdes.py +++ b/tests/schema_registry/_async/test_avro_serdes.py @@ -29,16 +29,16 @@ Schema, header_schema_id_serializer, ) +from confluent_kafka.schema_registry._async.serde import ( + FALLBACK_TYPE, + KAFKA_CLUSTER_ID, +) +from confluent_kafka.schema_registry.avro import AsyncAvroDeserializer, AsyncAvroSerializer from confluent_kafka.schema_registry.common.schema_registry_client import ( AssociationCreateOrUpdateInfo, AssociationCreateOrUpdateRequest, ) from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType -from confluent_kafka.schema_registry._async.serde import ( - KAFKA_CLUSTER_ID, - FALLBACK_TYPE, -) -from confluent_kafka.schema_registry.avro import AsyncAvroDeserializer, AsyncAvroSerializer from confluent_kafka.schema_registry.rule_registry import RuleOverride, RuleRegistry from confluent_kafka.schema_registry.rules.cel.cel_executor import CelExecutor from confluent_kafka.schema_registry.rules.cel.cel_field_executor import CelFieldExecutor diff --git a/tests/schema_registry/_async/test_json_serdes.py b/tests/schema_registry/_async/test_json_serdes.py index bf8e81838..abde55f18 100644 --- a/tests/schema_registry/_async/test_json_serdes.py +++ b/tests/schema_registry/_async/test_json_serdes.py @@ -27,6 +27,15 @@ Schema, header_schema_id_serializer, ) +from confluent_kafka.schema_registry._async.serde import ( + FALLBACK_TYPE, + KAFKA_CLUSTER_ID, +) +from confluent_kafka.schema_registry.common.schema_registry_client import ( + AssociationCreateOrUpdateInfo, + AssociationCreateOrUpdateRequest, +) +from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.json_schema import AsyncJSONDeserializer, AsyncJSONSerializer from confluent_kafka.schema_registry.rules.cel.cel_executor import CelExecutor from confluent_kafka.schema_registry.rules.cel.cel_field_executor import CelFieldExecutor @@ -49,15 +58,6 @@ SchemaReference, ServerConfig, ) -from confluent_kafka.schema_registry._async.serde import ( - FALLBACK_TYPE, - KAFKA_CLUSTER_ID, -) -from confluent_kafka.schema_registry.common.schema_registry_client import ( - AssociationCreateOrUpdateInfo, - AssociationCreateOrUpdateRequest, -) -from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.serde import RuleConditionError from confluent_kafka.serialization import MessageField, SerializationContext, SerializationError from tests.schema_registry._async.test_avro_serdes import FakeClock @@ -1460,14 +1460,16 @@ async def test_json_deeply_nested_refs(): assert obj == obj2 -_JSON_SCHEMA = json.dumps({ - "type": "object", - "title": "MyRecord", - "properties": { - "name": {"type": "string"}, - "id": {"type": "integer"}, - }, -}) +_JSON_SCHEMA = json.dumps( + { + "type": "object", + "title": "MyRecord", + "properties": { + "name": {"type": "string"}, + "id": {"type": "integer"}, + }, + } +) _JSON_OBJ = {"name": "Kafka", "id": 123} diff --git a/tests/schema_registry/_async/test_proto_serdes.py b/tests/schema_registry/_async/test_proto_serdes.py index f2ca0dd53..f61118e96 100644 --- a/tests/schema_registry/_async/test_proto_serdes.py +++ b/tests/schema_registry/_async/test_proto_serdes.py @@ -24,6 +24,15 @@ from confluent_kafka.schema_registry import Metadata, MetadataProperties, Schema, header_schema_id_serializer from confluent_kafka.schema_registry._async.protobuf import AsyncProtobufDeserializer, AsyncProtobufSerializer from confluent_kafka.schema_registry._async.schema_registry_client import AsyncSchemaRegistryClient +from confluent_kafka.schema_registry._async.serde import ( + FALLBACK_TYPE, + KAFKA_CLUSTER_ID, +) +from confluent_kafka.schema_registry.common.schema_registry_client import ( + AssociationCreateOrUpdateInfo, + AssociationCreateOrUpdateRequest, +) +from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.protobuf import _schema_to_str from confluent_kafka.schema_registry.rules.cel.cel_executor import CelExecutor from confluent_kafka.schema_registry.rules.cel.cel_field_executor import CelFieldExecutor @@ -46,15 +55,6 @@ RuleSet, ServerConfig, ) -from confluent_kafka.schema_registry._async.serde import ( - FALLBACK_TYPE, - KAFKA_CLUSTER_ID, -) -from confluent_kafka.schema_registry.common.schema_registry_client import ( - AssociationCreateOrUpdateInfo, - AssociationCreateOrUpdateRequest, -) -from confluent_kafka.schema_registry.common.serde import SubjectNameStrategyType from confluent_kafka.schema_registry.serde import RuleConditionError from confluent_kafka.serialization import MessageField, SerializationContext, SerializationError From caad794b497f741e711b3c1e8cb9e1d286947f20 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Thu, 12 Mar 2026 12:32:24 -0700 Subject: [PATCH 19/20] Fix mypy --- src/confluent_kafka/schema_registry/_async/avro.py | 14 +++++++------- .../schema_registry/_async/json_schema.py | 12 ++++++------ .../_async/mock_schema_registry_client.py | 4 ++-- .../schema_registry/_async/protobuf.py | 12 ++++++------ .../schema_registry/_async/serde.py | 2 +- src/confluent_kafka/schema_registry/_sync/avro.py | 14 +++++++------- .../schema_registry/_sync/json_schema.py | 12 ++++++------ .../_sync/mock_schema_registry_client.py | 4 ++-- .../schema_registry/_sync/protobuf.py | 12 ++++++------ src/confluent_kafka/schema_registry/_sync/serde.py | 2 +- 10 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/confluent_kafka/schema_registry/_async/avro.py b/src/confluent_kafka/schema_registry/_async/avro.py index e68533734..9f60b08b8 100644 --- a/src/confluent_kafka/schema_registry/_async/avro.py +++ b/src/confluent_kafka/schema_registry/_async/avro.py @@ -302,9 +302,9 @@ async def __init_impl( raise ValueError("use.latest.with.metadata must be a dict value") self.configure_subject_name_strategy( - subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), - subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), - subject_name_strategy=conf_copy.pop('subject.name.strategy'), + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) self._schema_id_serializer = cast( @@ -610,9 +610,9 @@ async def __init_impl( raise ValueError("use.latest.with.metadata must be a dict value") self.configure_subject_name_strategy( - subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), - subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), - subject_name_strategy=conf_copy.pop('subject.name.strategy'), + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) self._schema_id_deserializer = cast( @@ -700,7 +700,7 @@ async def __deserialize( writer_schema_raw = await self._get_writer_schema(schema_id, subject) writer_schema = await self._get_parsed_schema(writer_schema_raw) - if subject is None: + if subject is None and isinstance(writer_schema, dict): subject = ( ( await self._subject_name_func( diff --git a/src/confluent_kafka/schema_registry/_async/json_schema.py b/src/confluent_kafka/schema_registry/_async/json_schema.py index 4bdc8c37c..25f71be0a 100644 --- a/src/confluent_kafka/schema_registry/_async/json_schema.py +++ b/src/confluent_kafka/schema_registry/_async/json_schema.py @@ -308,9 +308,9 @@ async def __init_impl( raise ValueError("use.latest.with.metadata must be a dict value") self.configure_subject_name_strategy( - subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), - subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), - subject_name_strategy=conf_copy.pop('subject.name.strategy'), + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) self._schema_id_serializer = cast( @@ -619,9 +619,9 @@ async def __init_impl( raise ValueError("use.latest.with.metadata must be a dict value") self.configure_subject_name_strategy( - subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), - subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), - subject_name_strategy=conf_copy.pop('subject.name.strategy'), + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) self._schema_id_deserializer = cast( diff --git a/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py b/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py index 614fbac05..b12a1fd01 100644 --- a/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_async/mock_schema_registry_client.py @@ -167,11 +167,11 @@ def create_association(self, request: AssociationCreateOrUpdateRequest) -> Assoc resource_type = request.resource_type # Index resource_id by (namespace, name) - if resource_name and resource_namespace: + if resource_name and resource_namespace and resource_id: self.resource_id_index[(resource_namespace, resource_name)] = resource_id created_associations = [] - if request.associations: + if request.associations and resource_id is not None: for assoc_info in request.associations: association = Association( subject=assoc_info.subject, diff --git a/src/confluent_kafka/schema_registry/_async/protobuf.py b/src/confluent_kafka/schema_registry/_async/protobuf.py index ecd1e1eb7..ba953d383 100644 --- a/src/confluent_kafka/schema_registry/_async/protobuf.py +++ b/src/confluent_kafka/schema_registry/_async/protobuf.py @@ -297,9 +297,9 @@ async def __init_impl( raise ValueError("use.deprecated.format is no longer supported") self.configure_subject_name_strategy( - subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), - subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), - subject_name_strategy=conf_copy.pop('subject.name.strategy'), + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) self._ref_reference_subject_func = cast( @@ -612,9 +612,9 @@ async def __init_impl( raise ValueError("use.latest.with.metadata must be a dict value") self.configure_subject_name_strategy( - subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), - subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), - subject_name_strategy=conf_copy.pop('subject.name.strategy'), + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) self._schema_id_deserializer = cast( diff --git a/src/confluent_kafka/schema_registry/_async/serde.py b/src/confluent_kafka/schema_registry/_async/serde.py index 976905f90..8e1b58983 100644 --- a/src/confluent_kafka/schema_registry/_async/serde.py +++ b/src/confluent_kafka/schema_registry/_async/serde.py @@ -262,7 +262,7 @@ class AsyncBaseSerde(object): _rule_registry: Any # RuleRegistry _strategy_accepts_client: bool _subject_name_conf: Optional[dict] - _subject_name_func: Callable[[Optional['SerializationContext'], Optional[str]], Optional[str]] + _subject_name_func: Callable[..., Any] _field_transformer: Optional[FieldTransformer] def configure_subject_name_strategy( diff --git a/src/confluent_kafka/schema_registry/_sync/avro.py b/src/confluent_kafka/schema_registry/_sync/avro.py index 4525b13e6..6c00e7b15 100644 --- a/src/confluent_kafka/schema_registry/_sync/avro.py +++ b/src/confluent_kafka/schema_registry/_sync/avro.py @@ -298,9 +298,9 @@ def __init_impl( raise ValueError("use.latest.with.metadata must be a dict value") self.configure_subject_name_strategy( - subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), - subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), - subject_name_strategy=conf_copy.pop('subject.name.strategy'), + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) self._schema_id_serializer = cast( @@ -605,9 +605,9 @@ def __init_impl( raise ValueError("use.latest.with.metadata must be a dict value") self.configure_subject_name_strategy( - subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), - subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), - subject_name_strategy=conf_copy.pop('subject.name.strategy'), + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) self._schema_id_deserializer = cast( @@ -693,7 +693,7 @@ def __deserialize( writer_schema_raw = self._get_writer_schema(schema_id, subject) writer_schema = self._get_parsed_schema(writer_schema_raw) - if subject is None: + if subject is None and isinstance(writer_schema, dict): subject = ( ( self._subject_name_func(ctx, writer_schema.get("name"), self._registry, self._subject_name_conf) diff --git a/src/confluent_kafka/schema_registry/_sync/json_schema.py b/src/confluent_kafka/schema_registry/_sync/json_schema.py index 93da54919..f1aa491d4 100644 --- a/src/confluent_kafka/schema_registry/_sync/json_schema.py +++ b/src/confluent_kafka/schema_registry/_sync/json_schema.py @@ -306,9 +306,9 @@ def __init_impl( raise ValueError("use.latest.with.metadata must be a dict value") self.configure_subject_name_strategy( - subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), - subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), - subject_name_strategy=conf_copy.pop('subject.name.strategy'), + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) self._schema_id_serializer = cast( @@ -616,9 +616,9 @@ def __init_impl( raise ValueError("use.latest.with.metadata must be a dict value") self.configure_subject_name_strategy( - subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), - subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), - subject_name_strategy=conf_copy.pop('subject.name.strategy'), + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) self._schema_id_deserializer = cast( diff --git a/src/confluent_kafka/schema_registry/_sync/mock_schema_registry_client.py b/src/confluent_kafka/schema_registry/_sync/mock_schema_registry_client.py index 352579afe..ae44934b9 100644 --- a/src/confluent_kafka/schema_registry/_sync/mock_schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/_sync/mock_schema_registry_client.py @@ -167,11 +167,11 @@ def create_association(self, request: AssociationCreateOrUpdateRequest) -> Assoc resource_type = request.resource_type # Index resource_id by (namespace, name) - if resource_name and resource_namespace: + if resource_name and resource_namespace and resource_id: self.resource_id_index[(resource_namespace, resource_name)] = resource_id created_associations = [] - if request.associations: + if request.associations and resource_id is not None: for assoc_info in request.associations: association = Association( subject=assoc_info.subject, diff --git a/src/confluent_kafka/schema_registry/_sync/protobuf.py b/src/confluent_kafka/schema_registry/_sync/protobuf.py index 990e3e98c..e57e960e8 100644 --- a/src/confluent_kafka/schema_registry/_sync/protobuf.py +++ b/src/confluent_kafka/schema_registry/_sync/protobuf.py @@ -295,9 +295,9 @@ def __init_impl( raise ValueError("use.deprecated.format is no longer supported") self.configure_subject_name_strategy( - subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), - subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), - subject_name_strategy=conf_copy.pop('subject.name.strategy'), + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) self._ref_reference_subject_func = cast( @@ -605,9 +605,9 @@ def __init_impl( raise ValueError("use.latest.with.metadata must be a dict value") self.configure_subject_name_strategy( - subject_name_strategy_type=conf_copy.pop('subject.name.strategy.type'), - subject_name_strategy_conf=conf_copy.pop('subject.name.strategy.conf'), - subject_name_strategy=conf_copy.pop('subject.name.strategy'), + subject_name_strategy_type=cast(Any, conf_copy.pop('subject.name.strategy.type')), + subject_name_strategy_conf=cast(Any, conf_copy.pop('subject.name.strategy.conf')), + subject_name_strategy=cast(Any, conf_copy.pop('subject.name.strategy')), ) self._schema_id_deserializer = cast( diff --git a/src/confluent_kafka/schema_registry/_sync/serde.py b/src/confluent_kafka/schema_registry/_sync/serde.py index b511cc0e3..58b835006 100644 --- a/src/confluent_kafka/schema_registry/_sync/serde.py +++ b/src/confluent_kafka/schema_registry/_sync/serde.py @@ -262,7 +262,7 @@ class BaseSerde(object): _rule_registry: Any # RuleRegistry _strategy_accepts_client: bool _subject_name_conf: Optional[dict] - _subject_name_func: Callable[[Optional['SerializationContext'], Optional[str]], Optional[str]] + _subject_name_func: Callable[..., Any] _field_transformer: Optional[FieldTransformer] def configure_subject_name_strategy( From 761cdf8f35a8416546c0e707e00b2d13009ea515 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Thu, 12 Mar 2026 13:08:58 -0700 Subject: [PATCH 20/20] Fix docs --- .../schema_registry/_async/avro.py | 86 +++---- .../schema_registry/_async/json_schema.py | 238 +++++++++--------- .../schema_registry/_async/protobuf.py | 4 +- .../schema_registry/_sync/avro.py | 86 +++---- .../schema_registry/_sync/json_schema.py | 238 +++++++++--------- .../schema_registry/_sync/protobuf.py | 4 +- 6 files changed, 328 insertions(+), 328 deletions(-) diff --git a/src/confluent_kafka/schema_registry/_async/avro.py b/src/confluent_kafka/schema_registry/_async/avro.py index 9f60b08b8..65ec01a1c 100644 --- a/src/confluent_kafka/schema_registry/_async/avro.py +++ b/src/confluent_kafka/schema_registry/_async/avro.py @@ -128,7 +128,7 @@ class AsyncAvroSerializer(AsyncBaseSerializer): | | | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to ASSOCIATED if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-----------------------------------+----------+--------------------------------------------------+ | ``subject.name.strategy.conf`` | dict | Configuration dictionary passed to strategies | @@ -486,48 +486,48 @@ class AsyncAvroDeserializer(AsyncBaseDeserializer): Deserializer for Avro binary encoded data with Confluent Schema Registry framing. - +-----------------------------+----------+--------------------------------------------------+ - | Property Name | Type | Description | - +-----------------------------+----------+--------------------------------------------------+ - | | | Whether to use the latest subject version for | - | ``use.latest.version`` | bool | deserialization. | - | | | | - | | | Defaults to False. | - +-----------------------------+----------+--------------------------------------------------+ - | | | Whether to use the latest subject version with | - | ``use.latest.with.metadata``| dict | the given metadata. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+--------------------------------------------------+ - | | | The type of subject name strategy to use. | - |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | - | | | ASSOCIATED. | - | | | | - | | | Defaults to ASSOCIATED if neither this nor | - | | | subject.name.strategy is specified. | - +-----------------------------+----------+--------------------------------------------------+ - | | | Configuration dictionary passed to strategies | - |``subject.name.strategy.conf``| dict | that require additional configuration, such as | - | | | ASSOCIATED. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+--------------------------------------------------+ - | | | Callable(SerializationContext, str) -> str | - | | | | - | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka.schema_registry | - | | | namespace. Takes precedence over | - | | | subject.name.strategy.type if both are set. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+--------------------------------------------------+ - | | | Callable(bytes, SerializationContext, schema_id) | - | | | -> io.BytesIO | - | | | | - | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | - | | | Defaults to dual_schema_id_deserializer. | - +-----------------------------+----------+--------------------------------------------------+ + +----------------------------------+----------+--------------------------------------------------+ + | Property Name | Type | Description | + +----------------------------------+----------+--------------------------------------------------+ + | | | Whether to use the latest subject version for | + | ``use.latest.version`` | bool | deserialization. | + | | | | + | | | Defaults to False. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Whether to use the latest subject version with | + | ``use.latest.with.metadata`` | dict | the given metadata. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+--------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Callable(SerializationContext, str) -> str | + | | | | + | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Callable(bytes, SerializationContext, schema_id) | + | | | -> io.BytesIO | + | | | | + | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | + | | | Defaults to dual_schema_id_deserializer. | + +----------------------------------+----------+--------------------------------------------------+ Note: By default, Avro complex types are returned as dicts. This behavior can diff --git a/src/confluent_kafka/schema_registry/_async/json_schema.py b/src/confluent_kafka/schema_registry/_async/json_schema.py index 25f71be0a..048b8100b 100644 --- a/src/confluent_kafka/schema_registry/_async/json_schema.py +++ b/src/confluent_kafka/schema_registry/_async/json_schema.py @@ -94,78 +94,78 @@ class AsyncJSONSerializer(AsyncBaseSerializer): Configuration properties: - +-----------------------------+----------+----------------------------------------------------+ - | Property Name | Type | Description | - +=============================+==========+====================================================+ - | | | If True, automatically register the configured | - | ``auto.register.schemas`` | bool | schema with Confluent Schema Registry if it has | - | | | not previously been associated with the relevant | - | | | subject (determined via subject.name.strategy). | - | | | | - | | | Defaults to True. | - | | | | - | | | Raises SchemaRegistryError if the schema was not | - | | | registered against the subject, or could not be | - | | | successfully registered. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to normalize schemas, which will | - | ``normalize.schemas`` | bool | transform schemas to have a consistent format, | - | | | including ordering properties and references. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the given schema ID for | - | ``use.schema.id`` | int | serialization. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version for | - | ``use.latest.version`` | bool | serialization. | - | | | | - | | | WARNING: There is no check that the latest | - | | | schema is backwards compatible with the object | - | | | being serialized. | - | | | | - | | | Defaults to False. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version with | - | ``use.latest.with.metadata``| dict | the given metadata. | - | | | | - | | | WARNING: There is no check that the latest | - | | | schema is backwards compatible with the object | - | | | being serialized. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | The type of subject name strategy to use. | - |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | - | | | ASSOCIATED. | - | | | | - | | | Defaults to ASSOCIATED if neither this nor | - | | | subject.name.strategy is specified. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Configuration dictionary passed to strategies | - |``subject.name.strategy.conf``| dict | that require additional configuration, such as | - | | | ASSOCIATED. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(SerializationContext, str) -> str | - | | | | - | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka.schema_registry | - | | | namespace. Takes precedence over | - | | | subject.name.strategy.type if both are set. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to validate the payload against the | - | ``validate`` | bool | the given schema. | - | | | | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(bytes, SerializationContext, schema_id) | - | | | -> bytes | - | | | | - | ``schema.id.serializer`` | callable | Defines how the schema id/guid is serialized. | - | | | Defaults to prefix_schema_id_serializer. | - +-----------------------------+----------+----------------------------------------------------+ + +----------------------------------+----------+----------------------------------------------------+ + | Property Name | Type | Description | + +==================================+==========+====================================================+ + | | | If True, automatically register the configured | + | ``auto.register.schemas`` | bool | schema with Confluent Schema Registry if it has | + | | | not previously been associated with the relevant | + | | | subject (determined via subject.name.strategy). | + | | | | + | | | Defaults to True. | + | | | | + | | | Raises SchemaRegistryError if the schema was not | + | | | registered against the subject, or could not be | + | | | successfully registered. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to normalize schemas, which will | + | ``normalize.schemas`` | bool | transform schemas to have a consistent format, | + | | | including ordering properties and references. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the given schema ID for | + | ``use.schema.id`` | int | serialization. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version for | + | ``use.latest.version`` | bool | serialization. | + | | | | + | | | WARNING: There is no check that the latest | + | | | schema is backwards compatible with the object | + | | | being serialized. | + | | | | + | | | Defaults to False. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version with | + | ``use.latest.with.metadata`` | dict | the given metadata. | + | | | | + | | | WARNING: There is no check that the latest | + | | | schema is backwards compatible with the object | + | | | being serialized. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(SerializationContext, str) -> str | + | | | | + | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to validate the payload against the | + | ``validate`` | bool | the given schema. | + | | | | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(bytes, SerializationContext, schema_id) | + | | | -> bytes | + | | | | + | ``schema.id.serializer`` | callable | Defines how the schema id/guid is serialized. | + | | | Defaults to prefix_schema_id_serializer. | + +----------------------------------+----------+----------------------------------------------------+ Schemas are registered against subject names in Confluent Schema Registry that define a scope in which the schemas can be evolved. By default, the subject name @@ -485,53 +485,53 @@ class AsyncJSONDeserializer(AsyncBaseDeserializer): Configuration properties: - +-----------------------------+----------+----------------------------------------------------+ - | Property Name | Type | Description | - +=============================+==========+====================================================+ - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version for | - | ``use.latest.version`` | bool | deserialization. | - | | | | - | | | Defaults to False. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version with | - | ``use.latest.with.metadata``| dict | the given metadata. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | The type of subject name strategy to use. | - |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | - | | | ASSOCIATED. | - | | | | - | | | Defaults to ASSOCIATED if neither this nor | - | | | subject.name.strategy is specified. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Configuration dictionary passed to strategies | - |``subject.name.strategy.conf``| dict | that require additional configuration, such as | - | | | ASSOCIATED. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(SerializationContext, str) -> str | - | | | | - | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka.schema_registry | - | | | namespace. Takes precedence over | - | | | subject.name.strategy.type if both are set. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to validate the payload against the | - | ``validate`` | bool | the given schema. | - | | | | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(bytes, SerializationContext, schema_id) | - | | | -> io.BytesIO | - | | | | - | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | - | | | Defaults to dual_schema_id_deserializer. | - +-----------------------------+----------+----------------------------------------------------+ + +----------------------------------+----------+----------------------------------------------------+ + | Property Name | Type | Description | + +==================================+==========+====================================================+ + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version for | + | ``use.latest.version`` | bool | deserialization. | + | | | | + | | | Defaults to False. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version with | + | ``use.latest.with.metadata`` | dict | the given metadata. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(SerializationContext, str) -> str | + | | | | + | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to validate the payload against the | + | ``validate`` | bool | the given schema. | + | | | | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(bytes, SerializationContext, schema_id) | + | | | -> io.BytesIO | + | | | | + | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | + | | | Defaults to dual_schema_id_deserializer. | + +----------------------------------+----------+----------------------------------------------------+ Args: schema_str (str, Schema, optional): diff --git a/src/confluent_kafka/schema_registry/_async/protobuf.py b/src/confluent_kafka/schema_registry/_async/protobuf.py index ba953d383..cca5e1ed2 100644 --- a/src/confluent_kafka/schema_registry/_async/protobuf.py +++ b/src/confluent_kafka/schema_registry/_async/protobuf.py @@ -154,7 +154,7 @@ class AsyncProtobufSerializer(AsyncBaseSerializer): | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to ASSOCIATED if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-------------------------------------+----------+------------------------------------------------------+ | | | Configuration dictionary passed to strategies | @@ -541,7 +541,7 @@ class AsyncProtobufDeserializer(AsyncBaseDeserializer): | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to ASSOCIATED if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-------------------------------------+----------+------------------------------------------------------+ | | | Configuration dictionary passed to strategies | diff --git a/src/confluent_kafka/schema_registry/_sync/avro.py b/src/confluent_kafka/schema_registry/_sync/avro.py index 6c00e7b15..d17aac5a5 100644 --- a/src/confluent_kafka/schema_registry/_sync/avro.py +++ b/src/confluent_kafka/schema_registry/_sync/avro.py @@ -124,7 +124,7 @@ class AvroSerializer(BaseSerializer): | | | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to ASSOCIATED if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-----------------------------------+----------+--------------------------------------------------+ | ``subject.name.strategy.conf`` | dict | Configuration dictionary passed to strategies | @@ -481,48 +481,48 @@ class AvroDeserializer(BaseDeserializer): Deserializer for Avro binary encoded data with Confluent Schema Registry framing. - +-----------------------------+----------+--------------------------------------------------+ - | Property Name | Type | Description | - +-----------------------------+----------+--------------------------------------------------+ - | | | Whether to use the latest subject version for | - | ``use.latest.version`` | bool | deserialization. | - | | | | - | | | Defaults to False. | - +-----------------------------+----------+--------------------------------------------------+ - | | | Whether to use the latest subject version with | - | ``use.latest.with.metadata``| dict | the given metadata. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+--------------------------------------------------+ - | | | The type of subject name strategy to use. | - |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | - | | | ASSOCIATED. | - | | | | - | | | Defaults to ASSOCIATED if neither this nor | - | | | subject.name.strategy is specified. | - +-----------------------------+----------+--------------------------------------------------+ - | | | Configuration dictionary passed to strategies | - |``subject.name.strategy.conf``| dict | that require additional configuration, such as | - | | | ASSOCIATED. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+--------------------------------------------------+ - | | | Callable(SerializationContext, str) -> str | - | | | | - | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka.schema_registry | - | | | namespace. Takes precedence over | - | | | subject.name.strategy.type if both are set. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+--------------------------------------------------+ - | | | Callable(bytes, SerializationContext, schema_id) | - | | | -> io.BytesIO | - | | | | - | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | - | | | Defaults to dual_schema_id_deserializer. | - +-----------------------------+----------+--------------------------------------------------+ + +----------------------------------+----------+--------------------------------------------------+ + | Property Name | Type | Description | + +----------------------------------+----------+--------------------------------------------------+ + | | | Whether to use the latest subject version for | + | ``use.latest.version`` | bool | deserialization. | + | | | | + | | | Defaults to False. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Whether to use the latest subject version with | + | ``use.latest.with.metadata`` | dict | the given metadata. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+--------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Callable(SerializationContext, str) -> str | + | | | | + | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+--------------------------------------------------+ + | | | Callable(bytes, SerializationContext, schema_id) | + | | | -> io.BytesIO | + | | | | + | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | + | | | Defaults to dual_schema_id_deserializer. | + +----------------------------------+----------+--------------------------------------------------+ Note: By default, Avro complex types are returned as dicts. This behavior can diff --git a/src/confluent_kafka/schema_registry/_sync/json_schema.py b/src/confluent_kafka/schema_registry/_sync/json_schema.py index f1aa491d4..ff33aa4df 100644 --- a/src/confluent_kafka/schema_registry/_sync/json_schema.py +++ b/src/confluent_kafka/schema_registry/_sync/json_schema.py @@ -92,78 +92,78 @@ class JSONSerializer(BaseSerializer): Configuration properties: - +-----------------------------+----------+----------------------------------------------------+ - | Property Name | Type | Description | - +=============================+==========+====================================================+ - | | | If True, automatically register the configured | - | ``auto.register.schemas`` | bool | schema with Confluent Schema Registry if it has | - | | | not previously been associated with the relevant | - | | | subject (determined via subject.name.strategy). | - | | | | - | | | Defaults to True. | - | | | | - | | | Raises SchemaRegistryError if the schema was not | - | | | registered against the subject, or could not be | - | | | successfully registered. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to normalize schemas, which will | - | ``normalize.schemas`` | bool | transform schemas to have a consistent format, | - | | | including ordering properties and references. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the given schema ID for | - | ``use.schema.id`` | int | serialization. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version for | - | ``use.latest.version`` | bool | serialization. | - | | | | - | | | WARNING: There is no check that the latest | - | | | schema is backwards compatible with the object | - | | | being serialized. | - | | | | - | | | Defaults to False. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version with | - | ``use.latest.with.metadata``| dict | the given metadata. | - | | | | - | | | WARNING: There is no check that the latest | - | | | schema is backwards compatible with the object | - | | | being serialized. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | The type of subject name strategy to use. | - |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | - | | | ASSOCIATED. | - | | | | - | | | Defaults to ASSOCIATED if neither this nor | - | | | subject.name.strategy is specified. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Configuration dictionary passed to strategies | - |``subject.name.strategy.conf``| dict | that require additional configuration, such as | - | | | ASSOCIATED. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(SerializationContext, str) -> str | - | | | | - | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka.schema_registry | - | | | namespace. Takes precedence over | - | | | subject.name.strategy.type if both are set. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to validate the payload against the | - | ``validate`` | bool | the given schema. | - | | | | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(bytes, SerializationContext, schema_id) | - | | | -> bytes | - | | | | - | ``schema.id.serializer`` | callable | Defines how the schema id/guid is serialized. | - | | | Defaults to prefix_schema_id_serializer. | - +-----------------------------+----------+----------------------------------------------------+ + +----------------------------------+----------+----------------------------------------------------+ + | Property Name | Type | Description | + +==================================+==========+====================================================+ + | | | If True, automatically register the configured | + | ``auto.register.schemas`` | bool | schema with Confluent Schema Registry if it has | + | | | not previously been associated with the relevant | + | | | subject (determined via subject.name.strategy). | + | | | | + | | | Defaults to True. | + | | | | + | | | Raises SchemaRegistryError if the schema was not | + | | | registered against the subject, or could not be | + | | | successfully registered. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to normalize schemas, which will | + | ``normalize.schemas`` | bool | transform schemas to have a consistent format, | + | | | including ordering properties and references. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the given schema ID for | + | ``use.schema.id`` | int | serialization. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version for | + | ``use.latest.version`` | bool | serialization. | + | | | | + | | | WARNING: There is no check that the latest | + | | | schema is backwards compatible with the object | + | | | being serialized. | + | | | | + | | | Defaults to False. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version with | + | ``use.latest.with.metadata`` | dict | the given metadata. | + | | | | + | | | WARNING: There is no check that the latest | + | | | schema is backwards compatible with the object | + | | | being serialized. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(SerializationContext, str) -> str | + | | | | + | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to validate the payload against the | + | ``validate`` | bool | the given schema. | + | | | | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(bytes, SerializationContext, schema_id) | + | | | -> bytes | + | | | | + | ``schema.id.serializer`` | callable | Defines how the schema id/guid is serialized. | + | | | Defaults to prefix_schema_id_serializer. | + +----------------------------------+----------+----------------------------------------------------+ Schemas are registered against subject names in Confluent Schema Registry that define a scope in which the schemas can be evolved. By default, the subject name @@ -482,53 +482,53 @@ class JSONDeserializer(BaseDeserializer): Configuration properties: - +-----------------------------+----------+----------------------------------------------------+ - | Property Name | Type | Description | - +=============================+==========+====================================================+ - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version for | - | ``use.latest.version`` | bool | deserialization. | - | | | | - | | | Defaults to False. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to use the latest subject version with | - | ``use.latest.with.metadata``| dict | the given metadata. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | The type of subject name strategy to use. | - |``subject.name.strategy.type``| str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | - | | | ASSOCIATED. | - | | | | - | | | Defaults to ASSOCIATED if neither this nor | - | | | subject.name.strategy is specified. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Configuration dictionary passed to strategies | - |``subject.name.strategy.conf``| dict | that require additional configuration, such as | - | | | ASSOCIATED. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(SerializationContext, str) -> str | - | | | | - | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | - | | | constructed. Standard naming strategies are | - | | | defined in the confluent_kafka.schema_registry | - | | | namespace. Takes precedence over | - | | | subject.name.strategy.type if both are set. | - | | | | - | | | Defaults to None. | - +-----------------------------+----------+----------------------------------------------------+ - | | | Whether to validate the payload against the | - | ``validate`` | bool | the given schema. | - | | | | - +-----------------------------+----------+----------------------------------------------------+ - | | | Callable(bytes, SerializationContext, schema_id) | - | | | -> io.BytesIO | - | | | | - | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | - | | | Defaults to dual_schema_id_deserializer. | - +-----------------------------+----------+----------------------------------------------------+ + +----------------------------------+----------+----------------------------------------------------+ + | Property Name | Type | Description | + +==================================+==========+====================================================+ + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version for | + | ``use.latest.version`` | bool | deserialization. | + | | | | + | | | Defaults to False. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to use the latest subject version with | + | ``use.latest.with.metadata`` | dict | the given metadata. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | The type of subject name strategy to use. | + | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | + | | | ASSOCIATED. | + | | | | + | | | Defaults to ASSOCIATED if neither this nor | + | | | subject.name.strategy is specified. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Configuration dictionary passed to strategies | + | ``subject.name.strategy.conf`` | dict | that require additional configuration, such as | + | | | ASSOCIATED. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(SerializationContext, str) -> str | + | | | | + | ``subject.name.strategy`` | callable | Defines how Schema Registry subject names are | + | | | constructed. Standard naming strategies are | + | | | defined in the confluent_kafka.schema_registry | + | | | namespace. Takes precedence over | + | | | subject.name.strategy.type if both are set. | + | | | | + | | | Defaults to None. | + +----------------------------------+----------+----------------------------------------------------+ + | | | Whether to validate the payload against the | + | ``validate`` | bool | the given schema. | + | | | | + +----------------------------------+----------+----------------------------------------------------+ + | | | Callable(bytes, SerializationContext, schema_id) | + | | | -> io.BytesIO | + | | | | + | ``schema.id.deserializer`` | callable | Defines how the schema id/guid is deserialized. | + | | | Defaults to dual_schema_id_deserializer. | + +----------------------------------+----------+----------------------------------------------------+ Args: schema_str (str, Schema, optional): diff --git a/src/confluent_kafka/schema_registry/_sync/protobuf.py b/src/confluent_kafka/schema_registry/_sync/protobuf.py index e57e960e8..4b66845c5 100644 --- a/src/confluent_kafka/schema_registry/_sync/protobuf.py +++ b/src/confluent_kafka/schema_registry/_sync/protobuf.py @@ -152,7 +152,7 @@ class ProtobufSerializer(BaseSerializer): | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to ASSOCIATED if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-------------------------------------+----------+------------------------------------------------------+ | | | Configuration dictionary passed to strategies | @@ -534,7 +534,7 @@ class ProtobufDeserializer(BaseDeserializer): | ``subject.name.strategy.type`` | str | Valid values are: TOPIC, RECORD, TOPIC_RECORD, | | | | ASSOCIATED. | | | | | - | | | Defaults to ASSOCIATED if neither this nor | + | | | Defaults to ASSOCIATED if neither this nor | | | | subject.name.strategy is specified. | +-------------------------------------+----------+------------------------------------------------------+ | | | Configuration dictionary passed to strategies |