Skip to content

Commit db2f8f0

Browse files
fix(kafka): decode Confluent Protobuf message indexes as zigzag varints (#8670)
* fix(kafka): decode Confluent Protobuf message indexes as zigzag varints Confluent's Protobuf serializer writes the message index array as zigzag encoded varints (array length, then each index), with [0] as a single 0x00 byte. ProtobufDeserializer read it with _DecodeSignedVarint, so it read twice the real length and consumed payload bytes as indexes. Records for any message type other than the first in the schema either failed to decode or decoded with fields silently missing. Decode the array as zigzag varints. Tighten test_confluent_schema_registry_complex_schema to assert userId and name, which the 04 02 00 fixture record was losing, and add a parametrised test for indexes [0], [1], [2], [1, 0] and a multi-byte index. Closes #8665 * test(kafka): cover an invalid Confluent message index array length --------- Co-authored-by: Leandro Damascena <lcdama@amazon.pt>
1 parent 07f2a7a commit db2f8f0

2 files changed

Lines changed: 121 additions & 12 deletions

File tree

‎aws_lambda_powertools/utilities/kafka/deserializer/protobuf.py‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
from typing import Any
55

66
from google.protobuf.internal.decoder import ( # type: ignore[attr-defined]
7-
_DecodeSignedVarint, # ty: ignore[unresolved-import]
7+
_DecodeVarint, # ty: ignore[unresolved-import]
88
)
99
from google.protobuf.json_format import MessageToDict
1010

@@ -103,21 +103,29 @@ def _remove_message_index(self, data):
103103
"""
104104
Identifies and removes Confluent Schema Registry MessageIndex from bytes.
105105
Returns pure protobuf bytes.
106+
107+
Confluent serializers write the message index array as zigzag encoded varints:
108+
the array length followed by each index. The common case, index ``[0]``, is a single ``0x00`` byte.
109+
This matches ``_encode_varints`` / ``_read_index_array`` in ``confluent_kafka.schema_registry.protobuf``.
106110
"""
107111
buffer = memoryview(data)
108-
pos = 0
109112

110113
logger.debug("Removing message list bytes")
111114

112-
# Read first varint (index count or 0)
113-
first_value, new_pos = _DecodeSignedVarint(buffer, pos)
114-
pos = new_pos
115+
# Read the number of indexes (0 means the first message type in the schema)
116+
count, pos = _decode_zigzag_varint(buffer, 0)
117+
if count < 0:
118+
raise ValueError(f"Invalid Confluent message index array length: {count}")
115119

116120
# Skip index values if present
117-
if first_value != 0:
118-
for _ in range(first_value):
119-
_, new_pos = _DecodeSignedVarint(buffer, pos)
120-
pos = new_pos
121+
for _ in range(count):
122+
_, pos = _decode_zigzag_varint(buffer, pos)
121123

122124
# Return remaining bytes (pure protobuf)
123125
return data[pos:]
126+
127+
128+
def _decode_zigzag_varint(buffer: memoryview, pos: int) -> tuple[int, int]:
129+
"""Decode a zigzag encoded varint, as used by Confluent for message indexes."""
130+
value, new_pos = _DecodeVarint(buffer, pos)
131+
return (value >> 1) ^ -(value & 1), new_pos

‎tests/functional/kafka_consumer/_protobuf/test_kafka_consumer_with_protobuf.py‎

Lines changed: 104 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -357,18 +357,24 @@ def test_confluent_schema_registry_complex_schema(lambda_context):
357357
def handler(event: ConsumerRecords, context):
358358
for record in event.records:
359359
processed_records.append(
360-
{"email": record.value["email"], "age": record.value["age"]},
360+
{
361+
"userId": record.value.get("userId"),
362+
"name": record.value.get("name"),
363+
"email": record.value["email"],
364+
"age": record.value["age"],
365+
},
361366
)
362367
return {"processed": len(processed_records)}
363368

364369
# WHEN The handler processes a Kafka event containing Protobuf-encoded data
365370
result = handler(complex_event, lambda_context)
366371

367372
# THEN
368-
# The handler should successfully process both records
369-
# and return the correct count
373+
# The handler should successfully process all records
374+
# and return the correct count, without losing any field
370375
assert result == {"processed": 4}
371376
assert len(processed_records) == 4
377+
assert all(r["userId"] == "123" and r["name"] == "Test" for r in processed_records)
372378

373379

374380
def test_glue_schema_registry_complex_schema(lambda_context):
@@ -429,3 +435,98 @@ def handler(event: ConsumerRecords, context):
429435
# THEN
430436
# Ensure the error contains useful diagnostic information
431437
assert "Expected data is PROTOBUF but you sent " in str(excinfo.value)
438+
439+
440+
def _confluent_message_index(indexes: list[int]) -> bytes:
441+
# Same encoding as confluent_kafka's ProtobufSerializer: zigzag varints, [0] written as a single 0x00 byte
442+
def zigzag_varint(value: int) -> bytes:
443+
value = (value << 1) ^ (value >> 63)
444+
out = bytearray()
445+
while value & ~0x7F:
446+
out.append((value & 0x7F) | 0x80)
447+
value >>= 7
448+
out.append(value)
449+
return bytes(out)
450+
451+
if indexes == [0]:
452+
return b"\x00"
453+
return zigzag_varint(len(indexes)) + b"".join(zigzag_varint(i) for i in indexes)
454+
455+
456+
@pytest.mark.parametrize(
457+
"indexes",
458+
[
459+
pytest.param([0], id="first_message"),
460+
pytest.param([1], id="second_message"),
461+
pytest.param([2], id="third_message"),
462+
pytest.param([1, 0], id="nested_message"),
463+
pytest.param([70], id="multi_byte_index"),
464+
],
465+
)
466+
def test_confluent_schema_registry_message_index_is_zigzag_decoded(indexes, lambda_context):
467+
# GIVEN a Protobuf message framed with a Confluent message index array, as the Confluent serializer writes it
468+
profile = UserProfile(userId="123", name="Test", email="test@example.com", age=10)
469+
value = _confluent_message_index(indexes) + profile.SerializeToString()
470+
event = {
471+
"eventSource": "aws:kafka",
472+
"records": {
473+
"my-topic-1": [
474+
{
475+
"topic": "my-topic-1",
476+
"partition": 0,
477+
"offset": 15,
478+
"timestamp": 1545084650987,
479+
"timestampType": "CREATE_TIME",
480+
"key": None,
481+
"value": base64.b64encode(value).decode("utf-8"),
482+
"headers": [],
483+
"valueSchemaMetadata": {"schemaId": "123", "dataFormat": "PROTOBUF"},
484+
},
485+
],
486+
},
487+
}
488+
schema_config = SchemaConfig(value_schema_type="PROTOBUF", value_schema=UserProfile)
489+
490+
@kafka_consumer(schema_config=schema_config)
491+
def handler(event: ConsumerRecords, context):
492+
return next(event.records).value
493+
494+
# WHEN the handler processes the record
495+
result = handler(event, lambda_context)
496+
497+
# THEN the index array is skipped and every field is decoded
498+
assert result == {"userId": "123", "name": "Test", "email": "test@example.com", "age": 10}
499+
500+
501+
def test_confluent_schema_registry_invalid_message_index_length_raises(lambda_context):
502+
# GIVEN a record whose Confluent message index array has a negative length (zigzag 0x01 is -1)
503+
profile = UserProfile(userId="123", name="Test", email="test@example.com", age=10)
504+
value = b"\x01" + profile.SerializeToString()
505+
event = {
506+
"eventSource": "aws:kafka",
507+
"records": {
508+
"my-topic-1": [
509+
{
510+
"topic": "my-topic-1",
511+
"partition": 0,
512+
"offset": 15,
513+
"timestamp": 1545084650987,
514+
"timestampType": "CREATE_TIME",
515+
"key": None,
516+
"value": base64.b64encode(value).decode("utf-8"),
517+
"headers": [],
518+
"valueSchemaMetadata": {"schemaId": "123", "dataFormat": "PROTOBUF"},
519+
},
520+
],
521+
},
522+
}
523+
schema_config = SchemaConfig(value_schema_type="PROTOBUF", value_schema=UserProfile)
524+
525+
@kafka_consumer(schema_config=schema_config)
526+
def handler(event: ConsumerRecords, context):
527+
return [record.value for record in event.records]
528+
529+
# WHEN the record is deserialized
530+
# THEN the invalid index array is reported as a deserialization error instead of corrupt payload bytes
531+
with pytest.raises(KafkaConsumerDeserializationError, match="message index array length"):
532+
handler(event, lambda_context)

0 commit comments

Comments
 (0)