Skip to content

Commit 1545ba0

Browse files
committed
fix(batch): report short-circuited FIFO records that don't match the model
1 parent 5785377 commit 1545ba0

2 files changed

Lines changed: 35 additions & 1 deletion

File tree

‎aws_lambda_powertools/utilities/batch/sqs_fifo_partial_processor.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,8 +100,14 @@ def _process_record(self, record):
100100
fail_entire_batch = bool(self.fail_messages) and not self._skip_group_on_error
101101
fail_group_id = self._skip_group_on_error and self._current_group_id in self._failed_group_ids
102102
if fail_entire_batch or fail_group_id:
103+
try:
104+
record_to_fail = self._to_batch_type(record, event_type=self.event_type, model=self.model)
105+
except Exception:
106+
# The record never reaches the handler, so register it as a data class like other poison pills
107+
record_to_fail = self._to_batch_type(record, event_type=self.event_type)
108+
103109
return self.failure_handler(
104-
record=self._to_batch_type(record, event_type=self.event_type, model=self.model),
110+
record=record_to_fail,
105111
exception=self.group_circuit_breaker_exc if self._skip_group_on_error else self.circuit_breaker_exc,
106112
)
107113

‎tests/functional/batch/_pydantic/test_utilities_batch_pydantic.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -486,6 +486,34 @@ def lambda_handler(event, context):
486486
assert result["batchItemFailures"][2]["itemIdentifier"] == fifth_record.message_id
487487

488488

489+
@pytest.mark.parametrize("skip_group_on_error", [False, True])
490+
def test_sqs_fifo_batch_processor_model_with_validation_error_after_failure(
491+
skip_group_on_error: bool,
492+
record_handler_model: Callable,
493+
sqs_event_fifo_factory,
494+
order_event_factory,
495+
):
496+
# GIVEN a malformed record that is short-circuited because a previous record of its group failed
497+
first_record = sqs_event_fifo_factory(order_event_factory({"type": "success"}), "2")
498+
failed_record = sqs_event_fifo_factory(order_event_factory({"type": "fail"}), "1")
499+
malformed_record = sqs_event_fifo_factory(json.dumps({"poison": "pill"}), "1")
500+
fourth_record = sqs_event_fifo_factory(order_event_factory({"type": "success"}), "2")
501+
records = [first_record, failed_record, malformed_record, fourth_record]
502+
503+
# WHEN
504+
processor = SqsFifoPartialProcessor(model=OrderSqs, skip_group_on_error=skip_group_on_error)
505+
with processor(records, record_handler_model) as batch:
506+
batch.process()
507+
508+
# THEN the malformed record is reported instead of failing the whole batch
509+
expected_failures = [failed_record, malformed_record]
510+
if not skip_group_on_error:
511+
expected_failures.append(fourth_record)
512+
assert batch.response() == {
513+
"batchItemFailures": [{"itemIdentifier": record["messageId"]} for record in expected_failures],
514+
}
515+
516+
489517
def test_batch_processor_model_with_partial_validation_error(
490518
record_handler_model: Callable,
491519
sqs_event_factory,

0 commit comments

Comments
 (0)