Skip to content

Commit 5785377

Browse files
feat(data_classes,parser): add EventBridge Custom Event Bus support (#8658)
* feat(data_classes,parser): add EventBridge Custom Event Bus support * fix(parser): decode octet-stream bytes in EventBridgeV2Envelope, relax optional system metadata, fix doc link --------- Co-authored-by: Leandro Damascena <lcdama@amazon.pt>
1 parent 45f7a4d commit 5785377

12 files changed

Lines changed: 510 additions & 0 deletions

File tree

‎aws_lambda_powertools/utilities/data_classes/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from .connect_contact_flow_event import ConnectContactFlowEvent
2929
from .dynamo_db_stream_event import DynamoDBStreamEvent
3030
from .event_bridge_event import EventBridgeEvent
31+
from .event_bridge_v2_event import EventBridgeV2Event
3132
from .event_source import event_source
3233
from .kafka_event import KafkaEvent
3334
from .kinesis_firehose_event import (
@@ -74,6 +75,7 @@
7475
"ConnectContactFlowEvent",
7576
"DynamoDBStreamEvent",
7677
"EventBridgeEvent",
78+
"EventBridgeV2Event",
7779
"KafkaEvent",
7880
"KinesisFirehoseEvent",
7981
"KinesisStreamEvent",
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
from __future__ import annotations
2+
3+
import base64
4+
from functools import cached_property
5+
from typing import Any, Literal
6+
7+
from aws_lambda_powertools.utilities.data_classes.common import DictWrapper
8+
9+
OCTET_STREAM = "application/octet-stream"
10+
11+
12+
def _is_with_metadata(item: Any) -> bool:
13+
return isinstance(item, dict) and "Data" in item and "SystemMetadata" in item
14+
15+
16+
class EventBridgeV2SystemMetadata(DictWrapper):
17+
"""Metadata added by the bus (`SystemMetadata`), only present with the WITH_METADATA transformer"""
18+
19+
@property
20+
def event_id(self) -> str | None:
21+
"""Unique event id assigned by the bus (`aws:EventId`). It is not the id of the classic envelope."""
22+
return self.get("aws:EventId")
23+
24+
@property
25+
def sequence_number(self) -> str | None:
26+
"""Sequence number of the event within the bus (`aws:SequenceNumber`)."""
27+
return self.get("aws:SequenceNumber")
28+
29+
@property
30+
def delivery_type(self) -> Literal["LIVE", "REPLAY"] | None:
31+
"""Whether the event is delivered as it arrives (`LIVE`) or as part of a replay (`REPLAY`)."""
32+
return self.get("aws:DeliveryType")
33+
34+
@property
35+
def ingestion_time(self) -> str | None:
36+
"""Time the bus ingested the event (`aws:IngestionTime`)."""
37+
return self.get("aws:IngestionTime")
38+
39+
@property
40+
def event_group_id(self) -> str | None:
41+
"""FIFO group id the event belongs to (`EventGroupId`)."""
42+
return self.get("EventGroupId")
43+
44+
@property
45+
def deduplication_id(self) -> str | None:
46+
"""Deduplication id of the event (`DeduplicationId`)."""
47+
return self.get("DeduplicationId")
48+
49+
@property
50+
def content_type(self) -> str | None:
51+
"""Content type of the payload, e.g. `application/json` or `application/eventbridge+json` (`ContentType`)."""
52+
return self.get("ContentType")
53+
54+
@property
55+
def source(self) -> str | None:
56+
"""Event source (`aws:Source`), only present for events published with PutEvents."""
57+
return self.get("aws:Source")
58+
59+
@property
60+
def detail_type(self) -> str | None:
61+
"""Event detail type (`aws:DetailType`), only present for events published with PutEvents."""
62+
return self.get("aws:DetailType")
63+
64+
65+
class EventBridgeV2Record(DictWrapper):
66+
"""A single event delivered by an Amazon EventBridge Custom Event Bus"""
67+
68+
@property
69+
def data(self) -> Any:
70+
"""Event payload: a dict for JSON, bytes for `application/octet-stream`."""
71+
data = self["Data"]
72+
if self.system_metadata.content_type == OCTET_STREAM and isinstance(data, str):
73+
return base64.b64decode(data)
74+
return data
75+
76+
@property
77+
def detail(self) -> dict[str, Any] | None:
78+
"""Shortcut to `data["detail"]` when the event was published with PutEvents, otherwise None."""
79+
data = self["Data"]
80+
if isinstance(data, dict) and "detail-type" in data and "detail" in data:
81+
return data["detail"]
82+
return None
83+
84+
@property
85+
def metadata(self) -> dict[str, Any]:
86+
"""Producer metadata keys sent with PutRawEvents."""
87+
return self.get("Metadata") or {}
88+
89+
@cached_property
90+
def system_metadata(self) -> EventBridgeV2SystemMetadata:
91+
"""Metadata added by the bus; empty when the subscriber doesn't use the WITH_METADATA transformer."""
92+
return EventBridgeV2SystemMetadata(self.get("SystemMetadata") or {})
93+
94+
95+
class EventBridgeV2Event(DictWrapper):
96+
"""Amazon EventBridge Custom Event Bus event
97+
98+
Accepts what a subscriber can deliver to Lambda: a single object or an array, using the
99+
WITH_METADATA transformer (`Data`, `Metadata`, `SystemMetadata`) or RAW (classic envelope,
100+
PutEvents array or the bare PutRawEvents payload). RAW deliveries carry no marker, so
101+
they are exposed with empty metadata.
102+
103+
Documentation:
104+
--------------
105+
- https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-custom-bus.html
106+
"""
107+
108+
@cached_property
109+
def records(self) -> list[EventBridgeV2Record]:
110+
"""Events in the delivery. Always a list, even when the bus sent a single object."""
111+
items = self._data if isinstance(self._data, list) else [self._data]
112+
return [
113+
EventBridgeV2Record(
114+
item if _is_with_metadata(item) else {"Data": item, "Metadata": {}, "SystemMetadata": {}},
115+
)
116+
for item in items
117+
]

‎aws_lambda_powertools/utilities/parser/envelopes/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from .cloudwatch import CloudWatchLogsEnvelope
77
from .dynamodb import DynamoDBStreamEnvelope
88
from .event_bridge import EventBridgeEnvelope
9+
from .event_bridge_v2 import EventBridgeV2Envelope
910
from .kafka import KafkaEnvelope
1011
from .kinesis import KinesisDataStreamEnvelope
1112
from .kinesis_firehose import KinesisFirehoseEnvelope
@@ -24,6 +25,7 @@
2425
"CloudWatchLogsEnvelope",
2526
"DynamoDBStreamEnvelope",
2627
"EventBridgeEnvelope",
28+
"EventBridgeV2Envelope",
2729
"KinesisDataStreamEnvelope",
2830
"KinesisFirehoseEnvelope",
2931
"LambdaFunctionUrlEnvelope",
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
from __future__ import annotations
2+
3+
import logging
4+
from typing import TYPE_CHECKING, Any
5+
6+
from aws_lambda_powertools.utilities.parser.envelopes.base import BaseEnvelope
7+
from aws_lambda_powertools.utilities.parser.models import EventBridgeV2Model
8+
9+
if TYPE_CHECKING:
10+
from aws_lambda_powertools.utilities.parser.types import T
11+
12+
logger = logging.getLogger(__name__)
13+
14+
15+
def _is_with_metadata(item: Any) -> bool:
16+
return isinstance(item, dict) and "Data" in item and "SystemMetadata" in item
17+
18+
19+
def _unwrap_put_events(data: Any) -> Any:
20+
if isinstance(data, dict) and "detail-type" in data and "detail" in data:
21+
return data["detail"]
22+
return data
23+
24+
25+
class EventBridgeV2Envelope(BaseEnvelope):
26+
"""EventBridge Custom Event Bus envelope to extract the payload of each record
27+
28+
Yields `Data` for events published with PutRawEvents, and `Data.detail` for PutEvents.
29+
RAW deliveries (classic envelope, PutEvents array, bare payload) are also accepted.
30+
"""
31+
32+
def parse(self, data: dict[str, Any] | Any | None, model: type[T]) -> list[T | None]:
33+
"""Parses records found with model provided
34+
35+
Parameters
36+
----------
37+
data : dict
38+
Lambda event to be parsed
39+
model : type[T]
40+
Data model provided to parse after extracting data using envelope
41+
42+
Returns
43+
-------
44+
list[T | None]
45+
List of payloads parsed with model provided
46+
"""
47+
items = data if isinstance(data, list) else [data]
48+
49+
payloads: list[Any] = []
50+
for item in items:
51+
if _is_with_metadata(item):
52+
logger.debug(f"Parsing incoming data with EventBridge V2 model {EventBridgeV2Model}")
53+
payload = EventBridgeV2Model.model_validate(item).root[0].data
54+
else:
55+
payload = item
56+
if isinstance(payload, bytes):
57+
# application/octet-stream is delivered as bytes; let the JSON path of _parse handle it
58+
payload = payload.decode()
59+
payloads.append(_unwrap_put_events(payload))
60+
61+
logger.debug(f"Parsing EventBridge V2 payloads with {model}")
62+
return [self._parse(data=payload, model=model) for payload in payloads]

‎aws_lambda_powertools/utilities/parser/models/__init__.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@
7777
DynamoDBStreamRecordModel,
7878
)
7979
from .event_bridge import EventBridgeModel
80+
from .event_bridge_v2 import EventBridgeV2Model, EventBridgeV2RecordModel, EventBridgeV2SystemMetadataModel
8081
from .kafka import (
8182
KafkaBaseEventModel,
8283
KafkaMskEventModel,
@@ -199,6 +200,9 @@
199200
"DynamoDBStreamModel": "dynamodb",
200201
"DynamoDBStreamRecordModel": "dynamodb",
201202
"EventBridgeModel": "event_bridge",
203+
"EventBridgeV2Model": "event_bridge_v2",
204+
"EventBridgeV2RecordModel": "event_bridge_v2",
205+
"EventBridgeV2SystemMetadataModel": "event_bridge_v2",
202206
"KafkaBaseEventModel": "kafka",
203207
"KafkaMskEventModel": "kafka",
204208
"KafkaRecordModel": "kafka",
@@ -311,6 +315,9 @@ def __dir__() -> list[str]:
311315
"AppSyncEventsModel",
312316
"DynamoDBStreamModel",
313317
"EventBridgeModel",
318+
"EventBridgeV2Model",
319+
"EventBridgeV2RecordModel",
320+
"EventBridgeV2SystemMetadataModel",
314321
"DynamoDBStreamChangedRecordModel",
315322
"DynamoDBStreamLambdaOnFailureDestinationModel",
316323
"DynamoDBStreamRecordModel",
Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
from __future__ import annotations
2+
3+
import base64
4+
from typing import Any, Literal
5+
6+
from pydantic import BaseModel, ConfigDict, Field, RootModel, model_validator
7+
8+
OCTET_STREAM = "application/octet-stream"
9+
10+
11+
class EventBridgeV2SystemMetadataModel(BaseModel):
12+
model_config = ConfigDict(populate_by_name=True)
13+
14+
event_id: str | None = Field(None, alias="aws:EventId", description="Unique event id assigned by the bus.")
15+
sequence_number: str | None = Field(
16+
None,
17+
alias="aws:SequenceNumber",
18+
description="Sequence number of the event in the bus.",
19+
)
20+
delivery_type: Literal["LIVE", "REPLAY"] = Field(
21+
alias="aws:DeliveryType",
22+
description="Whether the event is delivered live or as part of a replay.",
23+
)
24+
ingestion_time: str | None = Field(None, alias="aws:IngestionTime", description="Time the bus ingested the event.")
25+
event_group_id: str | None = Field(None, alias="EventGroupId", description="FIFO group id of the event.")
26+
deduplication_id: str | None = Field(None, alias="DeduplicationId", description="Deduplication id of the event.")
27+
content_type: str | None = Field(
28+
None,
29+
alias="ContentType",
30+
description="Content type of the payload.",
31+
examples=["application/json", "application/eventbridge+json", "application/octet-stream"],
32+
)
33+
source: str | None = Field(None, alias="aws:Source", description="Event source, only for PutEvents.")
34+
detail_type: str | None = Field(None, alias="aws:DetailType", description="Detail type, only for PutEvents.")
35+
36+
37+
class EventBridgeV2RecordModel(BaseModel):
38+
data: Any = Field(
39+
alias="Data",
40+
description="Event payload. Bytes for `application/octet-stream`, the classic envelope for PutEvents.",
41+
)
42+
metadata: dict[str, Any] = Field(default_factory=dict, alias="Metadata", description="Producer metadata.")
43+
system_metadata: EventBridgeV2SystemMetadataModel = Field(
44+
alias="SystemMetadata",
45+
description="Metadata added by the bus.",
46+
)
47+
48+
model_config = ConfigDict(populate_by_name=True)
49+
50+
@model_validator(mode="before")
51+
@classmethod
52+
def decode_octet_stream(cls, values: Any) -> Any:
53+
if not isinstance(values, dict):
54+
return values
55+
data = values.get("Data", values.get("data"))
56+
system_metadata = values.get("SystemMetadata", values.get("system_metadata")) or {}
57+
content_type = system_metadata.get("ContentType") if isinstance(system_metadata, dict) else None
58+
if content_type == OCTET_STREAM and isinstance(data, str):
59+
return {**values, "Data": base64.b64decode(data)}
60+
return values
61+
62+
63+
class EventBridgeV2Model(RootModel[list[EventBridgeV2RecordModel]]):
64+
"""Amazon EventBridge Custom Event Bus delivery using the WITH_METADATA transformer.
65+
66+
Accepts a single object or an array, and always exposes a list of records in `root`.
67+
"""
68+
69+
@model_validator(mode="before")
70+
@classmethod
71+
def wrap_single_record(cls, value: Any) -> Any:
72+
return value if isinstance(value, list) else [value]

‎docs/utilities/data_classes.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ Each event source is linked to its corresponding GitHub file with the full set o
8787
| [Connect Contact Flow](#connect-contact-flow) | `ConnectContactFlowEvent` | [Github](https://github.com/aws-powertools/powertools-lambda-python/blob/develop/aws_lambda_powertools/utilities/data_classes/connect_contact_flow_event.py) |
8888
| [DynamoDB streams](#dynamodb-streams) | `DynamoDBStreamEvent`, `DynamoDBRecordEventName` | [Github](https://github.com/aws-powertools/powertools-lambda-python/blob/develop/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py) |
8989
| [EventBridge](#eventbridge) | `EventBridgeEvent` | [Github](https://github.com/aws-powertools/powertools-lambda-python/blob/develop/aws_lambda_powertools/utilities/data_classes/event_bridge_event.py) |
90+
| [EventBridge Custom Event Bus](#eventbridge-custom-event-bus) | `EventBridgeV2Event` | [Github](https://github.com/aws-powertools/powertools-lambda-python/blob/develop/aws_lambda_powertools/utilities/data_classes/event_bridge_v2_event.py) |
9091
| [Kafka](#kafka) | `KafkaEvent` | [Github](https://github.com/aws-powertools/powertools-lambda-python/blob/develop/aws_lambda_powertools/utilities/data_classes/kafka_event.py) |
9192
| [Kinesis Data Stream](#kinesis-streams) | `KinesisStreamEvent` | [Github](https://github.com/aws-powertools/powertools-lambda-python/blob/develop/aws_lambda_powertools/utilities/data_classes/kinesis_stream_event.py) |
9293
| [Kinesis Firehose Delivery Stream](#kinesis-firehose-delivery-stream) | `KinesisFirehoseEvent` | [Github](https://github.com/aws-powertools/powertools-lambda-python/blob/develop/aws_lambda_powertools/utilities/data_classes/kinesis_firehose_event.py) |
@@ -552,6 +553,23 @@ The class automatically deserializes DynamoDB types into their equivalent Python
552553
--8<-- "tests/events/eventBridgeEvent.json"
553554
```
554555

556+
### EventBridge Custom Event Bus
557+
558+
The [Custom Event Bus](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-custom-bus.html) delivers a single object or an array of events, depending on the subscriber batching. `EventBridgeV2Event` always exposes them through `records`.
559+
560+
Use the `WITH_METADATA` transformer when you need event ids, FIFO groups or replay detection (`system_metadata.delivery_type`). `RAW` deliveries carry no marker, so records are still accepted (classic envelope, PutEvents array or bare PutRawEvents payload) but `metadata` and `system_metadata` are empty.
561+
562+
=== "app.py"
563+
564+
```python hl_lines="1 5"
565+
--8<-- "examples/event_sources/src/eventBridgeV2Event.py"
566+
```
567+
568+
=== "EventBridge Custom Event Bus Example Event"
569+
```json
570+
--8<-- "tests/events/eventBridgeV2Event.json"
571+
```
572+
555573
### Kafka
556574

557575
This example is based on the AWS docs for [Amazon MSK](https://docs.aws.amazon.com/lambda/latest/dg/with-msk.html){target="_blank"} and [self-managed Apache Kafka](https://docs.aws.amazon.com/lambda/latest/dg/with-kafka.html){target="_blank"}.

‎docs/utilities/parser.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ The example above uses `SqsModel`. Other built-in models can be found below.
135135
| **DynamoDBStreamModel** | Lambda Event Source payload for Amazon DynamoDB Streams |
136136
| **DynamoDBStreamLambdaOnFailureDestinationModel** | Lambda on-failure destination payload for Amazon DynamoDB Streams |
137137
| **EventBridgeModel** | Lambda Event Source payload for Amazon EventBridge |
138+
| **EventBridgeV2Model** | Lambda Event Source payload for Amazon EventBridge Custom Event Bus (`WITH_METADATA`, single or array) |
138139
| **IoTCoreThingEvent** | Lambda Event Source payload for IoT Core Thing created, updated, or deleted. |
139140
| **IoTCoreThingTypeEvent** | Lambda Event Source payload for IoT Core Thing Type events. |
140141
| **IoTCoreThingTypeAssociationEvent** | Lambda Event Source payload for IoT Core Thing Type associated or disassociated with a Thing. |
@@ -209,6 +210,7 @@ You can use pre-built envelopes provided by the Parser to extract and parse spec
209210
| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------- |
210211
| **DynamoDBStreamEnvelope** | 1. Parses data using `DynamoDBStreamModel`. `` 2. Parses records in `NewImage` and `OldImage` keys using your model. `` 3. Returns a list with a dictionary containing `NewImage` and `OldImage` keys | `List[Dict[str, Optional[Model]]]` |
211212
| **EventBridgeEnvelope** | 1. Parses data using `EventBridgeModel`. ``2. Parses `detail` key using your model`` and returns it. | `Model` |
213+
| **EventBridgeV2Envelope** | 1. Parses data using `EventBridgeV2Model`. ``2. Parses `Data` (or `Data.detail` for PutEvents) of each record using your model`` and returns them in a list. Also accepts RAW deliveries. | `List[Model]` |
212214
| **SqsEnvelope** | 1. Parses data using `SqsModel`. ``2. Parses records in `body` key using your model`` and return them in a list. | `List[Model]` |
213215
| **CloudWatchLogsEnvelope** | 1. Parses data using `CloudwatchLogsModel` which will base64 decode and decompress it. ``2. Parses records in `message` key using your model`` and return them in a list. | `List[Model]` |
214216
| **KinesisDataStreamEnvelope** | 1. Parses data using `KinesisDataStreamModel` which will base64 decode it. ``2. Parses records in the `Records` key using your model`` and returns them in a list. | `List[Model]` |
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
from aws_lambda_powertools.utilities.data_classes import EventBridgeV2Event, event_source
2+
3+
4+
@event_source(data_class=EventBridgeV2Event)
5+
def lambda_handler(event: EventBridgeV2Event, context):
6+
for record in event.records:
7+
if record.system_metadata.delivery_type == "REPLAY":
8+
continue
9+
10+
# Do something with record.data, or record.detail when published with PutEvents
11+
print(record.system_metadata.event_id, record.data)

0 commit comments

Comments
 (0)