Skip to content

Commit 5ef2ce0

Browse files
committed
fix(streaming): keep CsvTransform options intact across repeated use
CsvTransform.transform popped 'encoding' and 'newline' from its own transform_options, so reusing an instance (or letting S3Object re-create its transformed stream) silently fell back to utf-8. Work on a copy instead.
1 parent bf2e917 commit 5ef2ce0

2 files changed

Lines changed: 29 additions & 4 deletions

File tree

‎aws_lambda_powertools/utilities/streaming/transformations/csv.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,12 @@ class CsvTransform(BaseTransform):
4848
"""
4949

5050
def transform(self, input_stream: IO[bytes]) -> DictReader:
51-
encoding = self.transform_options.pop("encoding", "utf-8")
52-
newline = self.transform_options.pop("newline", None)
51+
# Work on a copy: the same transform can be applied more than once (e.g. a reused instance, or when
52+
# S3Object re-creates its transformed stream), and each call must see the original options.
53+
options = dict(self.transform_options)
54+
encoding = options.pop("encoding", "utf-8")
55+
newline = options.pop("newline", None)
5356

5457
# csv module needs an Iterator[str], so we wrap the underlying stream into a TextIO
5558
iterator = io.TextIOWrapper(input_stream, encoding=encoding, newline=newline)
56-
return csv.DictReader(iterator, **self.transform_options)
59+
return csv.DictReader(iterator, **options)

‎tests/functional/streaming/_boto3/test_s3_object.py‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from aws_lambda_powertools.utilities.streaming import S3Object
1313
from aws_lambda_powertools.utilities.streaming._s3_seekable_io import _S3SeekableIO
1414
from aws_lambda_powertools.utilities.streaming.compat import PowertoolsStreamingBody
15-
from aws_lambda_powertools.utilities.streaming.transformations import GzipTransform, ZipTransform
15+
from aws_lambda_powertools.utilities.streaming.transformations import CsvTransform, GzipTransform, ZipTransform
1616

1717

1818
@pytest.fixture
@@ -114,3 +114,25 @@ def test_s3_closed_does_not_make_requests_for_zip(s3_client, s3_client_stub):
114114
# THEN no S3 request is issued (the stubber has no responses queued, so any call would fail)
115115
assert obj.closed is False
116116
s3_client_stub.assert_no_pending_responses()
117+
118+
119+
def _stub_get_object(s3_client_stub, payload: bytes):
120+
streaming_body = PowertoolsStreamingBody(raw_stream=io.BytesIO(payload), content_length=len(payload))
121+
s3_client_stub.add_response("get_object", {"Body": streaming_body})
122+
123+
124+
def test_csv_transform_keeps_options_when_reused(s3_client, s3_client_stub):
125+
# GIVEN a single CsvTransform with a non-default encoding applied to two objects
126+
transform = CsvTransform(encoding="latin-1", delimiter=";")
127+
payload = "name;city\ncafé;Zürich\n".encode("latin-1")
128+
129+
for _ in range(2):
130+
_stub_get_object(s3_client_stub, payload)
131+
132+
# WHEN each object is transformed
133+
rows = list(S3Object(bucket="bucket", key="key", boto3_client=s3_client).transform(transform))
134+
135+
# THEN the encoding is honoured every time, not only on the first use
136+
assert rows == [{"name": "café", "city": "Zürich"}]
137+
138+
assert transform.transform_options == {"encoding": "latin-1", "delimiter": ";"}

0 commit comments

Comments
 (0)