Skip to content

Commit ede714e

Browse files
fix(parameters): handle transform failures gracefully in get_parameters_by_name (#8623)
Co-authored-by: Leandro Damascena <lcdama@amazon.pt>
1 parent 92a28eb commit ede714e

2 files changed

Lines changed: 170 additions & 11 deletions

File tree

‎aws_lambda_powertools/utilities/parameters/ssm.py‎

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,11 @@
2727
SSM_PARAMETER_TIER,
2828
SSM_PARAMETER_TYPES,
2929
)
30-
from aws_lambda_powertools.utilities.parameters.exceptions import GetParameterError, SetParameterError
30+
from aws_lambda_powertools.utilities.parameters.exceptions import (
31+
GetParameterError,
32+
SetParameterError,
33+
TransformParameterError,
34+
)
3135
from aws_lambda_powertools.warnings import PowertoolsDeprecationWarning
3236

3337
if TYPE_CHECKING:
@@ -532,7 +536,7 @@ def _get_parameters_by_name_with_decrypt_option(
532536
for parameter, options in batch.items():
533537
try:
534538
response[parameter] = self.get(parameter, options["max_age"], options["transform"], options["decrypt"])
535-
except GetParameterError:
539+
except (GetParameterError, TransformParameterError):
536540
if raise_on_error:
537541
raise
538542
errors.append(parameter)
@@ -635,7 +639,12 @@ def _get_parameters_by_name(
635639
return ret, parameter_names
636640

637641
batch_errors = self._handle_any_invalid_get_parameter_errors(response, raise_on_error)
638-
transformed_params = self._transform_and_cache_get_parameters_response(response, parameters, raise_on_error)
642+
transformed_params, transform_errors = self._transform_and_cache_get_parameters_response(
643+
response,
644+
parameters,
645+
raise_on_error,
646+
)
647+
batch_errors.extend(transform_errors)
639648

640649
return transformed_params, batch_errors
641650

@@ -644,8 +653,9 @@ def _transform_and_cache_get_parameters_response(
644653
api_response: GetParametersResultTypeDef,
645654
parameters: dict[str, Any],
646655
raise_on_error: bool = True,
647-
) -> dict[str, Any]:
656+
) -> tuple[dict[str, Any], list[str]]:
648657
response: dict[str, Any] = {}
658+
transform_errors: list[str] = []
649659

650660
for parameter in api_response["Parameters"]:
651661
# For names with a version or label selector (e.g. "/a:prod"), SSM returns the bare name
@@ -657,19 +667,28 @@ def _transform_and_cache_get_parameters_response(
657667

658668
# NOTE: If transform is set, we do it before caching to reduce number of operations
659669
if transform:
660-
value = transform_value(
661-
value=value,
662-
transform=transform,
663-
raise_on_transform_error=raise_on_error,
664-
key=name,
665-
) # type: ignore[assignment]
670+
try:
671+
value = transform_value(
672+
value=value,
673+
transform=transform,
674+
raise_on_transform_error=True,
675+
key=name,
676+
) # type: ignore[assignment]
677+
except TransformParameterError:
678+
if raise_on_error:
679+
raise
680+
681+
# NOTE: Don't cache failed transforms, otherwise a corrected parameter
682+
# would keep returning None from the cache until max_age expires
683+
transform_errors.append(name)
684+
continue
666685

667686
_cache_key = (name, transform)
668687
self.add_to_cache(key=_cache_key, value=value, max_age=options["max_age"])
669688

670689
response[name] = value
671690

672-
return response
691+
return response, transform_errors
673692

674693
@staticmethod
675694
def _handle_any_invalid_get_parameter_errors(

‎tests/functional/parameters/_boto3/test_utilities_parameters.py‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1220,6 +1220,146 @@ def test_ssm_provider_get_parameters_by_name_do_not_raise_on_failure_transform(m
12201220
stubber.deactivate()
12211221

12221222

1223+
def test_ssm_provider_get_parameters_by_name_do_not_raise_on_transform_failure_with_decrypt(mock_name):
1224+
# GIVEN one parameter needs decryption and its value can't be transformed
1225+
# and a second parameter is successfully fetched through the batch API
1226+
decrypt_fail = f"/decrypt/{mock_name}"
1227+
success = f"/batch/{mock_name}"
1228+
params = {decrypt_fail: {"decrypt": True}, success: {}}
1229+
expected_value = {"value": "success"}
1230+
1231+
class FakeClient:
1232+
def get_parameter(self, **kwargs):
1233+
assert kwargs["Name"] == decrypt_fail
1234+
assert kwargs["WithDecryption"] is True
1235+
return {"Parameter": {"Name": decrypt_fail, "Value": "not json", "Type": "String", "Version": 1}}
1236+
1237+
def get_parameters(self, **kwargs):
1238+
assert kwargs["Names"] == [success]
1239+
return build_get_parameters_stub(params={success: json.dumps(expected_value)})
1240+
1241+
provider = parameters.SSMProvider(boto3_client=FakeClient())
1242+
1243+
# WHEN fail-fast is disabled in get_parameters_by_name
1244+
ret = provider.get_parameters_by_name(parameters=params, transform="json", raise_on_error=False)
1245+
1246+
# THEN no error is raised, the transform failure is reported under "_errors",
1247+
# and its value is not returned
1248+
assert ret[success] == expected_value
1249+
assert ret["_errors"] == [decrypt_fail]
1250+
assert decrypt_fail not in ret
1251+
1252+
1253+
def test_ssm_provider_get_parameters_by_name_do_not_cache_failed_transform(mock_name):
1254+
# GIVEN a parameter whose value can't be transformed
1255+
param = f"/{mock_name}"
1256+
params = {param: {}}
1257+
corrected_value = {"value": "corrected"}
1258+
1259+
class FakeClient:
1260+
def __init__(self):
1261+
self.get_parameters_calls = 0
1262+
1263+
def get_parameters(self, **kwargs):
1264+
self.get_parameters_calls += 1
1265+
assert kwargs["Names"] == [param]
1266+
1267+
if self.get_parameters_calls == 1:
1268+
return build_get_parameters_stub(params={param: "not json"})
1269+
1270+
return build_get_parameters_stub(params={param: json.dumps(corrected_value)})
1271+
1272+
client = FakeClient()
1273+
provider = parameters.SSMProvider(boto3_client=client)
1274+
1275+
# WHEN fail-fast is disabled in get_parameters_by_name
1276+
ret = provider.get_parameters_by_name(parameters=params, transform="json", raise_on_error=False)
1277+
1278+
# THEN the transform failure is reported and not cached
1279+
assert ret["_errors"] == [param]
1280+
assert param not in ret
1281+
assert (param, "json") not in provider.store
1282+
1283+
# WHEN fetching again after the parameter is corrected
1284+
ret = provider.get_parameters_by_name(parameters=params, transform="json", raise_on_error=False)
1285+
1286+
# THEN a fresh request is made and the corrected value is returned instead of a cached failure
1287+
assert client.get_parameters_calls == 2
1288+
assert ret["_errors"] == []
1289+
assert ret[param] == corrected_value
1290+
assert (param, "json") in provider.store
1291+
1292+
1293+
def test_ssm_provider_get_parameters_by_name_raise_on_transform_failure_with_decrypt(mock_name, config):
1294+
# GIVEN one parameter needs decryption and its value can't be transformed
1295+
decrypt_fail = f"/decrypt/{mock_name}"
1296+
success = f"/batch/{mock_name}"
1297+
params = {decrypt_fail: {"decrypt": True}, success: {}}
1298+
1299+
provider = parameters.SSMProvider(boto_config=config)
1300+
stubber = stub.Stubber(provider.client)
1301+
stubber.add_response(
1302+
"get_parameter",
1303+
{"Parameter": {"Name": decrypt_fail, "Value": "not json", "Type": "String", "Version": 1}},
1304+
{"Name": decrypt_fail, "WithDecryption": True},
1305+
)
1306+
stubber.activate()
1307+
1308+
# WHEN using get_parameters_by_name with fail-fast enabled (default)
1309+
try:
1310+
with pytest.raises(parameters.TransformParameterError):
1311+
provider.get_parameters_by_name(parameters=params, transform="json")
1312+
1313+
stubber.assert_no_pending_responses()
1314+
finally:
1315+
stubber.deactivate()
1316+
1317+
1318+
def test_ssm_provider_get_parameters_by_name_raise_on_batch_transform_failure(mock_name):
1319+
# GIVEN a batch parameter whose value can't be transformed
1320+
param = f"/{mock_name}"
1321+
params = {param: {}}
1322+
1323+
class FakeClient:
1324+
def get_parameters(self, **kwargs):
1325+
assert kwargs["Names"] == [param]
1326+
return build_get_parameters_stub(params={param: "not json"})
1327+
1328+
provider = parameters.SSMProvider(boto3_client=FakeClient())
1329+
1330+
# WHEN using get_parameters_by_name with fail-fast enabled (default)
1331+
with pytest.raises(parameters.TransformParameterError):
1332+
provider.get_parameters_by_name(parameters=params, transform="json")
1333+
1334+
1335+
def test_ssm_provider_get_parameters_by_name_json_null_is_not_a_transform_failure(mock_name):
1336+
# GIVEN parameters whose values are valid JSON null
1337+
decrypt_param = f"/decrypt/{mock_name}"
1338+
batch_param = f"/batch/{mock_name}"
1339+
params = {decrypt_param: {"decrypt": True}, batch_param: {}}
1340+
1341+
class FakeClient:
1342+
def get_parameter(self, **kwargs):
1343+
assert kwargs["Name"] == decrypt_param
1344+
assert kwargs["WithDecryption"] is True
1345+
return {"Parameter": {"Name": decrypt_param, "Value": "null", "Type": "String", "Version": 1}}
1346+
1347+
def get_parameters(self, **kwargs):
1348+
assert kwargs["Names"] == [batch_param]
1349+
return build_get_parameters_stub(params={batch_param: "null"})
1350+
1351+
provider = parameters.SSMProvider(boto3_client=FakeClient())
1352+
1353+
# WHEN fail-fast is disabled in get_parameters_by_name
1354+
ret = provider.get_parameters_by_name(parameters=params, transform="json", raise_on_error=False)
1355+
1356+
# THEN JSON null is a successful transform, not reported as a failure
1357+
assert ret["_errors"] == []
1358+
assert ret[decrypt_param] is None
1359+
assert ret[batch_param] is None
1360+
assert (batch_param, "json") in provider.store
1361+
1362+
12231363
def test_ssm_provider_get_parameters_by_name_auto_transform_infers_from_parameter_name(mock_name, mock_value, config):
12241364
# GIVEN transform="auto" and a parameter name suffixed with ".json"
12251365
# (auto-transform must infer the transform method from the parameter name, not its value)

0 commit comments

Comments
 (0)