From e7518b4360aef1f3548bfbe021dca93452afa877 Mon Sep 17 00:00:00 2001 From: john-jac <75442233+john-jac@users.noreply.github.com> Date: Fri, 29 Oct 2021 13:53:18 -0700 Subject: [PATCH 1/4] Update secrets_manager.py When a Secrets Manager call fails due to the secret not being accessible to the IAM principle, rather than just not existing, it reports: botocore.exceptions.ClientError: An error occurred (AccessDeniedException) when calling the GetSecretValue operation: User: arn:aws:sts::xxx is not authorized to perform: secretsmanager:GetSecretValue on resource: yyy because no identity-based policy allows the secretsmanager:GetSecretValue action For example, if a user wants to limit an environment to secrets that are tagged with a specific resource. Today, this causes a failure with the Scheduler as an uncaught exception. This PR addresses that condition, plus adds an additional catch for other exceptions that we're not aware of yet. --- .../providers/amazon/aws/secrets/secrets_manager.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/airflow/providers/amazon/aws/secrets/secrets_manager.py b/airflow/providers/amazon/aws/secrets/secrets_manager.py index 35ddf764bd7ff..94fa8c543fda1 100644 --- a/airflow/providers/amazon/aws/secrets/secrets_manager.py +++ b/airflow/providers/amazon/aws/secrets/secrets_manager.py @@ -253,3 +253,16 @@ def _get_secret(self, path_prefix, secret_id: str) -> Optional[str]: secret_id, ) return None + except self.client.exceptions.AccessDeniedException as e: + self.log.debug( + "An error occurred (AccessDeniedException) when calling the " + "get_secret_value operation: %s", + str(e), + ) + return None + except Exception as e: + self.log.debug( + "An unknown exception occurred when calling the get_secret_value operation: %s", + str(e), + ) + return None From b94c00729b1b9ed2575eda41f395f7a9661596d1 Mon Sep 17 00:00:00 2001 From: john-jac <75442233+john-jac@users.noreply.github.com> Date: Mon, 15 Nov 2021 10:27:46 -0800 Subject: [PATCH 2/4] Removed final generic Exception in secrets_manager.py --- airflow/providers/amazon/aws/secrets/secrets_manager.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/airflow/providers/amazon/aws/secrets/secrets_manager.py b/airflow/providers/amazon/aws/secrets/secrets_manager.py index 94fa8c543fda1..e4956661ebab1 100644 --- a/airflow/providers/amazon/aws/secrets/secrets_manager.py +++ b/airflow/providers/amazon/aws/secrets/secrets_manager.py @@ -260,9 +260,3 @@ def _get_secret(self, path_prefix, secret_id: str) -> Optional[str]: str(e), ) return None - except Exception as e: - self.log.debug( - "An unknown exception occurred when calling the get_secret_value operation: %s", - str(e), - ) - return None From cfa61397bd2bdc2ed45c5512bb6d64abb6971ab9 Mon Sep 17 00:00:00 2001 From: john-jac <75442233+john-jac@users.noreply.github.com> Date: Mon, 15 Nov 2021 11:04:20 -0800 Subject: [PATCH 3/4] Use built-in exception logging Co-authored-by: Tzu-ping Chung --- airflow/providers/amazon/aws/secrets/secrets_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/providers/amazon/aws/secrets/secrets_manager.py b/airflow/providers/amazon/aws/secrets/secrets_manager.py index e4956661ebab1..8a1cd94f6a5dc 100644 --- a/airflow/providers/amazon/aws/secrets/secrets_manager.py +++ b/airflow/providers/amazon/aws/secrets/secrets_manager.py @@ -253,10 +253,10 @@ def _get_secret(self, path_prefix, secret_id: str) -> Optional[str]: secret_id, ) return None - except self.client.exceptions.AccessDeniedException as e: + except self.client.exceptions.AccessDeniedException: self.log.debug( "An error occurred (AccessDeniedException) when calling the " - "get_secret_value operation: %s", - str(e), + "get_secret_value operation", + exc_info=True, ) return None From 8eefb0071e10ef6ab0c1734e7edbcd80d103a3f1 Mon Sep 17 00:00:00 2001 From: john-jac <75442233+john-jac@users.noreply.github.com> Date: Mon, 15 Nov 2021 13:05:36 -0800 Subject: [PATCH 4/4] Fixed static checks --- airflow/providers/amazon/aws/secrets/secrets_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/secrets/secrets_manager.py b/airflow/providers/amazon/aws/secrets/secrets_manager.py index 8a1cd94f6a5dc..cb8bd34639179 100644 --- a/airflow/providers/amazon/aws/secrets/secrets_manager.py +++ b/airflow/providers/amazon/aws/secrets/secrets_manager.py @@ -255,8 +255,7 @@ def _get_secret(self, path_prefix, secret_id: str) -> Optional[str]: return None except self.client.exceptions.AccessDeniedException: self.log.debug( - "An error occurred (AccessDeniedException) when calling the " - "get_secret_value operation", + "An error occurred (AccessDeniedException) when calling the get_secret_value operation", exc_info=True, ) return None