From 0d29e89407aa068af34612514749f68e1a9a0c6d Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Mon, 11 Sep 2023 01:12:56 +0530 Subject: [PATCH 1/2] Remove unnecessary call to keys() method on dictionaries --- airflow/providers/google/cloud/hooks/bigquery.py | 2 +- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py | 2 +- airflow/providers/google/cloud/utils/field_validator.py | 4 ++-- airflow/ti_deps/deps/trigger_rule_dep.py | 2 +- airflow/utils/process_utils.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airflow/providers/google/cloud/hooks/bigquery.py b/airflow/providers/google/cloud/hooks/bigquery.py index c98e7e92ab22e..8798abd5d85cd 100644 --- a/airflow/providers/google/cloud/hooks/bigquery.py +++ b/airflow/providers/google/cloud/hooks/bigquery.py @@ -659,7 +659,7 @@ def create_external_table( ], "googleSheetsOptions": ["skipLeadingRows"], } - if source_format in src_fmt_to_param_mapping.keys(): + if source_format in src_fmt_to_param_mapping: valid_configs = src_fmt_to_configs_mapping[src_fmt_to_param_mapping[source_format]] src_fmt_configs = _validate_src_fmt_configs( source_format, src_fmt_configs, valid_configs, backward_compatibility_configs diff --git a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py index 97b2943fb555a..f3ece4fb0794d 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py @@ -532,7 +532,7 @@ def _create_external_table(self): ], "googleSheetsOptions": ["skipLeadingRows"], } - if self.source_format in src_fmt_to_param_mapping.keys(): + if self.source_format in src_fmt_to_param_mapping: valid_configs = src_fmt_to_configs_mapping[src_fmt_to_param_mapping[self.source_format]] self.src_fmt_configs = self._validate_src_fmt_configs( self.source_format, self.src_fmt_configs, valid_configs, backward_compatibility_configs diff --git a/airflow/providers/google/cloud/utils/field_validator.py b/airflow/providers/google/cloud/utils/field_validator.py index 87aee5d7af027..afa88f1d0142e 100644 --- a/airflow/providers/google/cloud/utils/field_validator.py +++ b/airflow/providers/google/cloud/utils/field_validator.py @@ -258,7 +258,7 @@ def _validate_dict(self, children_validation_specs: dict, full_field_path: str, validation_spec=child_validation_spec, dictionary_to_validate=value, parent=full_field_path ) all_dict_keys = [spec["name"] for spec in children_validation_specs] - for field_name in value.keys(): + for field_name in value: if field_name not in all_dict_keys: self.log.warning( "The field '%s' is in the body, but is not specified in the " @@ -443,7 +443,7 @@ def validate(self, body_to_validate: dict) -> None: and nested_union_spec.get("api_version") != self._api_version ] ) - for field_name in body_to_validate.keys(): + for field_name in body_to_validate: if field_name not in all_field_names: self.log.warning( "The field '%s' is in the body, but is not specified in the " diff --git a/airflow/ti_deps/deps/trigger_rule_dep.py b/airflow/ti_deps/deps/trigger_rule_dep.py index 7bb4bf52137c8..36a4258c589c8 100644 --- a/airflow/ti_deps/deps/trigger_rule_dep.py +++ b/airflow/ti_deps/deps/trigger_rule_dep.py @@ -194,7 +194,7 @@ def _iter_upstream_conditions(relevant_tasks: dict) -> Iterator[ColumnOperators] return # Otherwise we need to figure out which map indexes are depended on # for each upstream by the current task instance. - for upstream_id in relevant_tasks.keys(): + for upstream_id in relevant_tasks: map_indexes = _get_relevant_upstream_map_indexes(upstream_id) if map_indexes is None: # All tis of this upstream are dependencies. yield (TaskInstance.task_id == upstream_id) diff --git a/airflow/utils/process_utils.py b/airflow/utils/process_utils.py index 1f7c4771e8c02..6e9bb47d21d5f 100644 --- a/airflow/utils/process_utils.py +++ b/airflow/utils/process_utils.py @@ -284,7 +284,7 @@ def patch_environ(new_env_variables: dict[str, str]) -> Generator[None, None, No After leaving the context, it restores its original state. :param new_env_variables: Environment variables to set """ - current_env_state = {key: os.environ.get(key) for key in new_env_variables.keys()} + current_env_state = {key: os.environ.get(key) for key in new_env_variables} os.environ.update(new_env_variables) try: yield From 5fc664b721da694c12db0acacfc9e11cd94ac190 Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Wed, 13 Sep 2023 14:57:07 +0530 Subject: [PATCH 2/2] Apply @potiuk's suggestion for raising RuntimeError when body is None --- airflow/providers/google/cloud/utils/field_validator.py | 2 ++ tests/providers/google/cloud/utils/test_field_validator.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/airflow/providers/google/cloud/utils/field_validator.py b/airflow/providers/google/cloud/utils/field_validator.py index ebad3333248e0..1a3ccbff40cb5 100644 --- a/airflow/providers/google/cloud/utils/field_validator.py +++ b/airflow/providers/google/cloud/utils/field_validator.py @@ -421,6 +421,8 @@ def validate(self, body_to_validate: dict) -> None: :param body_to_validate: body that must follow the specification :return: None """ + if body_to_validate is None: + raise RuntimeError("The body to validate is `None`. Please provide a dictionary to validate.") try: for validation_spec in self._validation_specs: self._validate_field(validation_spec=validation_spec, dictionary_to_validate=body_to_validate) diff --git a/tests/providers/google/cloud/utils/test_field_validator.py b/tests/providers/google/cloud/utils/test_field_validator.py index 920636e72447d..e504de795e172 100644 --- a/tests/providers/google/cloud/utils/test_field_validator.py +++ b/tests/providers/google/cloud/utils/test_field_validator.py @@ -40,7 +40,9 @@ def test_validate_should_fail_if_body_is_none(self): validator = GcpBodyFieldValidator(specification, "v1") - with pytest.raises(AttributeError): + with pytest.raises( + RuntimeError, match="The body to validate is `None`. Please provide a dictionary to validate." + ): validator.validate(body) def test_validate_should_fail_if_specification_is_none(self):