From 2d8886176634cc0bf5ba9b4430c46ec58245ffda Mon Sep 17 00:00:00 2001 From: 1fanwang <1fannnw@gmail.com> Date: Thu, 23 Jul 2026 14:08:02 -0700 Subject: [PATCH 1/2] Strip GlueDataQualityOperator ruleset after rendering ruleset is a template field, so it is rendered after __init__ runs. The constructor did self.ruleset = ruleset.strip(), which strips the un-rendered value. A templated ruleset that renders with surrounding whitespace is therefore never stripped, and validate_inputs() rejects it because the rendered value no longer ends with ']': AttributeError: RuleSet must starts with Rules = [ and ends with ] Store ruleset verbatim in the constructor and strip the rendered value at the start of execute(), before validation. related: #70296 Signed-off-by: 1fanwang <1fannnw@gmail.com> --- .../providers/amazon/aws/operators/glue.py | 5 +++- .../unit/amazon/aws/operators/test_glue.py | 26 +++++++++++++++++++ .../validate_operators_init_exemptions.txt | 1 - 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py b/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py index 3d46aecb32e0b..bf9615d1b038d 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py @@ -433,7 +433,7 @@ def __init__( ): super().__init__(**kwargs) self.name = name - self.ruleset = ruleset.strip() + self.ruleset = ruleset self.description = description self.update_rule_set = update_rule_set self.data_quality_ruleset_kwargs = data_quality_ruleset_kwargs or {} @@ -449,6 +449,9 @@ def validate_inputs(self) -> None: raise AttributeError("Target table must have DatabaseName and TableName") def execute(self, context: Context): + # ruleset is a template field; strip the rendered value here rather than in __init__, + # which only sees the un-rendered Jinja expression. + self.ruleset = self.ruleset.strip() self.validate_inputs() config = { diff --git a/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py b/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py index 230193feeb42b..fb694f3c005fc 100644 --- a/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py +++ b/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py @@ -17,6 +17,7 @@ from __future__ import annotations from collections.abc import Generator +from datetime import datetime from typing import TYPE_CHECKING from unittest import mock @@ -25,6 +26,7 @@ from boto3 import client from moto import mock_aws +from airflow.models.dag import DAG from airflow.providers.amazon.aws.hooks.glue import GlueDataQualityHook, GlueJobHook from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.amazon.aws.links.glue import GlueJobRunDetailsLink @@ -813,6 +815,30 @@ def test_execute_create_rule(self, glue_data_quality_mock_conn): Ruleset=self.RULE_SET, ) + @mock.patch.object(GlueDataQualityHook, "conn") + def test_execute_strips_rendered_ruleset(self, glue_data_quality_mock_conn): + # ruleset is a template field: __init__ sees the un-rendered "{{ ... }}" (nothing to strip), + # and rendering can yield surrounding whitespace. execute must strip the rendered value so + # validate_inputs passes and the API gets the clean ruleset. + with DAG("glue_dq_strip", schedule=None, start_date=datetime(2020, 1, 1)) as dag: + self.operator = GlueDataQualityOperator( + task_id="create_data_quality_ruleset", + name=self.RULE_SET_NAME, + ruleset="{{ params.rules }}", + dag=dag, + ) + self.operator.defer = mock.MagicMock() + self.operator.render_template_fields({"params": {"rules": f" {self.RULE_SET} "}}) + assert self.operator.ruleset == f" {self.RULE_SET} " + + self.operator.execute({}) + + glue_data_quality_mock_conn.create_data_quality_ruleset.assert_called_once_with( + Description="AWS Glue Data Quality Rule Set With Airflow", + Name=self.RULE_SET_NAME, + Ruleset=self.RULE_SET, + ) + @mock.patch.object(GlueDataQualityHook, "conn") def test_execute_create_rule_should_fail_if_rule_already_exists(self, glue_data_quality_mock_conn): self.operator = GlueDataQualityOperator( diff --git a/scripts/ci/prek/validate_operators_init_exemptions.txt b/scripts/ci/prek/validate_operators_init_exemptions.txt index 01fd5bc56dbb7..b6cbde980d002 100644 --- a/scripts/ci/prek/validate_operators_init_exemptions.txt +++ b/scripts/ci/prek/validate_operators_init_exemptions.txt @@ -14,7 +14,6 @@ providers/amazon/src/airflow/providers/amazon/aws/operators/dms.py::DmsModifyTas providers/amazon/src/airflow/providers/amazon/aws/operators/dms.py::DmsStartReplicationOperator providers/amazon/src/airflow/providers/amazon/aws/operators/ecs.py::EcsRunTaskOperator providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py::EmrAddStepsOperator -providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py::GlueDataQualityOperator providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStartDbClusterOperator providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStopDbClusterOperator providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py::S3DeleteObjectsOperator From 230a4fd238684e93d85c930e764a0786ea3fe94c Mon Sep 17 00:00:00 2001 From: 1fanwang <1fannnw@gmail.com> Date: Thu, 23 Jul 2026 23:08:38 -0700 Subject: [PATCH 2/2] Tighten GlueDataQuality ruleset-strip comments Signed-off-by: 1fanwang <1fannnw@gmail.com> --- .../amazon/src/airflow/providers/amazon/aws/operators/glue.py | 3 +-- providers/amazon/tests/unit/amazon/aws/operators/test_glue.py | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py b/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py index bf9615d1b038d..30077380eb641 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py @@ -449,8 +449,7 @@ def validate_inputs(self) -> None: raise AttributeError("Target table must have DatabaseName and TableName") def execute(self, context: Context): - # ruleset is a template field; strip the rendered value here rather than in __init__, - # which only sees the un-rendered Jinja expression. + # ruleset is a template field; strip the rendered value here, not in __init__. self.ruleset = self.ruleset.strip() self.validate_inputs() diff --git a/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py b/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py index fb694f3c005fc..99d22e3bb5e47 100644 --- a/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py +++ b/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py @@ -817,9 +817,7 @@ def test_execute_create_rule(self, glue_data_quality_mock_conn): @mock.patch.object(GlueDataQualityHook, "conn") def test_execute_strips_rendered_ruleset(self, glue_data_quality_mock_conn): - # ruleset is a template field: __init__ sees the un-rendered "{{ ... }}" (nothing to strip), - # and rendering can yield surrounding whitespace. execute must strip the rendered value so - # validate_inputs passes and the API gets the clean ruleset. + # ruleset is a template field; execute strips the rendered value (rendering can add whitespace). with DAG("glue_dq_strip", schedule=None, start_date=datetime(2020, 1, 1)) as dag: self.operator = GlueDataQualityOperator( task_id="create_data_quality_ruleset",