From 429c669e861d8b665f3c0587f34c93226d86ae20 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 5 Aug 2026 17:07:01 +0800 Subject: [PATCH] Keep not-ready providers out of the all extra A not-ready provider has never been published, so naming it in an extra makes that extra impossible to satisfy - there is no version on PyPI to resolve to. It broke constraints generation for 3.3.1rc1, where apache-airflow[all] could not resolve because it required apache-airflow-providers-ibm-mq. Only the extras change. The source tree still holds those providers, so they stay in the workspace and under mypy - which is how CI installs and checks them. --- pyproject.toml | 8 -------- scripts/ci/prek/common_prek_utils.py | 17 +++++++++++++--- .../ci/prek/update_airflow_pyproject_toml.py | 20 +++++++++++++++++-- uv.lock | 14 +------------ 4 files changed, 33 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index be805a23ca0ce..3a1b9d360f3d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -191,9 +191,6 @@ apache-airflow = "airflow.__main__:main" "common.compat" = [ "apache-airflow-providers-common-compat>=1.2.1" ] -"common.dataquality" = [ - "apache-airflow-providers-common-dataquality>=0.1.0" -] "common.io" = [ "apache-airflow-providers-common-io>=1.4.2" ] @@ -257,9 +254,6 @@ apache-airflow = "airflow.__main__:main" "http" = [ "apache-airflow-providers-http>=4.13.2" ] -"ibm.mq" = [ - "apache-airflow-providers-ibm-mq>=0.1.0; platform_machine !=\"aarch64\" and platform_machine !=\"arm64\"" -] "imap" = [ "apache-airflow-providers-imap>=3.8.0" ] @@ -444,7 +438,6 @@ apache-airflow = "airflow.__main__:main" "apache-airflow-providers-cohere>=1.4.0", "apache-airflow-providers-common-ai>=0.1.0", "apache-airflow-providers-common-compat>=1.2.1", - "apache-airflow-providers-common-dataquality>=0.1.0", "apache-airflow-providers-common-io>=1.4.2", "apache-airflow-providers-common-messaging>=2.0.0", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py "apache-airflow-providers-common-sql>=1.18.0", @@ -466,7 +459,6 @@ apache-airflow = "airflow.__main__:main" "apache-airflow-providers-grpc>=3.7.0", "apache-airflow-providers-hashicorp>=4.0.0", "apache-airflow-providers-http>=4.13.2", - "apache-airflow-providers-ibm-mq>=0.1.0; platform_machine !=\"aarch64\" and platform_machine !=\"arm64\"", "apache-airflow-providers-imap>=3.8.0", "apache-airflow-providers-influxdb>=2.8.0", "apache-airflow-providers-informatica>=0.1.1", diff --git a/scripts/ci/prek/common_prek_utils.py b/scripts/ci/prek/common_prek_utils.py index e316eb154aea6..559c2f0a047ae 100644 --- a/scripts/ci/prek/common_prek_utils.py +++ b/scripts/ci/prek/common_prek_utils.py @@ -591,19 +591,30 @@ def get_provider_base_dir_from_path(file_path: Path) -> Path | None: return None -def get_all_provider_ids(exclude_suspended_providers: bool = False) -> list[str]: +def get_all_provider_ids( + exclude_suspended_providers: bool = False, exclude_not_ready_providers: bool = False +) -> list[str]: """ Get all providers from the new provider structure + + :param exclude_suspended_providers: skip providers whose state is ``suspended`` + :param exclude_not_ready_providers: skip providers whose state is ``not-ready`` - those have + never been published, so anything describing what is installable must leave them out """ all_provider_ids = [] + excluded_states = set() + if exclude_suspended_providers: + excluded_states.add("suspended") + if exclude_not_ready_providers: + excluded_states.add("not-ready") for provider_file in AIRFLOW_PROVIDERS_ROOT_PATH.rglob("provider.yaml"): if provider_file.is_relative_to(AIRFLOW_PROVIDERS_ROOT_PATH / "src"): continue - if exclude_suspended_providers: + if excluded_states: import yaml provider_info = yaml.safe_load(provider_file.read_text()) - if provider_info.get("state") == "suspended": + if provider_info.get("state") in excluded_states: continue provider_id = get_provider_id_from_path(provider_file) if provider_id: diff --git a/scripts/ci/prek/update_airflow_pyproject_toml.py b/scripts/ci/prek/update_airflow_pyproject_toml.py index 2313790e0a5d5..f66a3d7ba5d49 100755 --- a/scripts/ci/prek/update_airflow_pyproject_toml.py +++ b/scripts/ci/prek/update_airflow_pyproject_toml.py @@ -269,9 +269,20 @@ def get_exclusion_marker(provider_dependencies: dict[str, Any]) -> str: all_optional_dependencies.append(f'"{optional}" = [\n "apache-airflow-core[{optional}]"\n]\n') optional_airflow_task_sdk_dependencies = get_optional_dependencies(AIRFLOW_TASK_SDK_PYPROJECT_TOML_FILE) all_optional_dependencies.append('"all-task-sdk" = [\n "apache-airflow-task-sdk[all]"\n]\n') + # Two lists, because the sections below describe two different things. + # + # `all_providers` describes the source tree: mypy has to type-check a not-ready provider and uv + # has to keep it in the workspace, which is how CI installs it. Only suspended providers drop out. + # + # `released_providers` describes what is installable from PyPI. A not-ready provider has never + # been published, so naming it in an extra makes that extra unsatisfiable - there is no version + # of it to resolve to. all_providers = sorted(get_all_provider_ids(exclude_suspended_providers=True)) + released_providers = sorted( + get_all_provider_ids(exclude_suspended_providers=True, exclude_not_ready_providers=True) + ) all_provider_lines = [] - for provider_id in all_providers: + for provider_id in released_providers: distribution_name = provider_distribution_name(provider_id) min_provider_version, comment = find_min_provider_version(provider_id) exclusion_marker = get_exclusion_marker(all_providers_dependencies.get(provider_id, {})) @@ -288,10 +299,15 @@ def get_exclusion_marker(provider_dependencies: dict[str, Any]) -> str: all_provider_lines.append(f' "{distribution_name}",\n') all_optional_dependencies.append('"all" = [\n') optional_apache_airflow_dependencies = get_optional_dependencies(AIRFLOW_PYPROJECT_TOML_FILE) + # Filtered against every provider id rather than against `all_providers`: a provider left out of + # `all_providers` still has an extra named after it, and testing only against the included ones + # would sweep that extra in here instead of dropping it - which is how a not-ready provider would + # come back into `all` through the side door. + every_provider_id = set(get_all_provider_ids()) all_local_extras = [ extra for extra in sorted(optional_apache_airflow_dependencies) - if extra not in all_providers and not extra.startswith("all") + if extra not in every_provider_id and not extra.startswith("all") ] all_optional_dependencies.append(f' "apache-airflow[{",".join(all_local_extras)}]",\n') all_optional_dependencies.append(' "apache-airflow-core[all]",\n') diff --git a/uv.lock b/uv.lock index f039ed5c273fd..867a95e35bc44 100644 --- a/uv.lock +++ b/uv.lock @@ -1032,7 +1032,6 @@ all = [ { name = "apache-airflow-providers-cohere" }, { name = "apache-airflow-providers-common-ai" }, { name = "apache-airflow-providers-common-compat" }, - { name = "apache-airflow-providers-common-dataquality" }, { name = "apache-airflow-providers-common-io" }, { name = "apache-airflow-providers-common-messaging" }, { name = "apache-airflow-providers-common-sql", extra = ["pandas", "polars"] }, @@ -1054,7 +1053,6 @@ all = [ { name = "apache-airflow-providers-grpc" }, { name = "apache-airflow-providers-hashicorp" }, { name = "apache-airflow-providers-http" }, - { name = "apache-airflow-providers-ibm-mq", marker = "platform_machine != 'aarch64' and platform_machine != 'arm64'" }, { name = "apache-airflow-providers-imap" }, { name = "apache-airflow-providers-influxdb" }, { name = "apache-airflow-providers-informatica" }, @@ -1217,9 +1215,6 @@ common-ai = [ common-compat = [ { name = "apache-airflow-providers-common-compat" }, ] -common-dataquality = [ - { name = "apache-airflow-providers-common-dataquality" }, -] common-io = [ { name = "apache-airflow-providers-common-io" }, ] @@ -1297,9 +1292,6 @@ hashicorp = [ http = [ { name = "apache-airflow-providers-http" }, ] -ibm-mq = [ - { name = "apache-airflow-providers-ibm-mq", marker = "platform_machine != 'aarch64' and platform_machine != 'arm64'" }, -] imap = [ { name = "apache-airflow-providers-imap" }, ] @@ -1632,8 +1624,6 @@ requires-dist = [ { name = "apache-airflow-providers-common-ai", marker = "extra == 'common-ai'", editable = "providers/common/ai" }, { name = "apache-airflow-providers-common-compat", marker = "extra == 'all'", editable = "providers/common/compat" }, { name = "apache-airflow-providers-common-compat", marker = "extra == 'common-compat'", editable = "providers/common/compat" }, - { name = "apache-airflow-providers-common-dataquality", marker = "extra == 'all'", editable = "providers/common/dataquality" }, - { name = "apache-airflow-providers-common-dataquality", marker = "extra == 'common-dataquality'", editable = "providers/common/dataquality" }, { name = "apache-airflow-providers-common-io", marker = "extra == 'all'", editable = "providers/common/io" }, { name = "apache-airflow-providers-common-io", marker = "extra == 'common-io'", editable = "providers/common/io" }, { name = "apache-airflow-providers-common-messaging", marker = "extra == 'all'", editable = "providers/common/messaging" }, @@ -1680,8 +1670,6 @@ requires-dist = [ { name = "apache-airflow-providers-hashicorp", marker = "extra == 'hashicorp'", editable = "providers/hashicorp" }, { name = "apache-airflow-providers-http", marker = "extra == 'all'", editable = "providers/http" }, { name = "apache-airflow-providers-http", marker = "extra == 'http'", editable = "providers/http" }, - { name = "apache-airflow-providers-ibm-mq", marker = "platform_machine != 'aarch64' and platform_machine != 'arm64' and extra == 'all'", editable = "providers/ibm/mq" }, - { name = "apache-airflow-providers-ibm-mq", marker = "platform_machine != 'aarch64' and platform_machine != 'arm64' and extra == 'ibm-mq'", editable = "providers/ibm/mq" }, { name = "apache-airflow-providers-imap", marker = "extra == 'all'", editable = "providers/imap" }, { name = "apache-airflow-providers-imap", marker = "extra == 'imap'", editable = "providers/imap" }, { name = "apache-airflow-providers-influxdb", marker = "extra == 'all'", editable = "providers/influxdb" }, @@ -1792,7 +1780,7 @@ requires-dist = [ { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.30.0" }, { name = "uv", marker = "extra == 'uv'", specifier = ">=0.11.29" }, ] -provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-dataquality", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] +provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "anthropic", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] [package.metadata.requires-dev] ci-image = [