Keep not-ready providers out of the all extra

pyproject.toml

@@ -191,9 +191,6 @@ apache-airflow = "airflow.__main__:main"
 "common.compat" = [
     "apache-airflow-providers-common-compat>=1.2.1"
 ]
-"common.dataquality" = [
-    "apache-airflow-providers-common-dataquality>=0.1.0"
-]
 "common.io" = [
     "apache-airflow-providers-common-io>=1.4.2"
 ]
@@ -257,9 +254,6 @@ apache-airflow = "airflow.__main__:main"
 "http" = [
     "apache-airflow-providers-http>=4.13.2"
 ]
-"ibm.mq" = [
-    "apache-airflow-providers-ibm-mq>=0.1.0; platform_machine !=\"aarch64\" and platform_machine !=\"arm64\""
-]
 "imap" = [
     "apache-airflow-providers-imap>=3.8.0"
 ]
@@ -444,7 +438,6 @@ apache-airflow = "airflow.__main__:main"
     "apache-airflow-providers-cohere>=1.4.0",
     "apache-airflow-providers-common-ai>=0.1.0",
     "apache-airflow-providers-common-compat>=1.2.1",
-    "apache-airflow-providers-common-dataquality>=0.1.0",
     "apache-airflow-providers-common-io>=1.4.2",
     "apache-airflow-providers-common-messaging>=2.0.0", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
     "apache-airflow-providers-common-sql>=1.18.0",
@@ -466,7 +459,6 @@ apache-airflow = "airflow.__main__:main"
     "apache-airflow-providers-grpc>=3.7.0",
     "apache-airflow-providers-hashicorp>=4.0.0",
     "apache-airflow-providers-http>=4.13.2",
-    "apache-airflow-providers-ibm-mq>=0.1.0; platform_machine !=\"aarch64\" and platform_machine !=\"arm64\"",
     "apache-airflow-providers-imap>=3.8.0",
     "apache-airflow-providers-influxdb>=2.8.0",
     "apache-airflow-providers-informatica>=0.1.1",

scripts/ci/prek/common_prek_utils.py

@@ -591,19 +591,30 @@ def get_provider_base_dir_from_path(file_path: Path) -> Path | None:
     return None
 
 
-def get_all_provider_ids(exclude_suspended_providers: bool = False) -> list[str]:
+def get_all_provider_ids(
+    exclude_suspended_providers: bool = False, exclude_not_ready_providers: bool = False
+) -> list[str]:
     """
     Get all providers from the new provider structure
+
+    :param exclude_suspended_providers: skip providers whose state is ``suspended``
+    :param exclude_not_ready_providers: skip providers whose state is ``not-ready`` - those have
+        never been published, so anything describing what is installable must leave them out
     """
     all_provider_ids = []
+    excluded_states = set()
+    if exclude_suspended_providers:
+        excluded_states.add("suspended")
+    if exclude_not_ready_providers:
+        excluded_states.add("not-ready")
     for provider_file in AIRFLOW_PROVIDERS_ROOT_PATH.rglob("provider.yaml"):
         if provider_file.is_relative_to(AIRFLOW_PROVIDERS_ROOT_PATH / "src"):
             continue
-        if exclude_suspended_providers:
+        if excluded_states:
             import yaml
 
             provider_info = yaml.safe_load(provider_file.read_text())
-            if provider_info.get("state") == "suspended":
+            if provider_info.get("state") in excluded_states:
                 continue
         provider_id = get_provider_id_from_path(provider_file)
         if provider_id:

scripts/ci/prek/update_airflow_pyproject_toml.py

@@ -269,9 +269,20 @@ def get_exclusion_marker(provider_dependencies: dict[str, Any]) -> str:
             all_optional_dependencies.append(f'"{optional}" = [\n    "apache-airflow-core[{optional}]"\n]\n')
     optional_airflow_task_sdk_dependencies = get_optional_dependencies(AIRFLOW_TASK_SDK_PYPROJECT_TOML_FILE)
     all_optional_dependencies.append('"all-task-sdk" = [\n    "apache-airflow-task-sdk[all]"\n]\n')
+    # Two lists, because the sections below describe two different things.
+    #
+    # `all_providers` describes the source tree: mypy has to type-check a not-ready provider and uv
+    # has to keep it in the workspace, which is how CI installs it. Only suspended providers drop out.
+    #
+    # `released_providers` describes what is installable from PyPI. A not-ready provider has never
+    # been published, so naming it in an extra makes that extra unsatisfiable - there is no version
+    # of it to resolve to.
     all_providers = sorted(get_all_provider_ids(exclude_suspended_providers=True))
+    released_providers = sorted(
+        get_all_provider_ids(exclude_suspended_providers=True, exclude_not_ready_providers=True)
+    )
     all_provider_lines = []
-    for provider_id in all_providers:
+    for provider_id in released_providers:
         distribution_name = provider_distribution_name(provider_id)
         min_provider_version, comment = find_min_provider_version(provider_id)
         exclusion_marker = get_exclusion_marker(all_providers_dependencies.get(provider_id, {}))
@@ -288,10 +299,15 @@ def get_exclusion_marker(provider_dependencies: dict[str, Any]) -> str:
             all_provider_lines.append(f'    "{distribution_name}",\n')
     all_optional_dependencies.append('"all" = [\n')
     optional_apache_airflow_dependencies = get_optional_dependencies(AIRFLOW_PYPROJECT_TOML_FILE)
+    # Filtered against every provider id rather than against `all_providers`: a provider left out of
+    # `all_providers` still has an extra named after it, and testing only against the included ones
+    # would sweep that extra in here instead of dropping it - which is how a not-ready provider would
+    # come back into `all` through the side door.
+    every_provider_id = set(get_all_provider_ids())
     all_local_extras = [
         extra
         for extra in sorted(optional_apache_airflow_dependencies)
-        if extra not in all_providers and not extra.startswith("all")
+        if extra not in every_provider_id and not extra.startswith("all")
     ]
     all_optional_dependencies.append(f'    "apache-airflow[{",".join(all_local_extras)}]",\n')
     all_optional_dependencies.append('    "apache-airflow-core[all]",\n')