From 9150121ad4c7985781afa8170a0273c489f0e53f Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 27 Feb 2026 14:50:12 +0100 Subject: [PATCH 1/2] Fix all build-system/requires including transitive dependencies Add a feature to fix all build-system/requires to make build reproducibility works. The build-system requires are managed automatically by the `upgrade-important-versions` prek hook. This should not only provide reproducibility, but also it should prevent cases where transient dependency upgrade might break building sdist (for example as it happened with virtualenv 21 breaking hatchling with https://github.com/pypa/hatch/issues/2193 --- .github/workflows/basic-tests.yml | 12 +- airflow-core/pyproject.toml | 17 +- airflow-ctl-tests/pyproject.toml | 14 +- airflow-ctl/pyproject.toml | 14 +- airflow-e2e-tests/pyproject.toml | 14 +- chart/pyproject.toml | 12 +- clients/python/pyproject.toml | 14 +- dev/breeze/pyproject.toml | 2 +- dev/pyproject.toml | 14 +- docker-stack-docs/pyproject.toml | 12 +- docker-tests/pyproject.toml | 14 +- helm-tests/pyproject.toml | 14 +- kubernetes-tests/pyproject.toml | 14 +- providers-summary-docs/pyproject.toml | 12 +- providers/edge3/pyproject.toml | 18 +- providers/fab/pyproject.toml | 18 +- pyproject.toml | 18 +- scripts/ci/prek/check_extra_packages_ref.py | 2 +- .../check_shared_distributions_structure.py | 2 +- scripts/ci/prek/upgrade_important_versions.py | 189 ++++++++++++++++++ shared/configuration/pyproject.toml | 14 +- shared/dagnode/pyproject.toml | 14 +- shared/listeners/pyproject.toml | 14 +- shared/logging/pyproject.toml | 14 +- shared/module_loading/pyproject.toml | 14 +- shared/observability/pyproject.toml | 14 +- shared/plugins_manager/pyproject.toml | 14 +- shared/providers_discovery/pyproject.toml | 14 +- shared/secrets_backend/pyproject.toml | 14 +- shared/secrets_masker/pyproject.toml | 14 +- shared/timezones/pyproject.toml | 14 +- task-sdk-integration-tests/pyproject.toml | 14 +- task-sdk/pyproject.toml | 14 +- 33 files changed, 546 insertions(+), 62 deletions(-) diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 6a2204d47a58c..5e68cedf47bb6 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -329,16 +329,18 @@ jobs: --hook-stage manual upgrade-important-versions || true if: always() env: - UPGRADE_PIP: "false" - UPGRADE_PYTHON: "false" + UPGRADE_FLIT: "false" + UPGRADE_GITPYTHON: "false" UPGRADE_GOLANG: "false" - UPGRADE_NODE_LTS: "false" UPGRADE_HATCH: "false" + UPGRADE_HATCHLING: "false" + UPGRADE_MYPY: "false" + UPGRADE_NODE_LTS: "false" + UPGRADE_PIP: "false" + UPGRADE_PYTHON: "false" UPGRADE_PYYAML: "false" - UPGRADE_GITPYTHON: "false" UPGRADE_RICH: "false" UPGRADE_RUFF: "false" - UPGRADE_MYPY: "false" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: "Run automated upgrade for important versions minus uv (failing if needed)" run: | diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml index aaaf859cd0c91..a3ed682945d23 100644 --- a/airflow-core/pyproject.toml +++ b/airflow-core/pyproject.toml @@ -17,15 +17,20 @@ [build-system] requires = [ - "GitPython==3.1.45", + "distlib==0.4.0", + "filelock==3.24.3", "gitdb==4.0.12", - "hatchling==1.27.0", - "packaging==25.0", - "pathspec==0.12.1", + "GitPython==3.1.46", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", "pluggy==1.6.0", "smmap==5.0.2", - "tomli==2.2.1; python_version < '3.11'", - "trove-classifiers==2025.9.11.17", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", ] build-backend = "hatchling.build" diff --git a/airflow-ctl-tests/pyproject.toml b/airflow-ctl-tests/pyproject.toml index 8adbe1f628535..9d5a270e3badb 100644 --- a/airflow-ctl-tests/pyproject.toml +++ b/airflow-ctl-tests/pyproject.toml @@ -17,7 +17,19 @@ # under the License. [build-system] -requires = [ "hatchling==1.27.0" ] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [project] diff --git a/airflow-ctl/pyproject.toml b/airflow-ctl/pyproject.toml index 67741244e0bb7..519af7895e361 100644 --- a/airflow-ctl/pyproject.toml +++ b/airflow-ctl/pyproject.toml @@ -69,7 +69,19 @@ YouTube = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" airflowctl = "airflowctl.__main__:main" [build-system] -requires = ["hatchling"] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [tool.hatch.version] diff --git a/airflow-e2e-tests/pyproject.toml b/airflow-e2e-tests/pyproject.toml index 992b69d09f7f8..37681ed5e41da 100644 --- a/airflow-e2e-tests/pyproject.toml +++ b/airflow-e2e-tests/pyproject.toml @@ -16,7 +16,19 @@ # under the License. [build-system] -requires = [ "hatchling==1.27.0" ] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [project] diff --git a/chart/pyproject.toml b/chart/pyproject.toml index a3287d70bb518..fae7150529b0b 100644 --- a/chart/pyproject.toml +++ b/chart/pyproject.toml @@ -17,7 +17,17 @@ [build-system] requires = [ - "hatchling==1.27.0", + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", ] build-backend = "hatchling.build" diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index b8b4802cc8632..08686c8f989dd 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -16,7 +16,19 @@ # under the License. [build-system] -requires = ["hatchling==1.27.0"] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [project] diff --git a/dev/breeze/pyproject.toml b/dev/breeze/pyproject.toml index e6b68817c3684..569ddeadbbba0 100644 --- a/dev/breeze/pyproject.toml +++ b/dev/breeze/pyproject.toml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. [build-system] -requires = ["flit_core >=3.11,<4"] +requires = ["flit_core >=3.12.0,<4"] build-backend = "flit_core.buildapi" [project] diff --git a/dev/pyproject.toml b/dev/pyproject.toml index 8d732f5f1d67b..adf95b66bfcfd 100644 --- a/dev/pyproject.toml +++ b/dev/pyproject.toml @@ -17,7 +17,19 @@ # under the License. [build-system] -requires = [ "hatchling==1.27.0" ] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [project] diff --git a/docker-stack-docs/pyproject.toml b/docker-stack-docs/pyproject.toml index c1a5ce513ab80..d35e39aabcdcc 100644 --- a/docker-stack-docs/pyproject.toml +++ b/docker-stack-docs/pyproject.toml @@ -17,7 +17,17 @@ [build-system] requires = [ - "hatchling==1.27.0", + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", ] build-backend = "hatchling.build" diff --git a/docker-tests/pyproject.toml b/docker-tests/pyproject.toml index ff646d8da67af..1e89ca81daced 100644 --- a/docker-tests/pyproject.toml +++ b/docker-tests/pyproject.toml @@ -17,7 +17,19 @@ # under the License. [build-system] -requires = [ "hatchling==1.27.0" ] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [project] diff --git a/helm-tests/pyproject.toml b/helm-tests/pyproject.toml index 1cac48c8b6861..0623d1c527ba2 100644 --- a/helm-tests/pyproject.toml +++ b/helm-tests/pyproject.toml @@ -17,7 +17,19 @@ # under the License. [build-system] -requires = [ "hatchling==1.27.0" ] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [project] diff --git a/kubernetes-tests/pyproject.toml b/kubernetes-tests/pyproject.toml index dd482e0680d8b..4f15d82050687 100644 --- a/kubernetes-tests/pyproject.toml +++ b/kubernetes-tests/pyproject.toml @@ -15,7 +15,19 @@ # specific language governing permissions and limitations # under the License. [build-system] -requires = [ "hatchling==1.27.0" ] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [project] diff --git a/providers-summary-docs/pyproject.toml b/providers-summary-docs/pyproject.toml index f507506e58531..90463476a0588 100644 --- a/providers-summary-docs/pyproject.toml +++ b/providers-summary-docs/pyproject.toml @@ -17,7 +17,17 @@ [build-system] requires = [ - "hatchling==1.27.0", + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", ] build-backend = "hatchling.build" diff --git a/providers/edge3/pyproject.toml b/providers/edge3/pyproject.toml index 0098e0f6c5936..71c2964076dd4 100644 --- a/providers/edge3/pyproject.toml +++ b/providers/edge3/pyproject.toml @@ -21,15 +21,17 @@ # `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY [build-system] requires = [ - "GitPython==3.1.45", - "gitdb==4.0.12", - "hatchling==1.27.0", - "packaging==25.0", - "pathspec==0.12.1", + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", "pluggy==1.6.0", - "smmap==5.0.2", - "tomli==2.2.1; python_version < '3.11'", - "trove-classifiers==2025.9.11.17", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", ] build-backend = "hatchling.build" diff --git a/providers/fab/pyproject.toml b/providers/fab/pyproject.toml index bbe80028ecf93..d3b4f3cb45809 100644 --- a/providers/fab/pyproject.toml +++ b/providers/fab/pyproject.toml @@ -21,15 +21,17 @@ # `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY [build-system] requires = [ - "GitPython==3.1.45", - "gitdb==4.0.12", - "hatchling==1.27.0", - "packaging==25.0", - "pathspec==0.12.1", + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", "pluggy==1.6.0", - "smmap==5.0.2", - "tomli==2.2.1; python_version < '3.11'", - "trove-classifiers==2025.9.11.17", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", ] build-backend = "hatchling.build" diff --git a/pyproject.toml b/pyproject.toml index df157b2e2d43e..337f7e0e5b625 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,15 +17,17 @@ [build-system] requires = [ - "GitPython==3.1.45", - "gitdb==4.0.12", - "hatchling==1.27.0", - "packaging==25.0", - "pathspec==0.12.1", + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", "pluggy==1.6.0", - "smmap==5.0.2", - "tomli==2.2.1; python_version < '3.11'", - "trove-classifiers==2025.9.11.17", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", ] build-backend = "hatchling.build" diff --git a/scripts/ci/prek/check_extra_packages_ref.py b/scripts/ci/prek/check_extra_packages_ref.py index f380f49531003..004e3b4699a48 100755 --- a/scripts/ci/prek/check_extra_packages_ref.py +++ b/scripts/ci/prek/check_extra_packages_ref.py @@ -19,7 +19,7 @@ # /// script # requires-python = ">=3.10,<3.11" # dependencies = [ -# "hatchling==1.27.0", +# "hatchling==1.29.0", # "rich>=13.6.0", # "tabulate>=0.9.0", # ] diff --git a/scripts/ci/prek/check_shared_distributions_structure.py b/scripts/ci/prek/check_shared_distributions_structure.py index 8063d9c6ace5b..41a9fb8112ecd 100755 --- a/scripts/ci/prek/check_shared_distributions_structure.py +++ b/scripts/ci/prek/check_shared_distributions_structure.py @@ -126,7 +126,7 @@ def check_private_classifier(pyproject: dict) -> bool: def check_build_system(pyproject: dict, shared_path: Path) -> bool: build_system = pyproject.get("build-system", {}) - if build_system == {"requires": ["hatchling"], "build-backend": "hatchling.build"}: + if build_system["build-backend"] == "hatchling.build": console.print( f" build-system is correct for [magenta]{shared_path.name}[/magenta] [bold green]OK[/bold green]" ) diff --git a/scripts/ci/prek/upgrade_important_versions.py b/scripts/ci/prek/upgrade_important_versions.py index 94ae953c8e9d6..12d81ef27e701 100755 --- a/scripts/ci/prek/upgrade_important_versions.py +++ b/scripts/ci/prek/upgrade_important_versions.py @@ -423,9 +423,11 @@ def get_env_bool(name: str, default: bool = True) -> bool: console.print("[bright_blue]Upgrading all important versions") # Package upgrade flags +UPGRADE_FLIT_CORE: bool = get_env_bool("UPGRADE_FLIT_CORE") UPGRADE_GITPYTHON: bool = get_env_bool("UPGRADE_GITPYTHON") UPGRADE_GOLANG: bool = get_env_bool("UPGRADE_GOLANG") UPGRADE_HATCH: bool = get_env_bool("UPGRADE_HATCH") +UPGRADE_HATCHLING: bool = get_env_bool("UPGRADE_HATCHLING") UPGRADE_MPROCS: bool = get_env_bool("UPGRADE_MPROCS") UPGRADE_NODE_LTS: bool = get_env_bool("UPGRADE_NODE_LTS") UPGRADE_PIP: bool = get_env_bool("UPGRADE_PIP") @@ -497,6 +499,20 @@ def apply_pattern_replacements( # Configuration for packages that follow simple version constant patterns SIMPLE_VERSION_PATTERNS: dict[str, list[tuple[str, str]]] = { + "flit_core": [ + (r"(flit_core==)([0-9.abrc]+)", "flit_core=={version}"), + (r"(flit_core >=)([0-9.abrc]+)", "flit_core >={version}"), + (r"(flit-core>=)([0-9.abrc]+)", "flit-core>={version}"), + (r"(flit-core==)([0-9.abrc]+)", "flit-core=={version}"), + (r"(flit>=)([0-9.abrc]+)", "flit>={version}"), + (r"(flit==)([0-9.abrc]+)", "flit=={version}"), + ], + "hatchling": [ + (r"(hatchling==)([0-9.abrc]+)", "hatchling=={version}"), + (r"(hatchling>=)([0-9.abrc]+)", "hatchling>={version}"), + (r"(HATCHLING_VERSION = )(\"[0-9.abrc]+\")", 'HATCHLING_VERSION = "{version}"'), + (r"(HATCHLING_VERSION=)(\"[0-9.abrc]+\")", 'HATCHLING_VERSION="{version}"'), + ], "hatch": [ (r"(HATCH_VERSION = )(\"[0-9.abrc]+\")", 'HATCH_VERSION = "{version}"'), (r"(HATCH_VERSION=)(\"[0-9.abrc]+\")", 'HATCH_VERSION="{version}"'), @@ -553,7 +569,9 @@ def fetch_all_package_versions() -> dict[str, str]: "pip": get_latest_pypi_version("pip", UPGRADE_PIP), "uv": get_latest_pypi_version("uv", UPGRADE_UV), "prek": get_latest_pypi_version("prek", UPGRADE_PREK), + "flit_core": get_latest_pypi_version("flit_core", UPGRADE_FLIT_CORE), "hatch": get_latest_pypi_version("hatch", UPGRADE_HATCH), + "hatchling": get_latest_pypi_version("hatchling", UPGRADE_HATCHLING), "pyyaml": get_latest_pypi_version("PyYAML", UPGRADE_PYYAML), "gitpython": get_latest_pypi_version("GitPython", UPGRADE_GITPYTHON), "ruff": get_latest_pypi_version("ruff", UPGRADE_RUFF), @@ -671,6 +689,164 @@ def sync_breeze_lock_file() -> None: ) +def resolve_hatchling_build_requires(with_gitpython: bool = False) -> list[str]: + """ + Resolve the full transitive dependency list for hatchling (with virtualenv<21) using uv pip compile. + + When with_gitpython is True, also includes GitPython and its transitive dependencies (gitdb, smmap). + Returns a sorted list of pinned requirement strings, with tomli carrying its python_version marker. + """ + packages = ["hatchling", "virtualenv<21"] + if with_gitpython: + packages.append("gitpython") + + result = subprocess.run( + ["uv", "pip", "compile", "-", "--resolution", "highest", "--python-version", "3.10"], + input="\n".join(packages) + "\n", + capture_output=True, + text=True, + check=True, + ) + + # Parse output: lines like "package==version" (skip comment lines and blank lines) + requires: list[str] = [] + for _line in result.stdout.splitlines(): + line = _line.strip() + if not line or line.startswith("#"): + continue + # Take only the "name==version" part (strip trailing comments) + pkg_spec = line.split()[0] + # Normalise package name to canonical casing + pkg_name_lower = pkg_spec.split("==")[0].lower().replace("-", "_") + pkg_version = pkg_spec.split("==")[1] if "==" in pkg_spec else "" + if not pkg_version: + continue + # Use canonical casing for known packages + CANONICAL_NAMES = { + "gitpython": "GitPython", + "gitdb": "gitdb", + "smmap": "smmap", + "hatchling": "hatchling", + "packaging": "packaging", + "pathspec": "pathspec", + "pluggy": "pluggy", + "trove_classifiers": "trove-classifiers", + "tomli": "tomli", + "virtualenv": "virtualenv", + "distlib": "distlib", + "filelock": "filelock", + "platformdirs": "platformdirs", + "typing_extensions": "typing-extensions", + } + canonical = CANONICAL_NAMES.get(pkg_name_lower, pkg_spec.split("==")[0]) + if pkg_name_lower == "tomli": + requires.append(f"{canonical}=={pkg_version}; python_version < '3.11'") + elif pkg_name_lower == "typing_extensions": + # typing_extensions is built-in from Python 3.11+ + requires.append(f"{canonical}=={pkg_version}; python_version < '3.11'") + else: + requires.append(f"{canonical}=={pkg_version}") + + return sorted(requires, key=lambda r: r.split("==")[0].lower()) + + +def _build_requires_block(requires: list[str]) -> str: + """Render the TOML requires array content (lines between the brackets).""" + lines = [] + for req in requires: + lines.append(f' "{req}",') + return "\n".join(lines) + + +def update_pyproject_build_requires( + hatchling_requires: list[str], + hatchling_with_git_requires: list[str], + versions: dict[str, str], +) -> bool: + """ + Scan all pyproject.toml files in the repo (excluding out/ directory). + + For each file: + - If build-system/requires contains flit_core → upgrade to latest flit_core version (if set). + - If build-system uses hatchling → replace the requires list with the resolved transitive deps. + airflow-core/pyproject.toml gets the gitpython-inclusive list; all others get the plain list. + + Returns True if any file was changed. + """ + changed = False + + # Pattern to match and replace the full requires = [...] block under [build-system] + # Handles both single-line and multi-line forms. + build_system_requires_re = re.compile( + r"(\[build-system\]\s*\n(?:[^\[]*?\n)*?requires\s*=\s*)(\[[^\]]*\])", + re.DOTALL, + ) + + flit_version = versions.get("flit_core", "") + + for pyproject_path in sorted(AIRFLOW_ROOT_PATH.rglob("pyproject.toml")): + # Skip anything under the out/ directory (reproducible build snapshots) + if "out/" in pyproject_path.as_posix().replace(str(AIRFLOW_ROOT_PATH) + "/", ""): + continue + + content = pyproject_path.read_text() + + # Determine if this file uses flit_core or hatchling + build_system_match = build_system_requires_re.search(content) + if not build_system_match: + continue + + requires_block = build_system_match.group(2) + + if "flit_core" in requires_block or "flit-core" in requires_block: + # Upgrade flit_core version if requested + if not (UPGRADE_FLIT_CORE and flit_version): + continue + new_content = apply_simple_regex_replacements( + content, flit_version, SIMPLE_VERSION_PATTERNS["flit_core"] + ) + if new_content != content: + pyproject_path.write_text(new_content) + console.print(f"[bright_blue]Updated flit_core in {pyproject_path}") + changed = True + + elif "hatchling" in requires_block or "hatchling" in content: + if not (UPGRADE_HATCHLING and hatchling_requires): + continue + # Choose the right list + is_airflow_core = pyproject_path == AIRFLOW_CORE_ROOT_PATH / "pyproject.toml" + target_requires = hatchling_with_git_requires if is_airflow_core else hatchling_requires + + new_requires_lines = _build_requires_block(target_requires) + new_requires_array = f"[\n{new_requires_lines}\n]" + + new_content = build_system_requires_re.sub( + lambda m, _new=new_requires_array: m.group(1) + _new, + content, + ) + + # Also apply the simple hatchling version pattern (for files that pin hatchling== elsewhere) + hatchling_version = next( + ( + r.split("==")[1].split(";")[0].strip() + for r in target_requires + if r.lower().startswith("hatchling==") + ), + "", + ) + if hatchling_version: + new_content = apply_simple_regex_replacements( + new_content, hatchling_version, SIMPLE_VERSION_PATTERNS["hatchling"] + ) + + if new_content != content: + pyproject_path.write_text(new_content) + console.print(f"[bright_blue]Updated hatchling build-system requires in {pyproject_path}") + changed = True + + return changed + + def main() -> None: """Main entry point for the version upgrade script.""" retrieve_gh_token(description="airflow-upgrade-important-versions", scopes="public_repo") @@ -681,6 +857,19 @@ def main() -> None: changed = process_all_files(versions, latest_python_versions) + # Resolve hatchling transitive dependencies and update all pyproject.toml build-system sections + if UPGRADE_HATCHLING or UPGRADE_FLIT_CORE: + console.print("[bright_blue]Resolving hatchling transitive build dependencies via uv pip compile") + hatchling_requires = resolve_hatchling_build_requires(with_gitpython=False) + hatchling_with_git_requires = resolve_hatchling_build_requires(with_gitpython=True) + if VERBOSE: + console.print(f"[bright_blue]Hatchling requires: {hatchling_requires}") + console.print(f"[bright_blue]Hatchling+GitPython requires: {hatchling_with_git_requires}") + pyproject_changed = update_pyproject_build_requires( + hatchling_requires, hatchling_with_git_requires, versions + ) + changed = changed or pyproject_changed + if changed: sync_breeze_lock_file() if not os.environ.get("CI"): diff --git a/shared/configuration/pyproject.toml b/shared/configuration/pyproject.toml index 3e65bec3cbbd6..26864255197d4 100644 --- a/shared/configuration/pyproject.toml +++ b/shared/configuration/pyproject.toml @@ -37,7 +37,19 @@ dev = [ ] [build-system] -requires = ["hatchling"] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] diff --git a/shared/dagnode/pyproject.toml b/shared/dagnode/pyproject.toml index d75d1cf3c54af..07f6efb605e96 100644 --- a/shared/dagnode/pyproject.toml +++ b/shared/dagnode/pyproject.toml @@ -33,7 +33,19 @@ dev = [ ] [build-system] -requires = ["hatchling"] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] diff --git a/shared/listeners/pyproject.toml b/shared/listeners/pyproject.toml index 445db6848e4c5..caa961a5a67bd 100644 --- a/shared/listeners/pyproject.toml +++ b/shared/listeners/pyproject.toml @@ -34,7 +34,19 @@ dev = [ ] [build-system] -requires = ["hatchling"] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] diff --git a/shared/logging/pyproject.toml b/shared/logging/pyproject.toml index c3abfbfa102b6..d181cdfc2c179 100644 --- a/shared/logging/pyproject.toml +++ b/shared/logging/pyproject.toml @@ -35,7 +35,19 @@ dev = [ ] [build-system] -requires = ["hatchling"] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] diff --git a/shared/module_loading/pyproject.toml b/shared/module_loading/pyproject.toml index 52fc5e636c0e1..71ae121525c36 100644 --- a/shared/module_loading/pyproject.toml +++ b/shared/module_loading/pyproject.toml @@ -34,7 +34,19 @@ dev = [ ] [build-system] -requires = ["hatchling"] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] diff --git a/shared/observability/pyproject.toml b/shared/observability/pyproject.toml index 0484e30a12a3f..48616af8752e2 100644 --- a/shared/observability/pyproject.toml +++ b/shared/observability/pyproject.toml @@ -51,7 +51,19 @@ dev = [ ] [build-system] -requires = ["hatchling"] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] diff --git a/shared/plugins_manager/pyproject.toml b/shared/plugins_manager/pyproject.toml index 769e8a0b5379f..71bd3ae05f2ce 100644 --- a/shared/plugins_manager/pyproject.toml +++ b/shared/plugins_manager/pyproject.toml @@ -38,7 +38,19 @@ dev = [ ] [build-system] -requires = ["hatchling"] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] diff --git a/shared/providers_discovery/pyproject.toml b/shared/providers_discovery/pyproject.toml index 46e59a5773966..bfb290af62e0b 100644 --- a/shared/providers_discovery/pyproject.toml +++ b/shared/providers_discovery/pyproject.toml @@ -41,7 +41,19 @@ dev = [ ] [build-system] -requires = ["hatchling"] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] diff --git a/shared/secrets_backend/pyproject.toml b/shared/secrets_backend/pyproject.toml index 7ebd2558ef53e..946ce071514c9 100644 --- a/shared/secrets_backend/pyproject.toml +++ b/shared/secrets_backend/pyproject.toml @@ -31,7 +31,19 @@ dev = [ ] [build-system] -requires = ["hatchling"] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] diff --git a/shared/secrets_masker/pyproject.toml b/shared/secrets_masker/pyproject.toml index 9a30e1646f63d..123a4d27a7c6b 100644 --- a/shared/secrets_masker/pyproject.toml +++ b/shared/secrets_masker/pyproject.toml @@ -36,7 +36,19 @@ dev = [ ] [build-system] -requires = ["hatchling"] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] diff --git a/shared/timezones/pyproject.toml b/shared/timezones/pyproject.toml index e56e33cb052e1..66e1b97ea4c1d 100644 --- a/shared/timezones/pyproject.toml +++ b/shared/timezones/pyproject.toml @@ -34,7 +34,19 @@ dev = [ ] [build-system] -requires = ["hatchling"] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] diff --git a/task-sdk-integration-tests/pyproject.toml b/task-sdk-integration-tests/pyproject.toml index 1c7d3595099fe..117fc326fcd1e 100644 --- a/task-sdk-integration-tests/pyproject.toml +++ b/task-sdk-integration-tests/pyproject.toml @@ -17,7 +17,19 @@ # under the License. [build-system] -requires = [ "hatchling==1.27.0" ] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [project] diff --git a/task-sdk/pyproject.toml b/task-sdk/pyproject.toml index 16caf6e8fd35d..8e62048ef7b37 100644 --- a/task-sdk/pyproject.toml +++ b/task-sdk/pyproject.toml @@ -118,7 +118,19 @@ YouTube = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" [build-system] -requires = ["hatchling"] +requires = [ + "distlib==0.4.0", + "filelock==3.24.3", + "hatchling==1.29.0", + "packaging==26.0", + "pathspec==1.0.4", + "platformdirs==4.9.2", + "pluggy==1.6.0", + "tomli==2.4.0; python_version < '3.11'", + "trove-classifiers==2026.1.14.14", + "typing-extensions==4.15.0; python_version < '3.11'", + "virtualenv==20.39.1", +] build-backend = "hatchling.build" [tool.hatch.version] From 69ecb6103298c5910502caf8b6a23c642aba4b4a Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 27 Feb 2026 16:05:18 +0100 Subject: [PATCH 2/2] Fix mypy 'Cannot infer type of lambda' error in upgrade script Co-Authored-By: Claude Opus 4.6 --- scripts/ci/prek/upgrade_important_versions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/ci/prek/upgrade_important_versions.py b/scripts/ci/prek/upgrade_important_versions.py index 12d81ef27e701..8df46b2e26fae 100755 --- a/scripts/ci/prek/upgrade_important_versions.py +++ b/scripts/ci/prek/upgrade_important_versions.py @@ -820,8 +820,11 @@ def update_pyproject_build_requires( new_requires_lines = _build_requires_block(target_requires) new_requires_array = f"[\n{new_requires_lines}\n]" + def _replace_requires(m: re.Match[str], _new: str = new_requires_array) -> str: + return m.group(1) + _new + new_content = build_system_requires_re.sub( - lambda m, _new=new_requires_array: m.group(1) + _new, + _replace_requires, content, )