diff --git a/INSTALL b/INSTALL index 35a00ee758c94..1483aef27c777 100644 --- a/INSTALL +++ b/INSTALL @@ -194,13 +194,17 @@ Google provider, tests, and all Hadoop providers with this command: This is not needed with `uv` and in the near future, when `pip` will support https://peps.python.org/pep-0735/ this installation will be simplified. -You can also install ALL possible dependencies of airflow with: +You can also install ALL possible dependencies of airflow (in editable mode only) with: pip install -e ".[all]" This is not really recommended usually (it takes a lot of time to resolve and install all dependencies) but if you want to replicate the CI environment where "all" extras are installed, you can do it this way. +You can also install ALL possible core dependencies of airflow (in editable mode only) with: + + pip install -e ".[all-core]" + Building airflow packages with Hatch ==================================== diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index e363a5f84a6f5..ba762f850b42d 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -243,7 +243,6 @@ These are extras that add dependencies needed for integration with external serv | zendesk | ``pip install 'apache-airflow[zendesk]'`` | Zendesk hooks | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ - Locally installed software extras ================================= @@ -354,28 +353,6 @@ pre-installed when Airflow is installed. | ssh | ``pip install 'apache-airflow[ssh]'`` | SSH hooks and operators | | +---------------------+-----------------------------------------------------+--------------------------------------+--------------+ -Production Bundle extras -------------------------- - -These are extras that install one or more extras as a bundle. - -+---------------------+-----------------------------------------------------+------------------------------------------------------------------------+ -| extra | install command | enables | -+=====================+=====================================================+========================================================================+ -| all | ``pip install 'apache-airflow[all]'`` | All Airflow user facing features (no devel and doc requirements) | -+---------------------+-----------------------------------------------------+------------------------------------------------------------------------+ -| all-core | ``pip install 'apache-airflow[all-core]'`` | All core airflow features that do not require installing providers | -+---------------------+-----------------------------------------------------+------------------------------------------------------------------------+ -| all-dbs | ``pip install 'apache-airflow[all-dbs]'`` | All database integrations | -+---------------------+-----------------------------------------------------+------------------------------------------------------------------------+ - -Development extras ------------------- - -The ``devel`` extras only make sense in editable mode. Users of Airflow should not be using them, unless they -start contributing back and install airflow from sources. Those extras are only available in Airflow when -it is installed in editable mode from sources (``pip install -e .[devel,EXTRAS]``). - Doc extras ========== diff --git a/hatch_build.py b/hatch_build.py index a895f311b7338..6b8294d1d75bd 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -507,6 +507,26 @@ def update_optional_dependencies_with_standard_provider_deps(optional_dependenci ] +def get_all_core_deps() -> list[str]: + all_core_deps: list[str] = [] + for deps in CORE_EXTRAS.values(): + all_core_deps.extend(deps) + return all_core_deps + + +def update_editable_optional_dependencies(optional_dependencies: dict[str, list[str]]): + optional_dependencies.update(CORE_EXTRAS) + optional_dependencies.update(DOC_EXTRAS) + update_optional_dependencies_with_editable_provider_deps(optional_dependencies) + all_deps: list[str] = [] + for extra, deps in optional_dependencies.items(): + if extra == "all": + raise RuntimeError("The 'all' extra should not be in the original optional_dependencies") + all_deps.extend(deps) + optional_dependencies["all"] = all_deps + optional_dependencies["all-core"] = get_all_core_deps() + + class CustomMetadataHook(MetadataHookInterface): """ Custom metadata hook that updates optional dependencies and dependencies of airflow. @@ -544,20 +564,12 @@ class CustomMetadataHook(MetadataHookInterface): def update(self, metadata: dict) -> None: optional_dependencies: dict[str, list[str]] = {} + update_editable_optional_dependencies(optional_dependencies) metadata["optional-dependencies"] = optional_dependencies - optional_dependencies.update(CORE_EXTRAS) - optional_dependencies.update(DOC_EXTRAS) - update_optional_dependencies_with_editable_provider_deps(optional_dependencies) dependencies: list[str] = [] - metadata["dependencies"] = dependencies dependencies.extend(DEPENDENCIES) dependencies.extend(PREINSTALLED_PROVIDER_REQUIREMENTS) - all_deps: list[str] = [] - for extra, deps in optional_dependencies.items(): - if extra == "all": - raise RuntimeError("The 'all' extra should not be in the original optional_dependencies") - all_deps.extend(deps) - optional_dependencies["all"] = all_deps + metadata["dependencies"] = dependencies class CustomBuildHook(BuildHookInterface[BuilderConfig]): @@ -602,17 +614,7 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: # Add preinstalled providers into the dependencies for standard packages self._dependencies.extend(PREINSTALLED_PROVIDER_REQUIREMENTS) else: - self._dependencies.extend(ALL_PREINSTALLED_PROVIDER_DEPS) - self.optional_dependencies.update(CORE_EXTRAS) - # only add doc extras for editable build - self.optional_dependencies.update(DOC_EXTRAS) - update_optional_dependencies_with_editable_provider_deps(self.optional_dependencies) - all_deps: list[str] = [] - for extra, deps in self.optional_dependencies.items(): - if extra == "all": - raise RuntimeError("The 'all' extra should not be in the original optional_dependencies") - all_deps.extend(deps) - self.optional_dependencies["all"] = all_deps + update_editable_optional_dependencies(self.optional_dependencies) # with hatchling, we can modify dependencies dynamically by modifying the build_data build_data["dependencies"] = self._dependencies diff --git a/newsfragments/47441.significant.rst b/newsfragments/47441.significant.rst new file mode 100644 index 0000000000000..05fe947efe0eb --- /dev/null +++ b/newsfragments/47441.significant.rst @@ -0,0 +1,15 @@ +There are no more production bundle or devel extras + +There are no more production ``all*`` or ``devel*`` bundle extras available in ``wheel`` package of airflow. +If you want to install airflow with all extras you can use ``uv pip install --all-extras`` command. + +* Types of change + + * [ ] Dag changes + * [ ] Config changes + * [ ] API changes + * [ ] CLI changes + * [ ] Behaviour changes + * [ ] Plugin changes + * [x] Dependency changes + * [ ] Code interface changes