diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc730277ce64d..409dfaf44d0e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,7 +175,7 @@ jobs: # We only push CI cache as PROD cache usually does not gain as much from fresh cache because # it uses prepared airflow and provider packages that invalidate the cache anyway most of the time push-early-buildx-cache-to-github-registry: - timeout-minutes: 50 + timeout-minutes: 110 name: "Push Early Image Cache" runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} needs: diff --git a/Dockerfile b/Dockerfile index 4ac935925fd6a..f5efda7abe996 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,6 +49,8 @@ ARG AIRFLOW_VERSION="2.8.1" ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm" ARG AIRFLOW_PIP_VERSION=24.0 +ARG AIRFLOW_UV_VERSION=0.1.10 +ARG AIRFLOW_USE_UV="false" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" ARG AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md" @@ -336,7 +338,6 @@ set -euo pipefail common::get_colors declare -a packages -: "${AIRFLOW_PIP_VERSION:?Should be set}" : "${INSTALL_MSSQL_CLIENT:?Should be true or false}" @@ -416,14 +417,13 @@ COPY <<"EOF" /install_pip_version.sh #!/usr/bin/env bash . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location -common::install_pip_version +common::install_packaging_tool EOF # The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -436,7 +436,6 @@ COPY <<"EOF" /install_airflow_dependencies_from_branch_tip.sh : "${AIRFLOW_BRANCH:?Should be set}" : "${INSTALL_MYSQL_CLIENT:?Should be true or false}" : "${INSTALL_POSTGRES_CLIENT:?Should be true or false}" -: "${AIRFLOW_PIP_VERSION:?Should be set}" function install_airflow_dependencies_from_branch_tip() { echo @@ -452,26 +451,27 @@ function install_airflow_dependencies_from_branch_tip() { # dependencies that we can cache and reuse when installing airflow using constraints and latest # pyproject.toml in the next step (when we install regular airflow). set -x - pip install --root-user-action ignore \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "apache-airflow[${AIRFLOW_EXTRAS}] @ https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" - common::install_pip_version + common::install_packaging_tool # Uninstall airflow and providers to keep only the dependencies. In the future when # planned https://github.com/pypa/pip/issues/11440 is implemented in pip we might be able to use this # flag and skip the remove step. - pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true + ${PACKAGING_TOOL_CMD} freeze | grep apache-airflow-providers | xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true set +x echo echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now target airflow can be reinstalled using mostly cached dependencies${COLOR_RESET}" echo - pip uninstall --yes apache-airflow || true + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} apache-airflow || true } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow_dependencies_from_branch_tip EOF @@ -481,6 +481,10 @@ COPY <<"EOF" /common.sh #!/usr/bin/env bash set -euo pipefail +: "${AIRFLOW_PIP_VERSION:?Should be set}" +: "${AIRFLOW_UV_VERSION:?Should be set}" +: "${AIRFLOW_USE_UV:?Should be set}" + function common::get_colors() { COLOR_BLUE=$'\e[34m' COLOR_GREEN=$'\e[32m' @@ -494,6 +498,40 @@ function common::get_colors() { export COLOR_YELLOW } +function common::get_packaging_tool() { + ## IMPORTANT: IF YOU MODIFY THIS FUNCTION YOU SHOULD ALSO MODIFY CORRESPONDING FUNCTION IN + ## `scripts/in_container/_in_container_utils.sh` + if [[ ${AIRFLOW_USE_UV} == "true" ]]; then + echo + echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" + echo + export PACKAGING_TOOL="uv" + export PACKAGING_TOOL_CMD="uv pip" + export EXTRA_INSTALL_FLAGS="" + export EXTRA_UNINSTALL_FLAGS="" + export RESOLUTION_HIGHEST_FLAG="--resolution highest" + export RESOLUTION_LOWEST_DIRECT_FLAG="--resolution lowest-direct" + # We need to lie about VIRTUAL_ENV to make uv works + # Until https://github.com/astral-sh/uv/issues/1396 is fixed + # In case we are running user installation, we need to set VIRTUAL_ENV to user's home + .local + if [[ ${PIP_USER=} == "true" ]]; then + VIRTUAL_ENV="${HOME}/.local" + else + VIRTUAL_ENV=$(python -c "import sys; print(sys.prefix)") + fi + export VIRTUAL_ENV + else + echo + echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}" + echo + export PACKAGING_TOOL="pip" + export PACKAGING_TOOL_CMD="pip" + export EXTRA_INSTALL_FLAGS="--root-user-action ignore" + export EXTRA_UNINSTALL_FLAGS="--yes" + export RESOLUTION_HIGHEST_FLAG="--upgrade-strategy eager" + export RESOLUTION_LOWEST_DIRECT_FLAG="--upgrade --upgrade-strategy only-if-needed" + fi +} function common::get_airflow_version_specification() { if [[ -z ${AIRFLOW_VERSION_SPECIFICATION=} @@ -529,20 +567,41 @@ function common::get_constraints_location() { fi } -function common::show_pip_version_and_location() { +function common::show_packaging_tool_version_and_location() { echo "PATH=${PATH}" - echo "pip on path: $(which pip)" - echo "Using pip: $(pip --version)" + if [[ ${PACKAGING_TOOL} == "pip" ]]; then + echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}" + echo "pip on path: $(which pip)" + echo "Using pip: $(pip --version)" + else + echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" + echo "uv on path: $(which uv)" + echo "Using uv: $(uv --version)" + fi } -function common::install_pip_version() { +function common::install_packaging_tool() { echo echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" echo if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then - pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}" + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}" else - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + fi + if [[ ${AIRFLOW_USE_UV} == "true" ]]; then + echo + echo "${COLOR_BLUE}Installing uv version ${AIRFLOW_UV_VERSION}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}" + else + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" + fi fi mkdir -p "${HOME}/.local/bin" } @@ -601,8 +660,6 @@ COPY <<"EOF" /install_from_docker_context_files.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - function install_airflow_and_providers_from_docker_context_files(){ if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,} @@ -619,7 +676,7 @@ function install_airflow_and_providers_from_docker_context_files(){ fi # shellcheck disable=SC2206 - local pip_flags=( + local packaging_flags=( # Don't quote this -- if it is empty we don't want it to create an # empty array element --find-links="file:///docker-context-files" @@ -669,7 +726,7 @@ function install_airflow_and_providers_from_docker_context_files(){ echo # force reinstall all airflow + provider packages with constraints found in set -x - pip install "${pip_flags[@]}" --root-user-action ignore --upgrade \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" --upgrade \ ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint "${local_constraints_file}" \ ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} set +x @@ -678,7 +735,7 @@ function install_airflow_and_providers_from_docker_context_files(){ echo "${COLOR_BLUE}Installing docker-context-files packages with constraints from GitHub${COLOR_RESET}" echo set -x - pip install "${pip_flags[@]}" --root-user-action ignore \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" \ ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} @@ -689,17 +746,16 @@ function install_airflow_and_providers_from_docker_context_files(){ echo "${COLOR_BLUE}Installing docker-context-files packages without constraints${COLOR_RESET}" echo set -x - pip install "${pip_flags[@]}" --root-user-action ignore \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} set +x fi - common::install_pip_version + common::install_packaging_tool pip check } function install_all_other_packages_from_docker_context_files() { - echo echo "${COLOR_BLUE}Force re-installing all other package from local files without dependencies${COLOR_RESET}" echo @@ -709,22 +765,22 @@ function install_all_other_packages_from_docker_context_files() { grep -v apache_airflow | grep -v apache-airflow || true) if [[ -n "${reinstalling_other_packages}" ]]; then set -x - pip install ${ADDITIONAL_PIP_INSTALL_FLAGS} \ - --root-user-action ignore --force-reinstall --no-deps --no-index ${reinstalling_other_packages} - common::install_pip_version + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + --force-reinstall --no-deps --no-index ${reinstalling_other_packages} + common::install_packaging_tool set +x fi } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow_and_providers_from_docker_context_files -common::show_pip_version_and_location install_all_other_packages_from_docker_context_files EOF @@ -734,8 +790,6 @@ COPY <<"EOF" /install_airflow.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - function install_airflow() { # Coherence check for editable installation mode. if [[ ${AIRFLOW_INSTALLATION_METHOD} != "." && \ @@ -760,22 +814,21 @@ function install_airflow() { echo "${COLOR_BLUE}Remove airflow and all provider packages installed before potentially${COLOR_RESET}" echo set -x - pip freeze | grep apache-airflow | xargs pip uninstall --yes 2>/dev/null || true + ${PACKAGING_TOOL_CMD} freeze | grep apache-airflow | xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true set +x echo echo "${COLOR_BLUE}Installing all packages with eager upgrade with ${AIRFLOW_INSTALL_EDITABLE_FLAG} mode${COLOR_RESET}" echo set -x - pip install --root-user-action ignore \ - --upgrade --upgrade-strategy eager \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} set +x - common::install_pip_version + common::install_packaging_tool echo - echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" + echo "${COLOR_BLUE}Running '${PACKAGING_TOOL} check'${COLOR_RESET}" echo pip check else @@ -783,17 +836,17 @@ function install_airflow() { echo "${COLOR_BLUE}Installing all packages with constraints and upgrade if needed${COLOR_RESET}" echo set -x - pip install --root-user-action ignore ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true - common::install_pip_version + common::install_packaging_tool # then upgrade if needed without using constraints to account for new limits in pyproject.toml - pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_LOWEST_DIRECT_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -804,10 +857,11 @@ function install_airflow() { } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow EOF @@ -819,7 +873,6 @@ set -euo pipefail : "${UPGRADE_TO_NEWER_DEPENDENCIES:?Should be true or false}" : "${ADDITIONAL_PYTHON_DEPS:?Should be set}" -: "${AIRFLOW_PIP_VERSION:?Should be set}" . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" @@ -829,10 +882,10 @@ function install_additional_dependencies() { echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}" echo set -x - pip install --root-user-action ignore --upgrade --upgrade-strategy eager \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -843,10 +896,10 @@ function install_additional_dependencies() { echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}" echo set -x - pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "${RESOLUTION_LOWEST_DIRECT_FLAG}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -856,10 +909,11 @@ function install_additional_dependencies() { } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_additional_dependencies EOF @@ -1246,6 +1300,8 @@ ARG INSTALL_MYSQL_CLIENT_TYPE="mariadb" ARG INSTALL_MSSQL_CLIENT="true" ARG INSTALL_POSTGRES_CLIENT="true" ARG AIRFLOW_PIP_VERSION +ARG AIRFLOW_UV_VERSION +ARG AIRFLOW_USE_UV ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \ INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE} \ @@ -1327,6 +1383,8 @@ RUN if [[ -f /docker-context-files/pip.conf ]]; then \ ARG ADDITIONAL_PIP_INSTALL_FLAGS="" ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} \ AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ AIRFLOW_VERSION=${AIRFLOW_VERSION} \ AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD} \ @@ -1342,7 +1400,6 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \ DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \ PATH=${PATH}:${AIRFLOW_USER_HOME_DIR}/.local/bin \ - AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \ ADDITIONAL_PIP_INSTALL_FLAGS=${ADDITIONAL_PIP_INSTALL_FLAGS} \ AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \ @@ -1458,12 +1515,16 @@ LABEL org.apache.airflow.distro="debian" \ ARG PYTHON_BASE_IMAGE ARG AIRFLOW_PIP_VERSION +ARG AIRFLOW_UV_VERSION +ARG AIRFLOW_USE_UV ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \ # Make sure noninteractive debian install is used and language variables set DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 LD_LIBRARY_PATH=/usr/local/lib \ - AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} + AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} ARG RUNTIME_APT_DEPS="" ARG ADDITIONAL_RUNTIME_APT_DEPS="" @@ -1503,10 +1564,14 @@ ENV PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH}" \ AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \ AIRFLOW_HOME=${AIRFLOW_HOME} -# THE 3 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON +# THE 7 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON # AND SHOULD BE REMOVED WHEN WORKAROUND IN install_mssql.sh IS REMOVED ARG AIRFLOW_PIP_VERSION=24.0 -ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} +ARG AIRFLOW_UV_VERSION=0.1.10 +ARG AIRFLOW_USE_UV="false" +ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} COPY --from=scripts common.sh /scripts/docker/ # Only copy mysql/mssql installation scripts for now - so that changing the other diff --git a/Dockerfile.ci b/Dockerfile.ci index f5044a2ef9b4c..8e927094e8a9e 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -296,7 +296,6 @@ set -euo pipefail common::get_colors declare -a packages -: "${AIRFLOW_PIP_VERSION:?Should be set}" : "${INSTALL_MSSQL_CLIENT:?Should be true or false}" @@ -376,14 +375,13 @@ COPY <<"EOF" /install_pip_version.sh #!/usr/bin/env bash . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location -common::install_pip_version +common::install_packaging_tool EOF # The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -396,7 +394,6 @@ COPY <<"EOF" /install_airflow_dependencies_from_branch_tip.sh : "${AIRFLOW_BRANCH:?Should be set}" : "${INSTALL_MYSQL_CLIENT:?Should be true or false}" : "${INSTALL_POSTGRES_CLIENT:?Should be true or false}" -: "${AIRFLOW_PIP_VERSION:?Should be set}" function install_airflow_dependencies_from_branch_tip() { echo @@ -412,26 +409,27 @@ function install_airflow_dependencies_from_branch_tip() { # dependencies that we can cache and reuse when installing airflow using constraints and latest # pyproject.toml in the next step (when we install regular airflow). set -x - pip install --root-user-action ignore \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "apache-airflow[${AIRFLOW_EXTRAS}] @ https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" - common::install_pip_version + common::install_packaging_tool # Uninstall airflow and providers to keep only the dependencies. In the future when # planned https://github.com/pypa/pip/issues/11440 is implemented in pip we might be able to use this # flag and skip the remove step. - pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true + ${PACKAGING_TOOL_CMD} freeze | grep apache-airflow-providers | xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true set +x echo echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now target airflow can be reinstalled using mostly cached dependencies${COLOR_RESET}" echo - pip uninstall --yes apache-airflow || true + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} apache-airflow || true } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow_dependencies_from_branch_tip EOF @@ -441,6 +439,10 @@ COPY <<"EOF" /common.sh #!/usr/bin/env bash set -euo pipefail +: "${AIRFLOW_PIP_VERSION:?Should be set}" +: "${AIRFLOW_UV_VERSION:?Should be set}" +: "${AIRFLOW_USE_UV:?Should be set}" + function common::get_colors() { COLOR_BLUE=$'\e[34m' COLOR_GREEN=$'\e[32m' @@ -454,6 +456,40 @@ function common::get_colors() { export COLOR_YELLOW } +function common::get_packaging_tool() { + ## IMPORTANT: IF YOU MODIFY THIS FUNCTION YOU SHOULD ALSO MODIFY CORRESPONDING FUNCTION IN + ## `scripts/in_container/_in_container_utils.sh` + if [[ ${AIRFLOW_USE_UV} == "true" ]]; then + echo + echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" + echo + export PACKAGING_TOOL="uv" + export PACKAGING_TOOL_CMD="uv pip" + export EXTRA_INSTALL_FLAGS="" + export EXTRA_UNINSTALL_FLAGS="" + export RESOLUTION_HIGHEST_FLAG="--resolution highest" + export RESOLUTION_LOWEST_DIRECT_FLAG="--resolution lowest-direct" + # We need to lie about VIRTUAL_ENV to make uv works + # Until https://github.com/astral-sh/uv/issues/1396 is fixed + # In case we are running user installation, we need to set VIRTUAL_ENV to user's home + .local + if [[ ${PIP_USER=} == "true" ]]; then + VIRTUAL_ENV="${HOME}/.local" + else + VIRTUAL_ENV=$(python -c "import sys; print(sys.prefix)") + fi + export VIRTUAL_ENV + else + echo + echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}" + echo + export PACKAGING_TOOL="pip" + export PACKAGING_TOOL_CMD="pip" + export EXTRA_INSTALL_FLAGS="--root-user-action ignore" + export EXTRA_UNINSTALL_FLAGS="--yes" + export RESOLUTION_HIGHEST_FLAG="--upgrade-strategy eager" + export RESOLUTION_LOWEST_DIRECT_FLAG="--upgrade --upgrade-strategy only-if-needed" + fi +} function common::get_airflow_version_specification() { if [[ -z ${AIRFLOW_VERSION_SPECIFICATION=} @@ -489,20 +525,41 @@ function common::get_constraints_location() { fi } -function common::show_pip_version_and_location() { +function common::show_packaging_tool_version_and_location() { echo "PATH=${PATH}" - echo "pip on path: $(which pip)" - echo "Using pip: $(pip --version)" + if [[ ${PACKAGING_TOOL} == "pip" ]]; then + echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}" + echo "pip on path: $(which pip)" + echo "Using pip: $(pip --version)" + else + echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" + echo "uv on path: $(which uv)" + echo "Using uv: $(uv --version)" + fi } -function common::install_pip_version() { +function common::install_packaging_tool() { echo echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" echo if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then - pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}" + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}" else - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + fi + if [[ ${AIRFLOW_USE_UV} == "true" ]]; then + echo + echo "${COLOR_BLUE}Installing uv version ${AIRFLOW_UV_VERSION}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}" + else + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" + fi fi mkdir -p "${HOME}/.local/bin" } @@ -548,7 +605,7 @@ function install_pipx_tools() { echo "${COLOR_BLUE}Installing pipx tools${COLOR_RESET}" echo # Make sure PIPX is installed in latest version - pip install --root-user-action ignore --upgrade "pipx>=1.2.1" + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pipx>=1.2.1" if [[ $(uname -m) != "aarch64" ]]; then # Do not install mssql-cli for ARM # Install all the tools we need available in command line but without impacting the current environment @@ -562,6 +619,7 @@ function install_pipx_tools() { } common::get_colors +common::get_packaging_tool install_pipx_tools EOF @@ -572,8 +630,6 @@ COPY <<"EOF" /install_airflow.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - function install_airflow() { # Coherence check for editable installation mode. if [[ ${AIRFLOW_INSTALLATION_METHOD} != "." && \ @@ -598,22 +654,21 @@ function install_airflow() { echo "${COLOR_BLUE}Remove airflow and all provider packages installed before potentially${COLOR_RESET}" echo set -x - pip freeze | grep apache-airflow | xargs pip uninstall --yes 2>/dev/null || true + ${PACKAGING_TOOL_CMD} freeze | grep apache-airflow | xargs ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true set +x echo echo "${COLOR_BLUE}Installing all packages with eager upgrade with ${AIRFLOW_INSTALL_EDITABLE_FLAG} mode${COLOR_RESET}" echo set -x - pip install --root-user-action ignore \ - --upgrade --upgrade-strategy eager \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} set +x - common::install_pip_version + common::install_packaging_tool echo - echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" + echo "${COLOR_BLUE}Running '${PACKAGING_TOOL} check'${COLOR_RESET}" echo pip check else @@ -621,17 +676,17 @@ function install_airflow() { echo "${COLOR_BLUE}Installing all packages with constraints and upgrade if needed${COLOR_RESET}" echo set -x - pip install --root-user-action ignore ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true - common::install_pip_version + common::install_packaging_tool # then upgrade if needed without using constraints to account for new limits in pyproject.toml - pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_LOWEST_DIRECT_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -642,10 +697,11 @@ function install_airflow() { } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow EOF @@ -657,7 +713,6 @@ set -euo pipefail : "${UPGRADE_TO_NEWER_DEPENDENCIES:?Should be true or false}" : "${ADDITIONAL_PYTHON_DEPS:?Should be set}" -: "${AIRFLOW_PIP_VERSION:?Should be set}" . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" @@ -667,10 +722,10 @@ function install_additional_dependencies() { echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}" echo set -x - pip install --root-user-action ignore --upgrade --upgrade-strategy eager \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -681,10 +736,10 @@ function install_additional_dependencies() { echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}" echo set -x - pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "${RESOLUTION_LOWEST_DIRECT_FLAG}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -694,10 +749,11 @@ function install_additional_dependencies() { } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_additional_dependencies EOF @@ -893,8 +949,12 @@ function check_boto_upgrade() { echo echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" echo - pip uninstall --root-user-action ignore aiobotocore s3fs -y || true - pip install --root-user-action ignore --upgrade boto3 botocore + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs || true + # We need to include oss2 as dependency as otherwise jmespath will be bumped and it will not pass + # the pip check test, Similarly gcloud-aio-auth limit is needed to be included as it bumps cryptography + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore "oss2>=2.14.0" "gcloud-aio-auth>=4.0.0,<5.0.0" pip check } @@ -903,25 +963,31 @@ function check_pydantic() { echo echo "${COLOR_YELLOW}Reinstalling airflow from local sources to account for pyproject.toml changes${COLOR_RESET}" echo - pip install --root-user-action ignore -e . + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} -e . echo echo "${COLOR_YELLOW}Remove pydantic and 3rd party libraries that depend on it${COLOR_RESET}" echo - pip uninstall --root-user-action ignore pydantic aws-sam-translator openai pyiceberg qdrant-client cfn-lint -y + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} pydantic aws-sam-translator openai \ + pyiceberg qdrant-client cfn-lint weaviate-client pip check elif [[ ${PYDANTIC=} == "v1" ]]; then echo echo "${COLOR_YELLOW}Reinstalling airflow from local sources to account for pyproject.toml changes${COLOR_RESET}" echo - pip install --root-user-action ignore -e . + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} -e . echo - echo "${COLOR_YELLOW}Uninstalling pyicberg which is not compatible with Pydantic 1${COLOR_RESET}" + echo "${COLOR_YELLOW}Uninstalling dependencies which are not compatible with Pydantic 1${COLOR_RESET}" echo - pip uninstall pyiceberg -y + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} pyiceberg waeviate-client echo echo "${COLOR_YELLOW}Downgrading Pydantic to < 2${COLOR_RESET}" echo - pip install --upgrade "pydantic<2.0.0" + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pydantic<2.0.0" pip check else echo @@ -939,7 +1005,8 @@ function check_download_sqlalchemy() { echo echo "${COLOR_BLUE}Downgrading sqlalchemy to minimum supported version: ${min_sqlalchemy_version}${COLOR_RESET}" echo - pip install --root-user-action ignore "sqlalchemy==${min_sqlalchemy_version}" + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "sqlalchemy==${min_sqlalchemy_version}" pip check } @@ -951,7 +1018,8 @@ function check_download_pendulum() { echo echo "${COLOR_BLUE}Downgrading pendulum to minimum supported version: ${min_pendulum_version}${COLOR_RESET}" echo - pip install --root-user-action ignore "pendulum==${min_pendulum_version}" + # shellcheck disable=SC2086 + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "pendulum==${min_pendulum_version}" pip check } @@ -1042,10 +1110,14 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \ COPY --from=scripts install_os_dependencies.sh /scripts/docker/ RUN bash /scripts/docker/install_os_dependencies.sh dev -# THE 3 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON +# THE 7 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON # AND SHOULD BE REMOVED WHEN WORKAROUND IN install_mssql.sh IS REMOVED ARG AIRFLOW_PIP_VERSION=24.0 -ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} +ARG AIRFLOW_UV_VERSION=0.1.10 +ARG AIRFLOW_USE_UV="true" +ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} COPY --from=scripts common.sh /scripts/docker/ # Only copy mysql/mssql installation scripts for now - so that changing the other @@ -1108,9 +1180,13 @@ ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" ARG AIRFLOW_CI_BUILD_EPOCH="10" ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" ARG AIRFLOW_PIP_VERSION=24.0 +ARG AIRFLOW_UV_VERSION=0.1.10 +ARG AIRFLOW_USE_UV="true" # Setup PIP # By default PIP install run without cache to make image smaller ARG PIP_NO_CACHE_DIR="true" +# By default UV install run without cache to make image smaller +ARG UV_NO_CACHE="true" # By default PIP has progress bar but you can disable it. ARG PIP_PROGRESS_BAR="on" # Optimizing installation of Cassandra driver (in case there are no prebuilt wheels which is the @@ -1138,6 +1214,8 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ AIRFLOW_VERSION=${AIRFLOW_VERSION} \ AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} \ # In the CI image we always: # * install MySQL, MsSQL # * install airflow from current sources, not from PyPI package @@ -1153,6 +1231,7 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ AIRFLOW_VERSION_SPECIFICATION="" \ PIP_NO_CACHE_DIR=${PIP_NO_CACHE_DIR} \ PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \ + UV_NO_CACHE=${UV_NO_CACHE} \ ADDITIONAL_PIP_INSTALL_FLAGS=${ADDITIONAL_PIP_INSTALL_FLAGS} \ CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY} \ CASS_DRIVER_NO_CYTHON=${CASS_DRIVER_NO_CYTHON} diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index 4c6fc2c016bb7..7807d94b1e242 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -428,6 +428,7 @@ can be used for CI images: | `DEPENDENCIES_EPOCH_NUMBER` | `2` | increasing this number will reinstall all apt dependencies | | `ADDITIONAL_PIP_INSTALL_FLAGS` | | additional `pip` flags passed to the installation commands (except when reinstalling `pip` itself) | | `PIP_NO_CACHE_DIR` | `true` | if true, then no pip cache will be stored | +| `UV_NO_CACHE` | `true` | if true, then no uv cache will be stored | | `HOME` | `/root` | Home directory of the root user (CI image has root user as default) | | `AIRFLOW_HOME` | `/root/airflow` | Airflow's HOME (that's where logs and sqlite databases are stored) | | `AIRFLOW_SOURCES` | `/opt/airflow` | Mounted sources of Airflow | @@ -447,6 +448,8 @@ can be used for CI images: | `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | | `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | | `AIRFLOW_PIP_VERSION` | `24.0` | PIP version used. | +| `AIRFLOW_UV_VERSION` | `0.1.10` | UV version used. | +| `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | | `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | Here are some examples of how CI images can built manually. CI is always diff --git a/dev/breeze/doc/images/output_ci-image_build.svg b/dev/breeze/doc/images/output_ci-image_build.svg index 17ea15e9e999f..f617eb05e72aa 100644 --- a/dev/breeze/doc/images/output_ci-image_build.svg +++ b/dev/breeze/doc/images/output_ci-image_build.svg @@ -1,4 +1,4 @@ - +