diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index da76fcbeffdaf..3862df0e13521 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -318,6 +318,7 @@ jobs: DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} + USE_UV: "true" build-ci-images-arm: timeout-minutes: 50 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc730277ce64d..b1876525ef9a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1890,6 +1890,63 @@ jobs: DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} + USE_UV: "true" + + build-prod-images-pip: + strategy: + matrix: + python-version: ${{fromJson(needs.build-info.outputs.python-versions)}} + timeout-minutes: 80 + name: Build pip PROD image${{matrix.python-version}} + runs-on: ["ubuntu-22.04"] + needs: [build-info, build-ci-images] + if: needs.build-info.outputs.canary-run == 'true' + env: + DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} + DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} + RUNS_ON: "${{needs.build-info.outputs.runs-on}}" + BACKEND: sqlite + VERSION_SUFFIX_FOR_PYPI: "dev0" + DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + steps: + - name: Cleanup repo + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' + - uses: actions/checkout@v3 + with: + ref: ${{ needs.build-info.outputs.targetCommitSha }} + persist-credentials: false + submodules: recursive + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' + - name: Build pip PROD Image${{ matrix.python-version }}:${{env.IMAGE_TAG}} + uses: ./.github/actions/build-prod-images + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' + with: + build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} + python-version: ${{ matrix.python-version }} + env: + UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} + PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} + DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} + # Do not override the "bookworm" image - just push a new bullseye image + # TODO: improve caching for that build + IMAGE_TAG: "pip-${{ github.event.pull_request.head.sha || github.sha }}" + USE_UV: "false" build-prod-images-bullseye: strategy: @@ -1946,6 +2003,7 @@ jobs: # Do not override the "bookworm" image - just push a new bullseye image # TODO: improve caching for that build IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}" + USE_UV: "true" build-prod-images-mysql-client: strategy: @@ -2002,6 +2060,7 @@ jobs: # Do not override the "mariadb" (original) image - just push a new mysql image # TODO: improve caching for that build IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}" + USE_UV: "true" build-prod-images-release-branch: @@ -2054,6 +2113,7 @@ jobs: DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} + USE_UV: "true" build-prod-images-bullseye-release-branch: strategy: @@ -2110,6 +2170,7 @@ jobs: # Do not override the "bookworm" image - just push a new bullseye image # TODO: improve caching for that build IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}" + USE_UV: "true" build-prod-images-mysql-release-branch: strategy: @@ -2166,6 +2227,8 @@ jobs: # Do not override the "mariadb" image - just push a new mysql image # TODO: improve caching for that build IMAGE_TAG: "mysql-${{ github.event.pull_request.head.sha || github.sha }}" + USE_UV: "true" + wait-for-prod-images: timeout-minutes: 80 diff --git a/Dockerfile b/Dockerfile index 4ac935925fd6a..4c6c60274a76b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,6 +49,8 @@ ARG AIRFLOW_VERSION="2.8.1" ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm" ARG AIRFLOW_PIP_VERSION=24.0 +ARG AIRFLOW_UV_VERSION=0.1.10 +ARG AIRFLOW_USE_UV="false" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" ARG AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md" @@ -336,7 +338,6 @@ set -euo pipefail common::get_colors declare -a packages -: "${AIRFLOW_PIP_VERSION:?Should be set}" : "${INSTALL_MSSQL_CLIENT:?Should be true or false}" @@ -416,14 +417,13 @@ COPY <<"EOF" /install_pip_version.sh #!/usr/bin/env bash . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location -common::install_pip_version +common::install_packaging_tool EOF # The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -436,7 +436,6 @@ COPY <<"EOF" /install_airflow_dependencies_from_branch_tip.sh : "${AIRFLOW_BRANCH:?Should be set}" : "${INSTALL_MYSQL_CLIENT:?Should be true or false}" : "${INSTALL_POSTGRES_CLIENT:?Should be true or false}" -: "${AIRFLOW_PIP_VERSION:?Should be set}" function install_airflow_dependencies_from_branch_tip() { echo @@ -452,26 +451,27 @@ function install_airflow_dependencies_from_branch_tip() { # dependencies that we can cache and reuse when installing airflow using constraints and latest # pyproject.toml in the next step (when we install regular airflow). set -x - pip install --root-user-action ignore \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "apache-airflow[${AIRFLOW_EXTRAS}] @ https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" - common::install_pip_version + common::install_packaging_tool # Uninstall airflow and providers to keep only the dependencies. In the future when # planned https://github.com/pypa/pip/issues/11440 is implemented in pip we might be able to use this # flag and skip the remove step. - pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true + ${PACKAGING_TOOL} freeze | grep apache-airflow-providers | xargs ${PACKAGING_TOOL} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true set +x echo echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now target airflow can be reinstalled using mostly cached dependencies${COLOR_RESET}" echo - pip uninstall --yes apache-airflow || true + ${PACKAGING_TOOL} uninstall ${EXTRA_UNINSTALL_FLAGS} apache-airflow || true } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow_dependencies_from_branch_tip EOF @@ -481,6 +481,10 @@ COPY <<"EOF" /common.sh #!/usr/bin/env bash set -euo pipefail +: "${AIRFLOW_PIP_VERSION:?Should be set}" +: "${AIRFLOW_UV_VERSION:?Should be set}" +: "${AIRFLOW_USE_UV:?Should be set}" + function common::get_colors() { COLOR_BLUE=$'\e[34m' COLOR_GREEN=$'\e[32m' @@ -494,6 +498,31 @@ function common::get_colors() { export COLOR_YELLOW } +function common::get_packaging_tool() { + if [[ ${AIRFLOW_USE_UV} == "true" ]]; then + echo + echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" + echo + export PACKAGING_TOOL="uv pip" + export EXTRA_INSTALL_FLAGS="" + export EXTRA_UNINSTALL_FLAGS="" + export RESOLUTION_HIGHEST_FLAG="--resolution highest" + export RESOLUTION_LOWEST_DIRECT_FLAG="--resolution lowest-direct" + # We need to lie about VIRTUAL_ENV to make uv works + # Until https://github.com/astral-sh/uv/issues/1396 is fixed + VIRTUAL_ENV=$(python -c "import sys; print(sys.prefix)") + export VIRTUAL_ENV + else + echo + echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}" + echo + export PACKAGING_TOOL="pip" + export EXTRA_INSTALL_FLAGS="--root-user-action ignore" + export EXTRA_UNINSTALL_FLAGS="--yes" + export RESOLUTION_HIGHEST_FLAG="--upgrade-strategy eager" + export RESOLUTION_LOWEST_DIRECT_FLAG="--upgrade --upgrade-strategy only-if-needed" + fi +} function common::get_airflow_version_specification() { if [[ -z ${AIRFLOW_VERSION_SPECIFICATION=} @@ -529,20 +558,41 @@ function common::get_constraints_location() { fi } -function common::show_pip_version_and_location() { +function common::show_packaging_tool_version_and_location() { echo "PATH=${PATH}" - echo "pip on path: $(which pip)" - echo "Using pip: $(pip --version)" + if [[ ${PACKAGING_TOOL} == "pip" ]]; then + echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}" + echo "pip on path: $(which pip)" + echo "Using pip: $(pip --version)" + else + echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" + echo "uv on path: $(which uv)" + echo "Using uv: $(uv --version)" + fi } -function common::install_pip_version() { +function common::install_packaging_tool() { echo echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" echo if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then - pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}" + # shellcheck disable=SC2086 + pip install ${EXTRA_INSTALL_FLAGS} --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}" else - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + # shellcheck disable=SC2086 + pip install ${EXTRA_INSTALL_FLAGS} --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + fi + if [[ ${AIRFLOW_USE_UV} == "true" ]]; then + echo + echo "${COLOR_BLUE}Installing uv version ${AIRFLOW_UV_VERSION}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then + # shellcheck disable=SC2086 + pip install ${EXTRA_INSTALL_FLAGS} --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}" + else + # shellcheck disable=SC2086 + pip install ${EXTRA_INSTALL_FLAGS} --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" + fi fi mkdir -p "${HOME}/.local/bin" } @@ -601,8 +651,6 @@ COPY <<"EOF" /install_from_docker_context_files.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - function install_airflow_and_providers_from_docker_context_files(){ if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,} @@ -619,7 +667,7 @@ function install_airflow_and_providers_from_docker_context_files(){ fi # shellcheck disable=SC2206 - local pip_flags=( + local packaging_flags=( # Don't quote this -- if it is empty we don't want it to create an # empty array element --find-links="file:///docker-context-files" @@ -669,7 +717,7 @@ function install_airflow_and_providers_from_docker_context_files(){ echo # force reinstall all airflow + provider packages with constraints found in set -x - pip install "${pip_flags[@]}" --root-user-action ignore --upgrade \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" --upgrade \ ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint "${local_constraints_file}" \ ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} set +x @@ -678,7 +726,7 @@ function install_airflow_and_providers_from_docker_context_files(){ echo "${COLOR_BLUE}Installing docker-context-files packages with constraints from GitHub${COLOR_RESET}" echo set -x - pip install "${pip_flags[@]}" --root-user-action ignore \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" \ ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} @@ -689,17 +737,16 @@ function install_airflow_and_providers_from_docker_context_files(){ echo "${COLOR_BLUE}Installing docker-context-files packages without constraints${COLOR_RESET}" echo set -x - pip install "${pip_flags[@]}" --root-user-action ignore \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} set +x fi - common::install_pip_version - pip check + common::install_packaging_tool + ${PACKAGING_TOOL} check } function install_all_other_packages_from_docker_context_files() { - echo echo "${COLOR_BLUE}Force re-installing all other package from local files without dependencies${COLOR_RESET}" echo @@ -709,22 +756,22 @@ function install_all_other_packages_from_docker_context_files() { grep -v apache_airflow | grep -v apache-airflow || true) if [[ -n "${reinstalling_other_packages}" ]]; then set -x - pip install ${ADDITIONAL_PIP_INSTALL_FLAGS} \ - --root-user-action ignore --force-reinstall --no-deps --no-index ${reinstalling_other_packages} - common::install_pip_version + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + --force-reinstall --no-deps --no-index ${reinstalling_other_packages} + common::install_packaging_tool set +x fi } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow_and_providers_from_docker_context_files -common::show_pip_version_and_location install_all_other_packages_from_docker_context_files EOF @@ -734,8 +781,6 @@ COPY <<"EOF" /install_airflow.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - function install_airflow() { # Coherence check for editable installation mode. if [[ ${AIRFLOW_INSTALLATION_METHOD} != "." && \ @@ -760,22 +805,21 @@ function install_airflow() { echo "${COLOR_BLUE}Remove airflow and all provider packages installed before potentially${COLOR_RESET}" echo set -x - pip freeze | grep apache-airflow | xargs pip uninstall --yes 2>/dev/null || true + ${PACKAGING_TOOL} freeze | grep apache-airflow | xargs ${PACKAGING_TOOL} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true set +x echo echo "${COLOR_BLUE}Installing all packages with eager upgrade with ${AIRFLOW_INSTALL_EDITABLE_FLAG} mode${COLOR_RESET}" echo set -x - pip install --root-user-action ignore \ - --upgrade --upgrade-strategy eager \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} set +x - common::install_pip_version + common::install_packaging_tool echo - echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" + echo "${COLOR_BLUE}Running '${PACKAGING_TOOL} check'${COLOR_RESET}" echo pip check else @@ -783,17 +827,17 @@ function install_airflow() { echo "${COLOR_BLUE}Installing all packages with constraints and upgrade if needed${COLOR_RESET}" echo set -x - pip install --root-user-action ignore ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true - common::install_pip_version + common::install_packaging_tool # then upgrade if needed without using constraints to account for new limits in pyproject.toml - pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_LOWEST_DIRECT_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -804,10 +848,11 @@ function install_airflow() { } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow EOF @@ -819,7 +864,6 @@ set -euo pipefail : "${UPGRADE_TO_NEWER_DEPENDENCIES:?Should be true or false}" : "${ADDITIONAL_PYTHON_DEPS:?Should be set}" -: "${AIRFLOW_PIP_VERSION:?Should be set}" . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" @@ -829,10 +873,10 @@ function install_additional_dependencies() { echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}" echo set -x - pip install --root-user-action ignore --upgrade --upgrade-strategy eager \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -843,10 +887,10 @@ function install_additional_dependencies() { echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}" echo set -x - pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} --upgrade "${RESOLUTION_LOWEST_DIRECT_FLAG}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -856,10 +900,11 @@ function install_additional_dependencies() { } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_additional_dependencies EOF @@ -1246,6 +1291,8 @@ ARG INSTALL_MYSQL_CLIENT_TYPE="mariadb" ARG INSTALL_MSSQL_CLIENT="true" ARG INSTALL_POSTGRES_CLIENT="true" ARG AIRFLOW_PIP_VERSION +ARG AIRFLOW_UV_VERSION +ARG AIRFLOW_USE_UV ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \ INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE} \ @@ -1327,6 +1374,8 @@ RUN if [[ -f /docker-context-files/pip.conf ]]; then \ ARG ADDITIONAL_PIP_INSTALL_FLAGS="" ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} \ AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ AIRFLOW_VERSION=${AIRFLOW_VERSION} \ AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD} \ @@ -1342,7 +1391,6 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \ DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \ PATH=${PATH}:${AIRFLOW_USER_HOME_DIR}/.local/bin \ - AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \ ADDITIONAL_PIP_INSTALL_FLAGS=${ADDITIONAL_PIP_INSTALL_FLAGS} \ AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \ @@ -1458,12 +1506,16 @@ LABEL org.apache.airflow.distro="debian" \ ARG PYTHON_BASE_IMAGE ARG AIRFLOW_PIP_VERSION +ARG AIRFLOW_UV_VERSION +ARG AIRFLOW_USE_UV ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \ # Make sure noninteractive debian install is used and language variables set DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 LD_LIBRARY_PATH=/usr/local/lib \ - AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} + AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} ARG RUNTIME_APT_DEPS="" ARG ADDITIONAL_RUNTIME_APT_DEPS="" @@ -1503,10 +1555,14 @@ ENV PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH}" \ AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \ AIRFLOW_HOME=${AIRFLOW_HOME} -# THE 3 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON +# THE 7 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON # AND SHOULD BE REMOVED WHEN WORKAROUND IN install_mssql.sh IS REMOVED ARG AIRFLOW_PIP_VERSION=24.0 -ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} +ARG AIRFLOW_UV_VERSION=0.1.10 +ARG AIRFLOW_USE_UV="false" +ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} COPY --from=scripts common.sh /scripts/docker/ # Only copy mysql/mssql installation scripts for now - so that changing the other diff --git a/Dockerfile.ci b/Dockerfile.ci index f5044a2ef9b4c..a0e141cecd9c5 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -296,7 +296,6 @@ set -euo pipefail common::get_colors declare -a packages -: "${AIRFLOW_PIP_VERSION:?Should be set}" : "${INSTALL_MSSQL_CLIENT:?Should be true or false}" @@ -376,14 +375,13 @@ COPY <<"EOF" /install_pip_version.sh #!/usr/bin/env bash . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location -common::install_pip_version +common::install_packaging_tool EOF # The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -396,7 +394,6 @@ COPY <<"EOF" /install_airflow_dependencies_from_branch_tip.sh : "${AIRFLOW_BRANCH:?Should be set}" : "${INSTALL_MYSQL_CLIENT:?Should be true or false}" : "${INSTALL_POSTGRES_CLIENT:?Should be true or false}" -: "${AIRFLOW_PIP_VERSION:?Should be set}" function install_airflow_dependencies_from_branch_tip() { echo @@ -412,26 +409,27 @@ function install_airflow_dependencies_from_branch_tip() { # dependencies that we can cache and reuse when installing airflow using constraints and latest # pyproject.toml in the next step (when we install regular airflow). set -x - pip install --root-user-action ignore \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "apache-airflow[${AIRFLOW_EXTRAS}] @ https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" - common::install_pip_version + common::install_packaging_tool # Uninstall airflow and providers to keep only the dependencies. In the future when # planned https://github.com/pypa/pip/issues/11440 is implemented in pip we might be able to use this # flag and skip the remove step. - pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true + ${PACKAGING_TOOL} freeze | grep apache-airflow-providers | xargs ${PACKAGING_TOOL} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true set +x echo echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now target airflow can be reinstalled using mostly cached dependencies${COLOR_RESET}" echo - pip uninstall --yes apache-airflow || true + ${PACKAGING_TOOL} uninstall ${EXTRA_UNINSTALL_FLAGS} apache-airflow || true } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow_dependencies_from_branch_tip EOF @@ -441,6 +439,10 @@ COPY <<"EOF" /common.sh #!/usr/bin/env bash set -euo pipefail +: "${AIRFLOW_PIP_VERSION:?Should be set}" +: "${AIRFLOW_UV_VERSION:?Should be set}" +: "${AIRFLOW_USE_UV:?Should be set}" + function common::get_colors() { COLOR_BLUE=$'\e[34m' COLOR_GREEN=$'\e[32m' @@ -454,6 +456,31 @@ function common::get_colors() { export COLOR_YELLOW } +function common::get_packaging_tool() { + if [[ ${AIRFLOW_USE_UV} == "true" ]]; then + echo + echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" + echo + export PACKAGING_TOOL="uv pip" + export EXTRA_INSTALL_FLAGS="" + export EXTRA_UNINSTALL_FLAGS="" + export RESOLUTION_HIGHEST_FLAG="--resolution highest" + export RESOLUTION_LOWEST_DIRECT_FLAG="--resolution lowest-direct" + # We need to lie about VIRTUAL_ENV to make uv works + # Until https://github.com/astral-sh/uv/issues/1396 is fixed + VIRTUAL_ENV=$(python -c "import sys; print(sys.prefix)") + export VIRTUAL_ENV + else + echo + echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}" + echo + export PACKAGING_TOOL="pip" + export EXTRA_INSTALL_FLAGS="--root-user-action ignore" + export EXTRA_UNINSTALL_FLAGS="--yes" + export RESOLUTION_HIGHEST_FLAG="--upgrade-strategy eager" + export RESOLUTION_LOWEST_DIRECT_FLAG="--upgrade --upgrade-strategy only-if-needed" + fi +} function common::get_airflow_version_specification() { if [[ -z ${AIRFLOW_VERSION_SPECIFICATION=} @@ -489,20 +516,41 @@ function common::get_constraints_location() { fi } -function common::show_pip_version_and_location() { +function common::show_packaging_tool_version_and_location() { echo "PATH=${PATH}" - echo "pip on path: $(which pip)" - echo "Using pip: $(pip --version)" + if [[ ${PACKAGING_TOOL} == "pip" ]]; then + echo "${COLOR_BLUE}Using 'pip' to install Airflow${COLOR_RESET}" + echo "pip on path: $(which pip)" + echo "Using pip: $(pip --version)" + else + echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" + echo "uv on path: $(which uv)" + echo "Using uv: $(uv --version)" + fi } -function common::install_pip_version() { +function common::install_packaging_tool() { echo echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" echo if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then - pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}" + # shellcheck disable=SC2086 + pip install ${EXTRA_INSTALL_FLAGS} --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}" else - pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + # shellcheck disable=SC2086 + pip install ${EXTRA_INSTALL_FLAGS} --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + fi + if [[ ${AIRFLOW_USE_UV} == "true" ]]; then + echo + echo "${COLOR_BLUE}Installing uv version ${AIRFLOW_UV_VERSION}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then + # shellcheck disable=SC2086 + pip install ${EXTRA_INSTALL_FLAGS} --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}" + else + # shellcheck disable=SC2086 + pip install ${EXTRA_INSTALL_FLAGS} --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" + fi fi mkdir -p "${HOME}/.local/bin" } @@ -548,7 +596,7 @@ function install_pipx_tools() { echo "${COLOR_BLUE}Installing pipx tools${COLOR_RESET}" echo # Make sure PIPX is installed in latest version - pip install --root-user-action ignore --upgrade "pipx>=1.2.1" + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} --upgrade "pipx>=1.2.1" if [[ $(uname -m) != "aarch64" ]]; then # Do not install mssql-cli for ARM # Install all the tools we need available in command line but without impacting the current environment @@ -562,6 +610,7 @@ function install_pipx_tools() { } common::get_colors +common::get_packaging_tool install_pipx_tools EOF @@ -572,8 +621,6 @@ COPY <<"EOF" /install_airflow.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -: "${AIRFLOW_PIP_VERSION:?Should be set}" - function install_airflow() { # Coherence check for editable installation mode. if [[ ${AIRFLOW_INSTALLATION_METHOD} != "." && \ @@ -598,22 +645,21 @@ function install_airflow() { echo "${COLOR_BLUE}Remove airflow and all provider packages installed before potentially${COLOR_RESET}" echo set -x - pip freeze | grep apache-airflow | xargs pip uninstall --yes 2>/dev/null || true + ${PACKAGING_TOOL} freeze | grep apache-airflow | xargs ${PACKAGING_TOOL} uninstall ${EXTRA_UNINSTALL_FLAGS} 2>/dev/null || true set +x echo echo "${COLOR_BLUE}Installing all packages with eager upgrade with ${AIRFLOW_INSTALL_EDITABLE_FLAG} mode${COLOR_RESET}" echo set -x - pip install --root-user-action ignore \ - --upgrade --upgrade-strategy eager \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} set +x - common::install_pip_version + common::install_packaging_tool echo - echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" + echo "${COLOR_BLUE}Running '${PACKAGING_TOOL} check'${COLOR_RESET}" echo pip check else @@ -621,17 +667,17 @@ function install_airflow() { echo "${COLOR_BLUE}Installing all packages with constraints and upgrade if needed${COLOR_RESET}" echo set -x - pip install --root-user-action ignore ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true - common::install_pip_version + common::install_packaging_tool # then upgrade if needed without using constraints to account for new limits in pyproject.toml - pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_LOWEST_DIRECT_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -642,10 +688,11 @@ function install_airflow() { } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_airflow EOF @@ -657,7 +704,6 @@ set -euo pipefail : "${UPGRADE_TO_NEWER_DEPENDENCIES:?Should be true or false}" : "${ADDITIONAL_PYTHON_DEPS:?Should be set}" -: "${AIRFLOW_PIP_VERSION:?Should be set}" . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" @@ -667,10 +713,10 @@ function install_additional_dependencies() { echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}" echo set -x - pip install --root-user-action ignore --upgrade --upgrade-strategy eager \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -681,10 +727,10 @@ function install_additional_dependencies() { echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}" echo set -x - pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \ + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} --upgrade "${RESOLUTION_LOWEST_DIRECT_FLAG}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} - common::install_pip_version + common::install_packaging_tool set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -694,10 +740,11 @@ function install_additional_dependencies() { } common::get_colors +common::get_packaging_tool common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -common::show_pip_version_and_location +common::show_packaging_tool_version_and_location install_additional_dependencies EOF @@ -893,8 +940,12 @@ function check_boto_upgrade() { echo echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" echo - pip uninstall --root-user-action ignore aiobotocore s3fs -y || true - pip install --root-user-action ignore --upgrade boto3 botocore + # shellcheck disable=SC2086 + ${PACKAGING_TOOL} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs || true + # We need to include oss2 as dependency as otherwise jmespath will be bumped and it will not pass + # the pip check test + # shellcheck disable=SC2086 + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore "oss2>=2.14.0" pip check } @@ -903,25 +954,31 @@ function check_pydantic() { echo echo "${COLOR_YELLOW}Reinstalling airflow from local sources to account for pyproject.toml changes${COLOR_RESET}" echo - pip install --root-user-action ignore -e . + # shellcheck disable=SC2086 + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} -e . echo echo "${COLOR_YELLOW}Remove pydantic and 3rd party libraries that depend on it${COLOR_RESET}" echo - pip uninstall --root-user-action ignore pydantic aws-sam-translator openai pyiceberg qdrant-client cfn-lint -y + # shellcheck disable=SC2086 + ${PACKAGING_TOOL} uninstall ${EXTRA_UNINSTALL_FLAGS} pydantic aws-sam-translator openai \ + pyiceberg qdrant-client cfn-lint weaviate-client pip check elif [[ ${PYDANTIC=} == "v1" ]]; then echo echo "${COLOR_YELLOW}Reinstalling airflow from local sources to account for pyproject.toml changes${COLOR_RESET}" echo - pip install --root-user-action ignore -e . + # shellcheck disable=SC2086 + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} -e . echo - echo "${COLOR_YELLOW}Uninstalling pyicberg which is not compatible with Pydantic 1${COLOR_RESET}" + echo "${COLOR_YELLOW}Uninstalling dependencies which are not compatible with Pydantic 1${COLOR_RESET}" echo - pip uninstall pyiceberg -y + # shellcheck disable=SC2086 + ${PACKAGING_TOOL} uninstall ${EXTRA_UNINSTALL_FLAGS} pyiceberg waeviate-client echo echo "${COLOR_YELLOW}Downgrading Pydantic to < 2${COLOR_RESET}" echo - pip install --upgrade "pydantic<2.0.0" + # shellcheck disable=SC2086 + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} --upgrade "pydantic<2.0.0" pip check else echo @@ -939,7 +996,8 @@ function check_download_sqlalchemy() { echo echo "${COLOR_BLUE}Downgrading sqlalchemy to minimum supported version: ${min_sqlalchemy_version}${COLOR_RESET}" echo - pip install --root-user-action ignore "sqlalchemy==${min_sqlalchemy_version}" + # shellcheck disable=SC2086 + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} "sqlalchemy==${min_sqlalchemy_version}" pip check } @@ -951,7 +1009,8 @@ function check_download_pendulum() { echo echo "${COLOR_BLUE}Downgrading pendulum to minimum supported version: ${min_pendulum_version}${COLOR_RESET}" echo - pip install --root-user-action ignore "pendulum==${min_pendulum_version}" + # shellcheck disable=SC2086 + ${PACKAGING_TOOL} install ${EXTRA_INSTALL_FLAGS} "pendulum==${min_pendulum_version}" pip check } @@ -1042,10 +1101,14 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \ COPY --from=scripts install_os_dependencies.sh /scripts/docker/ RUN bash /scripts/docker/install_os_dependencies.sh dev -# THE 3 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON +# THE 7 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON # AND SHOULD BE REMOVED WHEN WORKAROUND IN install_mssql.sh IS REMOVED ARG AIRFLOW_PIP_VERSION=24.0 -ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} +ARG AIRFLOW_UV_VERSION=0.1.10 +ARG AIRFLOW_USE_UV="true" +ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} COPY --from=scripts common.sh /scripts/docker/ # Only copy mysql/mssql installation scripts for now - so that changing the other @@ -1108,6 +1171,8 @@ ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" ARG AIRFLOW_CI_BUILD_EPOCH="10" ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" ARG AIRFLOW_PIP_VERSION=24.0 +ARG AIRFLOW_UV_VERSION=0.1.10 +ARG AIRFLOW_USE_UV="true" # Setup PIP # By default PIP install run without cache to make image smaller ARG PIP_NO_CACHE_DIR="true" @@ -1138,6 +1203,8 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ AIRFLOW_VERSION=${AIRFLOW_VERSION} \ AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ + AIRFLOW_USE_UV=${AIRFLOW_USE_UV} \ # In the CI image we always: # * install MySQL, MsSQL # * install airflow from current sources, not from PyPI package diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index 4c6fc2c016bb7..7bae55012566d 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -447,6 +447,8 @@ can be used for CI images: | `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | | `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | | `AIRFLOW_PIP_VERSION` | `24.0` | PIP version used. | +| `AIRFLOW_UV_VERSION` | `0.1.10` | UV version used. | +| `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | | `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | Here are some examples of how CI images can built manually. CI is always diff --git a/dev/breeze/doc/images/output_ci-image_build.svg b/dev/breeze/doc/images/output_ci-image_build.svg index 17ea15e9e999f..f617eb05e72aa 100644 --- a/dev/breeze/doc/images/output_ci-image_build.svg +++ b/dev/breeze/doc/images/output_ci-image_build.svg @@ -1,4 +1,4 @@ - +