diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index f29e199e4fb84..ec8f4354d0908 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -148,7 +148,6 @@ jobs: BACKEND: postgres PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - CONTINUE_ON_PIP_CHECK_FAILURE: "true" DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} CHECK_IF_BASE_PYTHON_IMAGE_UPDATED: > ${{ github.event_name == 'pull_request_target' && 'false' || 'true' }} diff --git a/BREEZE.rst b/BREEZE.rst index 90663657a85ad..66f94dff0da99 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1280,9 +1280,6 @@ This is the current syntax for `./breeze <./breeze>`_: --upgrade-to-newer-dependencies Upgrades PIP packages to latest versions available without looking at the constraints. - --continue-on-pip-check-failure - Continue even if 'pip check' fails. - -I, --production-image Use production image for entering the environment and builds (not for tests). @@ -2446,9 +2443,6 @@ This is the current syntax for `./breeze <./breeze>`_: --upgrade-to-newer-dependencies Upgrades PIP packages to latest versions available without looking at the constraints. - --continue-on-pip-check-failure - Continue even if 'pip check' fails. - **************************************************************************************************** Use different Airflow version at runtime in CI image diff --git a/Dockerfile b/Dockerfile index 9a7f8ecdc1fec..7e341f4ce1df3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -236,14 +236,11 @@ ARG INSTALL_FROM_PYPI="true" # * pyjwt<2.0.0: flask-jwt-extended requires it # * dill<0.3.3 required by apache-beam ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="pyjwt<2.0.0 dill<0.3.3 certifi<2021.0.0" -ARG CONTINUE_ON_PIP_CHECK_FAILURE="false" - ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \ INSTALL_FROM_DOCKER_CONTEXT_FILES=${INSTALL_FROM_DOCKER_CONTEXT_FILES} \ INSTALL_FROM_PYPI=${INSTALL_FROM_PYPI} \ - EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \ - CONTINUE_ON_PIP_CHECK_FAILURE=${CONTINUE_ON_PIP_CHECK_FAILURE} + EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} WORKDIR /opt/airflow diff --git a/Dockerfile.ci b/Dockerfile.ci index 552afd8f4d418..08f36412f269b 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -311,8 +311,6 @@ COPY setup.cfg ${AIRFLOW_SOURCES}/setup.cfg COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/__init__.py -ARG CONTINUE_ON_PIP_CHECK_FAILURE="false" - # The goal of this line is to install the dependencies from the most current setup.py from sources # This will be usually incremental small set of packages in CI optimized build, so it will be very fast # In non-CI optimized build this will install all dependencies before installing sources. diff --git a/IMAGES.rst b/IMAGES.rst index 82e69895f3842..bc34e6c6abe37 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -445,12 +445,6 @@ The following build arguments (``--build-arg`` in docker build command) can be u | | | upgraded to newer versions matching | | | | setup.py before installation. | +------------------------------------------+------------------------------------------+------------------------------------------+ -| ``CONTINUE_ON_PIP_CHECK_FAILURE`` | ``false`` | By default the image will fail if pip | -| | | check fails for it. This is good for | -| | | interactive building but on CI the | -| | | image should be built regardless - we | -| | | have a separate step to verify image. | -+------------------------------------------+------------------------------------------+------------------------------------------+ | ``AIRFLOW_PRE_CACHED_PIP_PACKAGES`` | ``true`` | Allows to pre-cache airflow PIP packages | | | | from the GitHub of Apache Airflow | | | | This allows to optimize iterations for | diff --git a/breeze b/breeze index 7aa42952fb7cb..420861eefb53d 100755 --- a/breeze +++ b/breeze @@ -1199,12 +1199,6 @@ function breeze::parse_arguments() { echo shift ;; - --continue-on-pip-check-failure) - export CONTINUE_ON_PIP_CHECK_FAILURE="true" - echo "Skip PIP check failure." - echo - shift - ;; --package-format) export PACKAGE_FORMAT="${2}" echo "Selected package type: ${PACKAGE_FORMAT}" @@ -2439,9 +2433,6 @@ ${FORMATTED_INSTALLATION_METHOD} --upgrade-to-newer-dependencies Upgrades PIP packages to latest versions available without looking at the constraints. ---continue-on-pip-check-failure - Continue even if 'pip check' fails. - " } diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index 8780970f613c5..f2507e05db423 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -79,12 +79,6 @@ for examples of using those arguments. +------------------------------------------+------------------------------------------+------------------------------------------+ | Build argument | Default value | Description | +==========================================+==========================================+==========================================+ -| ``CONTINUE_ON_PIP_CHECK_FAILURE`` | ``false`` | By default the image build fails if pip | -| | | check fails for it. This is good for | -| | | interactive building but on CI the | -| | | image should be built regardless - we | -| | | have a separate step to verify image. | -+------------------------------------------+------------------------------------------+------------------------------------------+ | ``UPGRADE_TO_NEWER_DEPENDENCIES`` | ``false`` | If set to true, the dependencies are | | | | upgraded to newer versions matching | | | | setup.py before installation. | diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index ca94e4cf316b6..4ead442d9d9de 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -669,7 +669,6 @@ Docker building ${AIRFLOW_CI_IMAGE}. --build-arg ADDITIONAL_RUNTIME_APT_DEPS="${ADDITIONAL_RUNTIME_APT_DEPS}" \ --build-arg ADDITIONAL_RUNTIME_APT_ENV="${ADDITIONAL_RUNTIME_APT_ENV}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ - --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg CONSTRAINTS_GITHUB_REPOSITORY="${CONSTRAINTS_GITHUB_REPOSITORY}" \ --build-arg AIRFLOW_CONSTRAINTS_REFERENCE="${DEFAULT_CONSTRAINTS_BRANCH}" \ --build-arg AIRFLOW_CONSTRAINTS="${AIRFLOW_CONSTRAINTS}" \ @@ -810,7 +809,6 @@ function build_images::build_prod_images() { --build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \ --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ - --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg BUILD_ID="${CI_BUILD_ID}" \ --build-arg COMMIT_SHA="${COMMIT_SHA}" \ --build-arg CONSTRAINTS_GITHUB_REPOSITORY="${CONSTRAINTS_GITHUB_REPOSITORY}" \ @@ -845,7 +843,6 @@ function build_images::build_prod_images() { --build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \ --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ - --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ @@ -1014,23 +1011,6 @@ ${COLOR_BLUE} ./breeze build-image --production-image --upgrade-to-newer-dependencies --python 3.6 ${COLOR_RESET} -* If you want to build the image regardless if 'pip check' fails for it, you can add - --continue-on-pip-check-failure flag and enter the image and inspect dependencies. - -CI image: - -${COLOR_BLUE} - ./breeze build-image --upgrade-to-newer-dependencies --python 3.6 --continue-on-pip-check-failure - docker run -it apache/airflow:main-3.6-ci bash -${COLOR_RESET} - -Production image: - -${COLOR_BLUE} - ./breeze build-image --production-image --upgrade-to-newer-dependencies --python 3.6 --continue-on-pip-check-failure - docker run -it apache/airflow:main-3.6 bash -${COLOR_RESET} - * You will see error messages there telling which requirements are conflicting and which packages caused the conflict. Add the limitation that caused the conflict to EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS variable in Dockerfile.ci. Note that the limitations might be different for Dockerfile.ci and Dockerfile diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index fefcb1a5b4040..0dc7869e499f3 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -446,9 +446,6 @@ function initialization::initialize_image_build_variables() { # Installs different airflow version than current from the sources export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""} - # Continue on PIP CHECK failure - export CONTINUE_ON_PIP_CHECK_FAILURE=${CONTINUE_ON_PIP_CHECK_FAILURE:="false"} - # Determines if airflow should be installed from a specified reference in GitHub export INSTALL_AIRFLOW_REFERENCE=${INSTALL_AIRFLOW_REFERENCE:=""} @@ -684,7 +681,6 @@ Common image build variables: INSTALL_FROM_PYPI: '${INSTALL_FROM_PYPI}' AIRFLOW_PRE_CACHED_PIP_PACKAGES: '${AIRFLOW_PRE_CACHED_PIP_PACKAGES}' UPGRADE_TO_NEWER_DEPENDENCIES: '${UPGRADE_TO_NEWER_DEPENDENCIES}' - CONTINUE_ON_PIP_CHECK_FAILURE: '${CONTINUE_ON_PIP_CHECK_FAILURE}' CHECK_IMAGE_FOR_REBUILD: '${CHECK_IMAGE_FOR_REBUILD}' AIRFLOW_CONSTRAINTS_LOCATION: '${AIRFLOW_CONSTRAINTS_LOCATION}' AIRFLOW_CONSTRAINTS_REFERENCE: '${AIRFLOW_CONSTRAINTS_REFERENCE}' diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh index 6c035ae5def09..4f9c05f6b7680 100755 --- a/scripts/docker/install_additional_dependencies.sh +++ b/scripts/docker/install_additional_dependencies.sh @@ -23,7 +23,6 @@ test -v ADDITIONAL_PYTHON_DEPS test -v EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS test -v AIRFLOW_INSTALL_USER_FLAG test -v AIRFLOW_PIP_VERSION -test -v CONTINUE_ON_PIP_CHECK_FAILURE # shellcheck source=scripts/docker/common.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" @@ -41,7 +40,7 @@ function install_additional_dependencies() { ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check else echo echo Installing additional dependencies upgrading only if needed @@ -51,7 +50,7 @@ function install_additional_dependencies() { ${ADDITIONAL_PYTHON_DEPS} # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check fi } diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh index 49040275a04ac..e2bca4fc839a0 100755 --- a/scripts/docker/install_airflow.sh +++ b/scripts/docker/install_airflow.sh @@ -60,7 +60,7 @@ function install_airflow() { # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check else \ echo echo Installing all packages with constraints and upgrade if needed @@ -76,7 +76,7 @@ function install_airflow() { "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check fi } diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index 813d1b02df7e0..d8ed6bc72bd9a 100755 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -96,7 +96,7 @@ function install_airflow_and_providers_from_docker_context_files(){ # make sure correct PIP version is left installed pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check }