From b41a9387dbb19a5c4aec03e8dae49ab0a9717607 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 4 Aug 2021 15:13:34 +0200 Subject: [PATCH 1/2] Remove CONTINUE_ON_PIP_CHECK_FAILURE parameter This parameter was useful when upgrading new dependencies, however it is going to be replaced with better approach in the upcoming image convention change. --- .github/workflows/build-images.yml | 1 - BREEZE.rst | 6 ------ Dockerfile | 5 +---- Dockerfile.ci | 2 -- IMAGES.rst | 6 ------ breeze | 9 --------- docs/docker-stack/build-arg-ref.rst | 6 ------ scripts/ci/libraries/_build_images.sh | 20 ------------------- scripts/ci/libraries/_initialization.sh | 4 ---- .../docker/install_additional_dependencies.sh | 5 ++--- scripts/docker/install_airflow.sh | 4 ++-- .../install_from_docker_context_files.sh | 2 +- 12 files changed, 6 insertions(+), 64 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index f29e199e4fb84..ec8f4354d0908 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -148,7 +148,6 @@ jobs: BACKEND: postgres PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - CONTINUE_ON_PIP_CHECK_FAILURE: "true" DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} CHECK_IF_BASE_PYTHON_IMAGE_UPDATED: > ${{ github.event_name == 'pull_request_target' && 'false' || 'true' }} diff --git a/BREEZE.rst b/BREEZE.rst index 90663657a85ad..66f94dff0da99 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1280,9 +1280,6 @@ This is the current syntax for `./breeze <./breeze>`_: --upgrade-to-newer-dependencies Upgrades PIP packages to latest versions available without looking at the constraints. - --continue-on-pip-check-failure - Continue even if 'pip check' fails. - -I, --production-image Use production image for entering the environment and builds (not for tests). @@ -2446,9 +2443,6 @@ This is the current syntax for `./breeze <./breeze>`_: --upgrade-to-newer-dependencies Upgrades PIP packages to latest versions available without looking at the constraints. - --continue-on-pip-check-failure - Continue even if 'pip check' fails. - **************************************************************************************************** Use different Airflow version at runtime in CI image diff --git a/Dockerfile b/Dockerfile index 9a7f8ecdc1fec..7e341f4ce1df3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -236,14 +236,11 @@ ARG INSTALL_FROM_PYPI="true" # * pyjwt<2.0.0: flask-jwt-extended requires it # * dill<0.3.3 required by apache-beam ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="pyjwt<2.0.0 dill<0.3.3 certifi<2021.0.0" -ARG CONTINUE_ON_PIP_CHECK_FAILURE="false" - ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \ INSTALL_FROM_DOCKER_CONTEXT_FILES=${INSTALL_FROM_DOCKER_CONTEXT_FILES} \ INSTALL_FROM_PYPI=${INSTALL_FROM_PYPI} \ - EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \ - CONTINUE_ON_PIP_CHECK_FAILURE=${CONTINUE_ON_PIP_CHECK_FAILURE} + EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} WORKDIR /opt/airflow diff --git a/Dockerfile.ci b/Dockerfile.ci index 552afd8f4d418..08f36412f269b 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -311,8 +311,6 @@ COPY setup.cfg ${AIRFLOW_SOURCES}/setup.cfg COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/__init__.py -ARG CONTINUE_ON_PIP_CHECK_FAILURE="false" - # The goal of this line is to install the dependencies from the most current setup.py from sources # This will be usually incremental small set of packages in CI optimized build, so it will be very fast # In non-CI optimized build this will install all dependencies before installing sources. diff --git a/IMAGES.rst b/IMAGES.rst index 82e69895f3842..bc34e6c6abe37 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -445,12 +445,6 @@ The following build arguments (``--build-arg`` in docker build command) can be u | | | upgraded to newer versions matching | | | | setup.py before installation. | +------------------------------------------+------------------------------------------+------------------------------------------+ -| ``CONTINUE_ON_PIP_CHECK_FAILURE`` | ``false`` | By default the image will fail if pip | -| | | check fails for it. This is good for | -| | | interactive building but on CI the | -| | | image should be built regardless - we | -| | | have a separate step to verify image. | -+------------------------------------------+------------------------------------------+------------------------------------------+ | ``AIRFLOW_PRE_CACHED_PIP_PACKAGES`` | ``true`` | Allows to pre-cache airflow PIP packages | | | | from the GitHub of Apache Airflow | | | | This allows to optimize iterations for | diff --git a/breeze b/breeze index 7aa42952fb7cb..420861eefb53d 100755 --- a/breeze +++ b/breeze @@ -1199,12 +1199,6 @@ function breeze::parse_arguments() { echo shift ;; - --continue-on-pip-check-failure) - export CONTINUE_ON_PIP_CHECK_FAILURE="true" - echo "Skip PIP check failure." - echo - shift - ;; --package-format) export PACKAGE_FORMAT="${2}" echo "Selected package type: ${PACKAGE_FORMAT}" @@ -2439,9 +2433,6 @@ ${FORMATTED_INSTALLATION_METHOD} --upgrade-to-newer-dependencies Upgrades PIP packages to latest versions available without looking at the constraints. ---continue-on-pip-check-failure - Continue even if 'pip check' fails. - " } diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index 8780970f613c5..f2507e05db423 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -79,12 +79,6 @@ for examples of using those arguments. +------------------------------------------+------------------------------------------+------------------------------------------+ | Build argument | Default value | Description | +==========================================+==========================================+==========================================+ -| ``CONTINUE_ON_PIP_CHECK_FAILURE`` | ``false`` | By default the image build fails if pip | -| | | check fails for it. This is good for | -| | | interactive building but on CI the | -| | | image should be built regardless - we | -| | | have a separate step to verify image. | -+------------------------------------------+------------------------------------------+------------------------------------------+ | ``UPGRADE_TO_NEWER_DEPENDENCIES`` | ``false`` | If set to true, the dependencies are | | | | upgraded to newer versions matching | | | | setup.py before installation. | diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index ca94e4cf316b6..4ead442d9d9de 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -669,7 +669,6 @@ Docker building ${AIRFLOW_CI_IMAGE}. --build-arg ADDITIONAL_RUNTIME_APT_DEPS="${ADDITIONAL_RUNTIME_APT_DEPS}" \ --build-arg ADDITIONAL_RUNTIME_APT_ENV="${ADDITIONAL_RUNTIME_APT_ENV}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ - --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg CONSTRAINTS_GITHUB_REPOSITORY="${CONSTRAINTS_GITHUB_REPOSITORY}" \ --build-arg AIRFLOW_CONSTRAINTS_REFERENCE="${DEFAULT_CONSTRAINTS_BRANCH}" \ --build-arg AIRFLOW_CONSTRAINTS="${AIRFLOW_CONSTRAINTS}" \ @@ -810,7 +809,6 @@ function build_images::build_prod_images() { --build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \ --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ - --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg BUILD_ID="${CI_BUILD_ID}" \ --build-arg COMMIT_SHA="${COMMIT_SHA}" \ --build-arg CONSTRAINTS_GITHUB_REPOSITORY="${CONSTRAINTS_GITHUB_REPOSITORY}" \ @@ -845,7 +843,6 @@ function build_images::build_prod_images() { --build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \ --build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \ --build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \ - --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ @@ -1014,23 +1011,6 @@ ${COLOR_BLUE} ./breeze build-image --production-image --upgrade-to-newer-dependencies --python 3.6 ${COLOR_RESET} -* If you want to build the image regardless if 'pip check' fails for it, you can add - --continue-on-pip-check-failure flag and enter the image and inspect dependencies. - -CI image: - -${COLOR_BLUE} - ./breeze build-image --upgrade-to-newer-dependencies --python 3.6 --continue-on-pip-check-failure - docker run -it apache/airflow:main-3.6-ci bash -${COLOR_RESET} - -Production image: - -${COLOR_BLUE} - ./breeze build-image --production-image --upgrade-to-newer-dependencies --python 3.6 --continue-on-pip-check-failure - docker run -it apache/airflow:main-3.6 bash -${COLOR_RESET} - * You will see error messages there telling which requirements are conflicting and which packages caused the conflict. Add the limitation that caused the conflict to EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS variable in Dockerfile.ci. Note that the limitations might be different for Dockerfile.ci and Dockerfile diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index fefcb1a5b4040..0dc7869e499f3 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -446,9 +446,6 @@ function initialization::initialize_image_build_variables() { # Installs different airflow version than current from the sources export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""} - # Continue on PIP CHECK failure - export CONTINUE_ON_PIP_CHECK_FAILURE=${CONTINUE_ON_PIP_CHECK_FAILURE:="false"} - # Determines if airflow should be installed from a specified reference in GitHub export INSTALL_AIRFLOW_REFERENCE=${INSTALL_AIRFLOW_REFERENCE:=""} @@ -684,7 +681,6 @@ Common image build variables: INSTALL_FROM_PYPI: '${INSTALL_FROM_PYPI}' AIRFLOW_PRE_CACHED_PIP_PACKAGES: '${AIRFLOW_PRE_CACHED_PIP_PACKAGES}' UPGRADE_TO_NEWER_DEPENDENCIES: '${UPGRADE_TO_NEWER_DEPENDENCIES}' - CONTINUE_ON_PIP_CHECK_FAILURE: '${CONTINUE_ON_PIP_CHECK_FAILURE}' CHECK_IMAGE_FOR_REBUILD: '${CHECK_IMAGE_FOR_REBUILD}' AIRFLOW_CONSTRAINTS_LOCATION: '${AIRFLOW_CONSTRAINTS_LOCATION}' AIRFLOW_CONSTRAINTS_REFERENCE: '${AIRFLOW_CONSTRAINTS_REFERENCE}' diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh index 6c035ae5def09..4f9c05f6b7680 100755 --- a/scripts/docker/install_additional_dependencies.sh +++ b/scripts/docker/install_additional_dependencies.sh @@ -23,7 +23,6 @@ test -v ADDITIONAL_PYTHON_DEPS test -v EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS test -v AIRFLOW_INSTALL_USER_FLAG test -v AIRFLOW_PIP_VERSION -test -v CONTINUE_ON_PIP_CHECK_FAILURE # shellcheck source=scripts/docker/common.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" @@ -41,7 +40,7 @@ function install_additional_dependencies() { ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check else echo echo Installing additional dependencies upgrading only if needed @@ -51,7 +50,7 @@ function install_additional_dependencies() { ${ADDITIONAL_PYTHON_DEPS} # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check fi } diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh index 49040275a04ac..e2bca4fc839a0 100755 --- a/scripts/docker/install_airflow.sh +++ b/scripts/docker/install_airflow.sh @@ -60,7 +60,7 @@ function install_airflow() { # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check else \ echo echo Installing all packages with constraints and upgrade if needed @@ -76,7 +76,7 @@ function install_airflow() { "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check fi } diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index 813d1b02df7e0..d8ed6bc72bd9a 100755 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -96,7 +96,7 @@ function install_airflow_and_providers_from_docker_context_files(){ # make sure correct PIP version is left installed pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" - pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE} + pip check } From 8d80d873cf98dcfd495e72888e3a989387cf35fe Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 4 Aug 2021 15:33:40 +0200 Subject: [PATCH 2/2] Optimizes structure of the Dockerfiles and use latest tools This PR optimizes the structure of Dockerfile by moving some expensive operations before the COPY sources so that rebuilding image when only few sources change is much faster. At the same time, we upgrade PIP and HELM chart used to latest versions and clean-up some parameter inconsistencies. --- BREEZE.rst | 4 +- Dockerfile | 22 ++++---- Dockerfile.ci | 50 ++++++++++--------- breeze-complete | 2 +- scripts/ci/libraries/_initialization.sh | 6 +-- ...l_airflow_dependencies_from_branch_tip.sh} | 11 ++-- 6 files changed, 49 insertions(+), 46 deletions(-) rename scripts/docker/{install_airflow_from_branch_tip.sh => install_airflow_dependencies_from_branch_tip.sh} (85%) diff --git a/BREEZE.rst b/BREEZE.rst index 66f94dff0da99..683d2fb320f98 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -2390,9 +2390,9 @@ This is the current syntax for `./breeze <./breeze>`_: Helm version - only used in case one of kind-cluster commands is used. One of: - v3.2.4 + v3.6.3 - Default: v3.2.4 + Default: v3.6.3 --executor EXECUTOR Executor to use in a kubernetes cluster. diff --git a/Dockerfile b/Dockerfile index 7e341f4ce1df3..782e5b4bbd9a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,8 @@ ARG AIRFLOW_GID="50000" ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" -ARG AIRFLOW_PIP_VERSION=21.1.2 +ARG AIRFLOW_PIP_VERSION=21.2.2 +ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" # By default PIP has progress bar but you can disable it. ARG PIP_PROGRESS_BAR="on" @@ -108,12 +109,13 @@ ARG DEV_APT_COMMAND="\ && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list" ARG ADDITIONAL_DEV_APT_COMMAND="echo" +ARG ADDITIONAL_DEV_APT_ENV="" ENV DEV_APT_DEPS=${DEV_APT_DEPS} \ ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \ DEV_APT_COMMAND=${DEV_APT_COMMAND} \ ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND} \ - ADDITIONAL_DEV_APT_ENV="" + ADDITIONAL_DEV_APT_ENV=${ADDITIONAL_DEV_APT_ENV} # Note missing man directories on debian-buster # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 @@ -216,7 +218,7 @@ ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ RUN bash /scripts/docker/install_pip_version.sh; \ if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ - bash /scripts/docker/install_airflow_from_branch_tip.sh; \ + bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi COPY ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO} @@ -273,7 +275,7 @@ RUN if [[ -f /docker-context-files/requirements.txt ]]; then \ ARG BUILD_ID ARG COMMIT_SHA -ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" +ARG AIRFLOW_IMAGE_REPOSITORY ARG AIRFLOW_IMAGE_DATE_CREATED ENV BUILD_ID=${BUILD_ID} COMMIT_SHA=${COMMIT_SHA} @@ -290,15 +292,14 @@ LABEL org.apache.airflow.distro="debian" \ org.opencontainers.image.created=${AIRFLOW_IMAGE_DATE_CREATED} \ org.opencontainers.image.authors="dev@airflow.apache.org" \ org.opencontainers.image.url="https://airflow.apache.org" \ - org.opencontainers.image.documentation="https://airflow.apache.org/docs/apache-airflow/stable/production-deployment.html" \ - org.opencontainers.image.source="https://github.com/apache/airflow" \ + org.opencontainers.image.documentation="https://airflow.apache.org/docs/docker-stack/index.html" \ org.opencontainers.image.version="${AIRFLOW_VERSION}" \ org.opencontainers.image.revision="${COMMIT_SHA}" \ org.opencontainers.image.vendor="Apache Software Foundation" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.ref.name="airflow-build-image" \ org.opencontainers.image.title="Build Image Segment for Production Airflow Image" \ - org.opencontainers.image.description="Installed Apache Airflow with build-time dependencies" + org.opencontainers.image.description="Reference build-time dependencies image for production-ready Apache Airflow image" ############################################################################################## # This is the actual Airflow image - much smaller than the build one. We copy @@ -376,7 +377,7 @@ ARG AIRFLOW_HOME ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow" ARG BUILD_ID ARG COMMIT_SHA -ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" +ARG AIRFLOW_IMAGE_REPOSITORY ARG AIRFLOW_IMAGE_DATE_CREATED # By default PIP will install everything in ~/.local ARG PIP_USER="true" @@ -465,15 +466,14 @@ LABEL org.apache.airflow.distro="debian" \ org.opencontainers.image.created=${AIRFLOW_IMAGE_DATE_CREATED} \ org.opencontainers.image.authors="dev@airflow.apache.org" \ org.opencontainers.image.url="https://airflow.apache.org" \ - org.opencontainers.image.documentation="https://airflow.apache.org/docs/apache-airflow/stable/production-deployment.html" \ - org.opencontainers.image.source="https://github.com/apache/airflow" \ + org.opencontainers.image.documentation="https://airflow.apache.org/docs/docker-stack/index.html" \ org.opencontainers.image.version="${AIRFLOW_VERSION}" \ org.opencontainers.image.revision="${COMMIT_SHA}" \ org.opencontainers.image.vendor="Apache Software Foundation" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.ref.name="airflow" \ org.opencontainers.image.title="Production Airflow Image" \ - org.opencontainers.image.description="Installed Apache Airflow" + org.opencontainers.image.description="Reference, production-ready Apache Airflow image" ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] diff --git a/Dockerfile.ci b/Dockerfile.ci index 08f36412f269b..13e9107137576 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -22,6 +22,8 @@ SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" ARG AIRFLOW_VERSION="2.2.0.dev0" +ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" + # By increasing this number we can do force build of all dependencies ARG DEPENDENCIES_EPOCH_NUMBER="6" @@ -126,6 +128,15 @@ ARG RUNTIME_APT_DEPS="\ unzip \ vim \ xxd" + +# Install Helm +ARG HELM_VERSION="v3.6.3" + +RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \ + && HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz" \ + && curl --location "${HELM_URL}" | tar -xvz -O "${SYSTEM}"-amd64/helm > /usr/local/bin/helm \ + && chmod +x /usr/local/bin/helm + ARG ADDITIONAL_RUNTIME_APT_DEPS="" ARG RUNTIME_APT_COMMAND="" ARG ADDITIONAL_RUNTIME_APT_COMMAND="" @@ -219,7 +230,7 @@ ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" # By default in the image, we are installing all providers when installing from sources ARG INSTALL_PROVIDERS_FROM_SOURCES="true" ARG INSTALL_FROM_PYPI="true" -ARG AIRFLOW_PIP_VERSION=21.1.2 +ARG AIRFLOW_PIP_VERSION=21.2.2 # Setup PIP # By default PIP install run without cache to make image smaller ARG PIP_NO_CACHE_DIR="true" @@ -282,7 +293,7 @@ ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENT RUN bash /scripts/docker/install_pip_version.sh; \ if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ - bash /scripts/docker/install_airflow_from_branch_tip.sh; \ + bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi # Generate random hex dump file so that we can determine whether it's faster to rebuild the image @@ -335,11 +346,13 @@ RUN chmod a+x /entrypoint COPY scripts/docker/load.bash /opt/bats/lib/ -# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not -# copying over stuff that is accidentally generated or that we do not need (such as egg-info) -# if you want to add something that is missing and you expect to see it in the image you can -# add it with ! in .dockerignore next to the airflow, test etc. directories there -COPY . ${AIRFLOW_SOURCES}/ +# Additional python deps to install +ARG ADDITIONAL_PYTHON_DEPS="" + +RUN bash /scripts/docker/install_pip_version.sh; \ + if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ + bash /scripts/docker/install_additional_dependencies.sh; \ + fi # Install autocomplete for airflow RUN if command -v airflow; then \ @@ -349,27 +362,16 @@ RUN if command -v airflow; then \ # Install autocomplete for Kubectl RUN echo "source /etc/bash_completion" >> ~/.bashrc -WORKDIR ${AIRFLOW_SOURCES} - -# Install Helm -ARG HELM_VERSION="v3.2.4" - -RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \ - && HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz" \ - && curl --location "${HELM_URL}" | tar -xvz -O "${SYSTEM}"-amd64/helm > /usr/local/bin/helm \ - && chmod +x /usr/local/bin/helm - -# Additional python deps to install -ARG ADDITIONAL_PYTHON_DEPS="" +# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not +# copying over stuff that is accidentally generated or that we do not need (such as egg-info) +# if you want to add something that is missing and you expect to see it in the image you can +# add it with ! in .dockerignore next to the airflow, test etc. directories there +COPY . ${AIRFLOW_SOURCES}/ -RUN bash /scripts/docker/install_pip_version.sh; \ - if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ - bash /scripts/docker/install_additional_dependencies.sh; \ - fi +WORKDIR ${AIRFLOW_SOURCES} ARG BUILD_ID ARG COMMIT_SHA -ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" ARG AIRFLOW_IMAGE_DATE_CREATED ENV PATH="/files/bin/:/opt/airflow/scripts/in_container/bin/:${HOME}:${PATH}" \ diff --git a/breeze-complete b/breeze-complete index e5b5d3f254d7f..01efc7c79cc6c 100644 --- a/breeze-complete +++ b/breeze-complete @@ -29,7 +29,7 @@ _breeze_allowed_integrations="cassandra kerberos mongo openldap pinot rabbitmq r _breeze_allowed_generate_constraints_modes="source-providers pypi-providers no-providers" _breeze_allowed_kubernetes_modes="image" _breeze_allowed_kubernetes_versions="v1.20.2 v1.19.7 v1.18.15" -_breeze_allowed_helm_versions="v3.2.4" +_breeze_allowed_helm_versions="v3.6.3" _breeze_allowed_kind_versions="v0.11.1" _breeze_allowed_mysql_versions="5.7 8" _breeze_allowed_mssql_versions="2017-latest 2019-latest" diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 0dc7869e499f3..223d9ea999343 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -221,7 +221,7 @@ function initialization::initialize_files_for_rebuild_check() { "scripts/docker/common.sh" "scripts/docker/install_additional_dependencies.sh" "scripts/docker/install_airflow.sh" - "scripts/docker/install_airflow_from_branch_tip.sh" + "scripts/docker/install_airflow_dependencies_from_branch_tip.sh" "scripts/docker/install_from_docker_context_files.sh" "scripts/docker/install_mysql.sh" "airflow/www/package.json" @@ -422,7 +422,7 @@ function initialization::initialize_image_build_variables() { export INSTALLED_PROVIDERS export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv" - AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.1"} + AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.2.2"} export AIRFLOW_PIP_VERSION # We also pin version of wheel used to get consistent builds @@ -500,7 +500,7 @@ function initialization::initialize_kubernetes_variables() { CURRENT_KIND_VERSIONS+=("v0.11.1") export CURRENT_KIND_VERSIONS # Currently supported versions of Helm - CURRENT_HELM_VERSIONS+=("v3.2.4") + CURRENT_HELM_VERSIONS+=("v3.6.3") export CURRENT_HELM_VERSIONS # Current executor in chart CURRENT_EXECUTOR+=("KubernetesExecutor") diff --git a/scripts/docker/install_airflow_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh similarity index 85% rename from scripts/docker/install_airflow_from_branch_tip.sh rename to scripts/docker/install_airflow_dependencies_from_branch_tip.sh index 925a872fa50ea..61aaa13ef467f 100755 --- a/scripts/docker/install_airflow_from_branch_tip.sh +++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -30,28 +30,29 @@ . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" -function install_airflow_from_branch_tip() { +function install_airflow_dependencies_from_branch_tip() { echo echo "Installing airflow from ${AIRFLOW_BRANCH}. It is used to cache dependencies" echo if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,} fi - # Install latest set of dependencies using constraints + # Install latest set of dependencies using constraints. In case constraints were upgraded and there + # are conflicts, this might fail, but it should be fixed in the following installation steps pip install ${AIRFLOW_INSTALL_USER_FLAG} \ "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" + --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true # make sure correct PIP version is used pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes || true echo echo Uninstalling just airflow. Dependencies remain. echo - pip uninstall --yes apache-airflow + pip uninstall --yes apache-airflow || true } common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location -install_airflow_from_branch_tip +install_airflow_dependencies_from_branch_tip