diff --git a/Dockerfile b/Dockerfile index 5eaaff424e076..a325091f4a27a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,35 +47,31 @@ ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" ARG AIRFLOW_PIP_VERSION=21.3.1 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" +# By default latest released version of airflow is installed (when empty) but this value can be overridden +# and we can install version according to specification (For example ==2.0.2 or <3.0.0). +ARG AIRFLOW_VERSION_SPECIFICATION="" + # By default PIP has progress bar but you can disable it. ARG PIP_PROGRESS_BAR="on" - ############################################################################################## # This is the build image where we build all dependencies ############################################################################################## FROM ${PYTHON_BASE_IMAGE} as airflow-build-image -SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] + +# Nolog bash flag is currently ignored - but you can replace it with +# xtrace - to show commands executed) +SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] ARG PYTHON_BASE_IMAGE ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \ DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 -# Install curl and gnupg2 - needed for many other installation steps -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - curl \ - gnupg2 \ - && apt-get autoremove -yqq --purge \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - ARG DEV_APT_DEPS="\ apt-transport-https \ apt-utils \ build-essential \ ca-certificates \ - gnupg \ dirmngr \ freetds-bin \ freetds-dev \ @@ -103,10 +99,13 @@ ARG DEV_APT_DEPS="\ unixodbc \ unixodbc-dev \ yarn" + ARG ADDITIONAL_DEV_APT_DEPS="" ARG DEV_APT_COMMAND="\ - curl --fail --location https://deb.nodesource.com/setup_14.x | bash - \ - && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ + curl --silent --fail --location https://deb.nodesource.com/setup_14.x | \ + bash -o pipefail -o errexit -o nolog - \ + && curl --silent https://dl.yarnpkg.com/debian/pubkey.gpg | \ + apt-key add - >/dev/null 2>&1\ && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list" ARG ADDITIONAL_DEV_APT_COMMAND="echo" ARG ADDITIONAL_DEV_APT_ENV="" @@ -120,11 +119,14 @@ ENV DEV_APT_DEPS=${DEV_APT_DEPS} \ # Note missing man directories on debian-buster # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 # Install basic and additional apt dependencies -RUN mkdir -pv /usr/share/man/man1 \ +RUN apt-get update \ + && apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 \ + && apt-get install -y --no-install-recommends curl gnupg2 \ + && mkdir -pv /usr/share/man/man1 \ && mkdir -pv /usr/share/man/man7 \ && export ${ADDITIONAL_DEV_APT_ENV?} \ - && bash -o pipefail -e -u -x -c "${DEV_APT_COMMAND}" \ - && bash -o pipefail -e -u -x -c "${ADDITIONAL_DEV_APT_COMMAND}" \ + && bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}" \ + && bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_DEV_APT_COMMAND}" \ && apt-get update \ && apt-get install -y --no-install-recommends \ ${DEV_APT_DEPS} \ @@ -153,6 +155,9 @@ ARG PIP_PROGRESS_BAR ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="false" # This is airflow version that is put in the label of the image build ARG AIRFLOW_VERSION +# By default latest released version of airflow is installed (when empty) but this value can be overridden +# and we can install version according to specification (For example ==2.0.2 or <3.0.0). +ARG AIRFLOW_VERSION_SPECIFICATION # By default we install providers from PyPI but in case of Breeze build we want to install providers # from local sources without the need of preparing provider packages upfront. This value is # automatically overridden by Breeze scripts. @@ -163,9 +168,6 @@ ARG INSTALL_PROVIDERS_FROM_SOURCES="false" # Airflow checked out together with the Dockerfile and AIRFLOW_SOURCES_FROM and AIRFLOW_SOURCES_TO # set to "." and "/opt/airflow" respectively. ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow" -# By default latest released version of airflow is installed (when empty) but this value can be overridden -# and we can install version according to specification (For example ==2.0.2 or <3.0.0). -ARG AIRFLOW_VERSION_SPECIFICATION="" # By default we do not upgrade to latest dependencies ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" # By default we install latest airflow from PyPI so we do not need to copy sources of Airflow @@ -201,8 +203,8 @@ ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \ COPY scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh /scripts/docker/ -RUN bash ./scripts/docker/install_mysql.sh dev \ - && bash ./scripts/docker/install_mssql.sh +RUN bash -o pipefail -o errexit -o nounset -o nolog ./scripts/docker/install_mysql.sh dev \ + && bash -o pipefail -o errexit -o nounset -o nolog ./scripts/docker/install_mssql.sh ENV PATH=${PATH}:/opt/mssql-tools/bin COPY docker-context-files /docker-context-files @@ -237,10 +239,11 @@ ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ # the cache is only used when "upgrade to newer dependencies" is not set to automatically # account for removed dependencies (we do not install them in the first place) # Upgrade to specific PIP version -RUN bash /scripts/docker/install_pip_version.sh; \ +RUN bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_pip_version.sh; \ if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ - bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ + bash -o pipefail -o errexit -o nounset -o nolog \ + /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi COPY --chown=airflow:0 ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO} @@ -272,15 +275,15 @@ WORKDIR /opt/airflow RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \ # only compile assets if the prod image is build from sources # otherwise they are already compiled-in - bash /scripts/docker/compile_www_assets.sh; \ + bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/compile_www_assets.sh; \ fi; \ if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \ - bash /scripts/docker/install_from_docker_context_files.sh; \ + bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_from_docker_context_files.sh; \ elif [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \ - bash /scripts/docker/install_airflow.sh; \ + bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_airflow.sh; \ fi; \ if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ - bash /scripts/docker/install_additional_dependencies.sh; \ + bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_additional_dependencies.sh; \ fi; \ find "${AIRFLOW_USER_HOME_DIR}/.local/" -name '*.pyc' -print0 | xargs -0 rm -f || true ; \ find "${AIRFLOW_USER_HOME_DIR}/.local/" -type d -name '__pycache__' -print0 | xargs -0 rm -rf || true ; \ @@ -328,7 +331,10 @@ LABEL org.apache.airflow.distro="debian" \ # installed Airflow and all it's dependencies from the build image to make it smaller. ############################################################################################## FROM ${PYTHON_BASE_IMAGE} as main -SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] + +# Nolog bash flag is currently ignored - but you can replace it with other flags (for example +# xtrace - to show commands executed) +SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] ARG AIRFLOW_UID @@ -350,15 +356,6 @@ ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \ AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} -# Install curl and gnupg2 - needed for many other installation steps -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - curl \ - gnupg2 \ - && apt-get autoremove -yqq --purge \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - ARG RUNTIME_APT_DEPS="\ apt-transport-https \ apt-utils \ @@ -366,7 +363,6 @@ ARG RUNTIME_APT_DEPS="\ curl \ dumb-init \ freetds-bin \ - gnupg \ gosu \ krb5-user \ ldap-utils \ @@ -400,6 +396,7 @@ ARG BUILD_ID ARG COMMIT_SHA ARG AIRFLOW_IMAGE_REPOSITORY ARG AIRFLOW_IMAGE_DATE_CREATED +ARG AIRFLOW_VERSION_SPECIFICATION ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \ ADDITIONAL_RUNTIME_APT_DEPS=${ADDITIONAL_RUNTIME_APT_DEPS} \ @@ -414,6 +411,7 @@ ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \ PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH}" \ GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm" \ AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD} \ + AIRFLOW_VERSION_SPECIFICATION=${AIRFLOW_VERSION_SPECIFICATION} \ BUILD_ID=${BUILD_ID} \ COMMIT_SHA=${COMMIT_SHA} \ # By default PIP installs everything to ~/.local @@ -422,11 +420,14 @@ ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \ # Note missing man directories on debian-buster # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 # Install basic and additional apt dependencies -RUN mkdir -pv /usr/share/man/man1 \ +RUN apt-get update \ + && apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 \ + && apt-get install -y --no-install-recommends curl gnupg2 \ + && mkdir -pv /usr/share/man/man1 \ && mkdir -pv /usr/share/man/man7 \ && export ${ADDITIONAL_RUNTIME_APT_ENV?} \ - && bash -o pipefail -e -u -x -c "${RUNTIME_APT_COMMAND}" \ - && bash -o pipefail -e -u -x -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" \ + && bash -o pipefail -o errexit -o nounset -o nolog -c "${RUNTIME_APT_COMMAND}" \ + && bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" \ && apt-get update \ && apt-get install -y --no-install-recommends \ ${RUNTIME_APT_DEPS} \ @@ -437,14 +438,15 @@ RUN mkdir -pv /usr/share/man/man1 \ # Only copy install_m(y/s)sql and install_pip_version.sh. We do not need any other scripts in the final image. COPY scripts/docker/install_mysql.sh /scripts/docker/install_mssql.sh scripts/docker/install_pip_version.sh \ - /scripts/docker/ + scripts/docker/common.sh /scripts/docker/ # fix permission issue in Azure DevOps when running the scripts RUN chmod a+x /scripts/docker/install_mysql.sh && \ /scripts/docker/install_mysql.sh prod && \ chmod a+x /scripts/docker/install_mssql.sh && \ /scripts/docker/install_mssql.sh && \ - adduser --quiet "airflow" --uid "${AIRFLOW_UID}" --gid "0" --home "${AIRFLOW_USER_HOME_DIR}" && \ + adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \ + --quiet "airflow" --uid "${AIRFLOW_UID}" --gid "0" --home "${AIRFLOW_USER_HOME_DIR}" && \ # Make Airflow files belong to the root group and are accessible. This is to accommodate the guidelines from # OpenShift https://docs.openshift.com/enterprise/3.0/creating_images/guidelines.html mkdir -pv "${AIRFLOW_HOME}"; \ @@ -463,8 +465,7 @@ COPY --chown=airflow:0 scripts/in_container/prod/clean-logs.sh /clean-logs # See https://github.com/apache/airflow/issues/9248 RUN chmod a+x /entrypoint /clean-logs && \ - chmod g=u /etc/passwd && \ - bash /scripts/docker/install_pip_version.sh + chmod g=u /etc/passwd WORKDIR ${AIRFLOW_HOME} @@ -474,6 +475,8 @@ RUN usermod -g 0 airflow -G 0 USER ${AIRFLOW_UID} +RUN /scripts/docker/install_pip_version.sh + LABEL org.apache.airflow.distro="debian" \ org.apache.airflow.distro.version="buster" \ org.apache.airflow.module="airflow" \ diff --git a/Dockerfile.ci b/Dockerfile.ci index 0d956c25b9c4a..62a61f5b94453 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -18,7 +18,9 @@ ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" FROM ${PYTHON_BASE_IMAGE} as main -SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"] +# Nolog bash flag is currently ignored - but you can replace it with other flags (for example +# xtrace - to show commands executed) +SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" ARG AIRFLOW_VERSION="2.2.0.dev0" @@ -31,24 +33,17 @@ ARG DEPENDENCIES_EPOCH_NUMBER="6" ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} AIRFLOW_VERSION=${AIRFLOW_VERSION} \ DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \ - DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER} + DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER} \ + INSTALL_MYSQL_CLIENT="true" \ + INSTALL_MSSQL_CLIENT="true" # Print versions RUN echo "Base image: ${PYTHON_BASE_IMAGE}, Airflow version: ${AIRFLOW_VERSION}" -# Install curl and gnupg2 - needed to download nodejs in the next step -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - curl \ - gnupg2 \ - && apt-get autoremove -yqq --purge \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - ARG ADDITIONAL_DEV_APT_DEPS="" ARG DEV_APT_COMMAND="\ - curl --fail --location https://deb.nodesource.com/setup_14.x | bash - \ - && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ + curl --silent --fail --location https://deb.nodesource.com/setup_14.x | bash - \ + && curl --silent --fail https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - >/dev/null 2>&1 \ && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list" ARG ADDITIONAL_DEV_APT_COMMAND="" ARG ADDITIONAL_DEV_ENV_VARS="" @@ -57,16 +52,15 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \ ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \ ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND} -# As of August 2021, Debian buster-slim does not include Python2 by default and we need it -# as we still support running Python2 via PythonVirtualenvOperator -# TODO: Remove python2 when we stop supporting it - # Install basic and additional apt dependencies -RUN mkdir -pv /usr/share/man/man1 \ +RUN apt-get update \ + && apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 \ + && apt-get install -y --no-install-recommends curl gnupg2 \ + && mkdir -pv /usr/share/man/man1 \ && mkdir -pv /usr/share/man/man7 \ && export ${ADDITIONAL_DEV_ENV_VARS?} \ - && bash -o pipefail -e -u -x -c "${DEV_APT_COMMAND}" \ - && bash -o pipefail -e -u -x -c "${ADDITIONAL_DEV_APT_COMMAND}" \ + && bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}" \ + && bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_DEV_APT_COMMAND}" \ && apt-get update \ && apt-get install -y --no-install-recommends \ apt-utils \ @@ -101,18 +95,18 @@ RUN mkdir -pv /usr/share/man/man1 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY scripts/docker/*.sh /scripts/docker/ -RUN bash /scripts/docker/install_mysql.sh dev \ - && bash /scripts/docker/install_mssql.sh \ - && adduser airflow \ - && echo "airflow:airflow" | chpasswd \ +# Only copy mysql/mssql installation scripts for now - so that changing the other +# scripts which are needed much later will not invalidate the docker layer here +COPY scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh /scripts/docker/ +RUN bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_mysql.sh dev \ + && bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_mssql.sh \ + && adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \ + --quiet "airflow" --home "/home/airflow" \ + && echo -e "airflow\nairflow" | passwd airflow 2>&1 \ && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ && chmod 0440 /etc/sudoers.d/airflow -# The latest buster images do not have libpython 2.7 installed and it is needed -# To run virtualenv tests with python 2 ARG RUNTIME_APT_DEPS="\ - gnupg \ libgcc-8-dev \ apt-transport-https \ bash-completion \ @@ -138,7 +132,7 @@ ARG HELM_VERSION="v3.6.3" RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \ && HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz" \ - && curl --location "${HELM_URL}" | tar -xvz -O "${SYSTEM}"-amd64/helm > /usr/local/bin/helm \ + && curl --silent --location "${HELM_URL}" | tar -xz -O "${SYSTEM}"-amd64/helm > /usr/local/bin/helm \ && chmod +x /usr/local/bin/helm ARG ADDITIONAL_RUNTIME_APT_DEPS="" @@ -152,7 +146,6 @@ ARG HOME=/root ARG AIRFLOW_HOME=/root/airflow ARG AIRFLOW_SOURCES=/opt/airflow - ENV RUNTIME_APT_DEP=${RUNTIME_APT_DEPS} \ ADDITIONAL_RUNTIME_APT_DEPS=${ADDITIONAL_RUNTIME_APT_DEPS} \ RUNTIME_APT_COMMAND=${RUNTIME_APT_COMMAND} \ @@ -168,8 +161,8 @@ RUN mkdir -pv /usr/share/man/man1 \ && mkdir -pv /usr/share/man/man7 \ && export ${ADDITIONAL_DEV_APT_ENV?} \ && export ${ADDITIONAL_RUNTIME_APT_ENV?} \ - && bash -o pipefail -e -u -x -c "${RUNTIME_APT_COMMAND}" \ - && bash -o pipefail -e -u -x -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" \ + && bash -o pipefail -o errexit -o nounset -o nolog -c "${RUNTIME_APT_COMMAND}" \ + && bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" \ && apt-get update \ && apt-get install --no-install-recommends -y \ ${RUNTIME_APT_DEPS} \ @@ -177,7 +170,7 @@ RUN mkdir -pv /usr/share/man/man1 \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ - && curl https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_CLI_VERSION}.tgz \ + && curl --silent "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_CLI_VERSION}.tgz" \ | tar -C /usr/bin --strip-components=1 -xvzf - docker/docker WORKDIR ${AIRFLOW_SOURCES} @@ -194,7 +187,7 @@ ARG BATS_FILE_VERSION="0.2.0" RUN curl -sSL https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.tar.gz -o /tmp/bats.tgz \ && tar -zxf /tmp/bats.tgz -C /tmp \ - && /bin/bash /tmp/bats-core-${BATS_VERSION}/install.sh /opt/bats && rm -rf \ + && bash -o pipefail -o errexit -o nounset -o nolog /tmp/bats-core-${BATS_VERSION}/install.sh /opt/bats && rm -rf \ && mkdir -p /opt/bats/lib/bats-support \ && curl -sSL https://github.com/bats-core/bats-support/archive/v${BATS_SUPPORT_VERSION}.tar.gz -o /tmp/bats-support.tgz \ && tar -zxf /tmp/bats-support.tgz -C /opt/bats/lib/bats-support --strip 1 && rm -rf /tmp/* \ @@ -276,6 +269,8 @@ ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \ UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} +COPY scripts/docker/*.sh scripts/docker/install_pip_version.sh /scripts/docker/ + # In case of CI builds we want to pre-install main version of airflow dependencies so that # We do not have to always reinstall it from the scratch. # And is automatically reinstalled from the scratch every time patch release of python gets released @@ -283,10 +278,12 @@ ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENT # are uninstalled, only dependencies remain. # the cache is only used when "upgrade to newer dependencies" is not set to automatically # account for removed dependencies (we do not install them in the first place) -RUN bash /scripts/docker/install_pip_version.sh; \ +RUN echo -e "\n\e[32mThe 'Running pip as the root user' warnings below are not valid but we can't disable them :(\e[0m\n"; \ + echo -e "\n\e[34mSee https://github.com/pypa/pip/issues/10556 for details.\e[0m\n" ; \ + bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_pip_version.sh; \ if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ - bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ + bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi # Generate random hex dump file so that we can determine whether it's faster to rebuild the image @@ -297,13 +294,6 @@ RUN head -c 30 /dev/urandom | xxd -ps >/build-cache-hash # Link dumb-init for backwards compatibility (so that older images also work) RUN ln -sf /usr/bin/dumb-init /usr/local/bin/dumb-init -# Install NPM dependencies here. The NPM dependencies don't change that often and we already have pip -# installed dependencies in case of CI optimised build, so it is ok to install NPM deps here -# Rather than after setup.py is added. -COPY airflow/www/yarn.lock airflow/www/package.json ${AIRFLOW_SOURCES}/airflow/www/ - -RUN yarn --cwd airflow/www install --frozen-lockfile --no-cache && yarn cache clean - # Note! We are copying everything with airflow:airflow user:group even if we use root to run the scripts # This is fine as root user will be able to use those dirs anyway. @@ -322,17 +312,16 @@ COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/__init__.py # But in cron job we will install latest versions matching setup.py to see if there is no breaking change # and push the constraints if everything is successful RUN if [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \ - bash /scripts/docker/install_airflow.sh; \ + bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_airflow.sh; \ fi # Copy all the www/ files we need to compile assets. Done as two separate COPY # commands so as otherwise it copies the _contents_ of static/ in to www/ -COPY airflow/www/webpack.config.js ${AIRFLOW_SOURCES}/airflow/www/ +COPY airflow/www/webpack.config.js airflow/www/package.json airflow/www/yarn.lock ${AIRFLOW_SOURCES}/airflow/www/ COPY airflow/www/static ${AIRFLOW_SOURCES}/airflow/www/static/ -COPY airflow/www/compile_assets.sh ${AIRFLOW_SOURCES}/airflow/www/compile_assets.sh # Package JS/css for production -RUN bash airflow/www/compile_assets.sh +RUN bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/compile_www_assets.sh COPY scripts/in_container/entrypoint_ci.sh /entrypoint RUN chmod a+x /entrypoint @@ -342,9 +331,9 @@ COPY scripts/docker/load.bash /opt/bats/lib/ # Additional python deps to install ARG ADDITIONAL_PYTHON_DEPS="" -RUN bash /scripts/docker/install_pip_version.sh; \ +RUN bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_pip_version.sh; \ if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ - bash /scripts/docker/install_additional_dependencies.sh; \ + bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_additional_dependencies.sh; \ fi # Install autocomplete for airflow diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh index 1b3fd56724279..36097d19f5b73 100755 --- a/scripts/docker/common.sh +++ b/scripts/docker/common.sh @@ -17,16 +17,22 @@ # under the License. set -euo pipefail -test -v INSTALL_MYSQL_CLIENT -test -v INSTALL_MSSQL_CLIENT -test -v AIRFLOW_REPO -test -v AIRFLOW_BRANCH -test -v AIRFLOW_PIP_VERSION +function common::get_colors() { + COLOR_BLUE=$'\e[34m' + COLOR_GREEN=$'\e[32m' + COLOR_RED=$'\e[31m' + COLOR_RESET=$'\e[0m' + COLOR_YELLOW=$'\e[33m' + export COLOR_BLUE + export COLOR_GREEN + export COLOR_RED + export COLOR_RESET + export COLOR_YELLOW +} -set -x function common::get_airflow_version_specification() { - if [[ -z ${AIRFLOW_VERSION_SPECIFICATION} + if [[ -z ${AIRFLOW_VERSION_SPECIFICATION=} && -n ${AIRFLOW_VERSION} && ${AIRFLOW_INSTALLATION_METHOD} != "." ]]; then AIRFLOW_VERSION_SPECIFICATION="==${AIRFLOW_VERSION}" @@ -41,10 +47,9 @@ function common::override_pip_version_if_needed() { fi } - function common::get_constraints_location() { # auto-detect Airflow-constraint reference and location - if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE}" ]]; then + if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then if [[ ${AIRFLOW_VERSION} =~ v?2.* ]]; then AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION} else @@ -52,7 +57,7 @@ function common::get_constraints_location() { fi fi - if [[ -z ${AIRFLOW_CONSTRAINTS_LOCATION} ]]; then + if [[ -z ${AIRFLOW_CONSTRAINTS_LOCATION=} ]]; then local constraints_base="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${AIRFLOW_CONSTRAINTS_REFERENCE}" local python_version python_version="$(python --version 2>/dev/stdout | cut -d " " -f 2 | cut -d "." -f 1-2)" diff --git a/scripts/docker/compile_www_assets.sh b/scripts/docker/compile_www_assets.sh index 59a7017fd157f..b9034be8e92d7 100755 --- a/scripts/docker/compile_www_assets.sh +++ b/scripts/docker/compile_www_assets.sh @@ -17,12 +17,18 @@ # under the License. # shellcheck disable=SC2086 set -euo pipefail -set -x + +BUILD_TYPE=${BUILD_TYPE="prod"} + +COLOR_BLUE=$'\e[34m' +readonly COLOR_BLUE +COLOR_RESET=$'\e[0m' +readonly COLOR_RESET # Installs additional dependencies passed as Argument to the Docker build command function compile_www_assets() { echo - echo Compiling WWW assets + echo "${COLOR_BLUE}Compiling www assets${COLOR_RESET}" echo local md5sum_file md5sum_file="static/dist/sum.md5" @@ -30,13 +36,33 @@ function compile_www_assets() { local www_dir if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." ]]; then # In case we are building from sources in production image, we should build the assets - www_dir="${AIRFLOW_SOURCES_TO}/airflow/www" + www_dir="${AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES}}/airflow/www" else www_dir="$(python -m site --user-site)/airflow/www" fi pushd ${www_dir} || exit 1 - yarn install --frozen-lockfile --no-cache - yarn run prod + set +e + yarn install --frozen-lockfile --no-cache 2>/tmp/out-yarn-install.txt + local res=$? + if [[ ${res} != 0 ]]; then + >&2 echo + >&2 echo "Error when running yarn install:" + >&2 echo + >&2 cat /tmp/out-yarn-install.txt && rm -f /tmp/out-yarn-install.txt + exit 1 + fi + rm -f /tmp/out-yarn-install.txt + yarn run "${BUILD_TYPE}" 2>/tmp/out-yarn-run.txt + res=$? + if [[ ${res} != 0 ]]; then + >&2 echo + >&2 echo "Error when running yarn install:" + >&2 echo + >&2 cat /tmp/out-yarn-run.txt && rm -rf /tmp/out-yarn-run.txt + exit 1 + fi + rm -f /tmp/out-yarn-run.txt + set -e find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > "${md5sum_file}" rm -rf "${www_dir}/node_modules" rm -vf "${www_dir}"/{package.json,yarn.lock,.eslintignore,.eslintrc,.stylelintignore,.stylelintrc,compile_assets.sh,webpack.config.js} diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh index f0dc054abf61a..112ff63b0eccd 100755 --- a/scripts/docker/install_additional_dependencies.sh +++ b/scripts/docker/install_additional_dependencies.sh @@ -18,10 +18,10 @@ # shellcheck disable=SC2086 set -euo pipefail -test -v UPGRADE_TO_NEWER_DEPENDENCIES -test -v ADDITIONAL_PYTHON_DEPS -test -v EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS -test -v AIRFLOW_PIP_VERSION +: "${UPGRADE_TO_NEWER_DEPENDENCIES:?Should be true or false}" +: "${ADDITIONAL_PYTHON_DEPS:?Should be set}" +: "${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS:?Should be set}" +: "${AIRFLOW_PIP_VERSION:?Should be set}" # shellcheck source=scripts/docker/common.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" @@ -33,25 +33,32 @@ set -x function install_additional_dependencies() { if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then echo - echo Installing additional dependencies while upgrading to newer dependencies + echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}" echo pip install --upgrade --upgrade-strategy eager \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + echo + echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" + echo pip check else echo - echo Installing additional dependencies upgrading only if needed + echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}" echo pip install --upgrade --upgrade-strategy only-if-needed \ ${ADDITIONAL_PYTHON_DEPS} # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + echo + echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" + echo pip check fi } +common::get_colors common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh index e89e09b2dadee..b2ece66697ef7 100755 --- a/scripts/docker/install_airflow.sh +++ b/scripts/docker/install_airflow.sh @@ -29,13 +29,15 @@ # shellcheck source=scripts/docker/common.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" +: "${AIRFLOW_PIP_VERSION:?Should be set}" + function install_airflow() { # Coherence check for editable installation mode. if [[ ${AIRFLOW_INSTALLATION_METHOD} != "." && \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} == "--editable" ]]; then echo - echo "ERROR! You can only use --editable flag when installing airflow from sources!" - echo " Current installation method is '${AIRFLOW_INSTALLATION_METHOD} and should be '.'" + echo "${COLOR_RED}ERROR! You can only use --editable flag when installing airflow from sources!${COLOR_RESET}" + echo "{COLOR_RED} Current installation method is '${AIRFLOW_INSTALLATION_METHOD} and should be '.'${COLOR_RESET}" exit 1 fi # Remove mysql from extras if client is not going to be installed @@ -44,7 +46,7 @@ function install_airflow() { fi if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then echo - echo Installing all packages with eager upgrade + echo "${COLOR_BLUE}Installing all packages with eager upgrade${COLOR_RESET}" echo # eager upgrade pip install --upgrade --upgrade-strategy eager \ @@ -57,13 +59,15 @@ function install_airflow() { pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" fi - # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + echo + echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" + echo pip check else \ echo - echo Installing all packages with constraints and upgrade if needed + echo "${COLOR_BLUE}Installing all packages with constraints and upgrade if needed${COLOR_RESET}" echo pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ @@ -76,11 +80,15 @@ function install_airflow() { "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + echo + echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" + echo pip check fi } +common::get_colors common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location diff --git a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh index 8ebb619d39982..38577c19b1685 100755 --- a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh +++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -29,10 +29,14 @@ # shellcheck source=scripts/docker/common.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" +: "${AIRFLOW_REPO:?Should be set}" +: "${AIRFLOW_BRANCH:?Should be set}" +: "${INSTALL_MYSQL_CLIENT:?Should be true or false}" +: "${AIRFLOW_PIP_VERSION:?Should be set}" function install_airflow_dependencies_from_branch_tip() { echo - echo "Installing airflow from ${AIRFLOW_BRANCH}. It is used to cache dependencies" + echo "${COLOR_BLUE}Installing airflow from ${AIRFLOW_BRANCH}. It is used to cache dependencies${COLOR_RESET}" echo if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,} @@ -46,11 +50,12 @@ function install_airflow_dependencies_from_branch_tip() { pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true echo - echo Uninstalling just airflow. Dependencies remain. + echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now target airflow can be reinstalled using mostly cached dependencies${COLOR_RESET}" echo pip uninstall --yes apache-airflow || true } +common::get_colors common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index 7be9ef1eebbc4..1bd78b5f16315 100755 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -25,6 +25,8 @@ # shellcheck source=scripts/docker/common.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" +: "${AIRFLOW_PIP_VERSION:?Should be set}" + function install_airflow_and_providers_from_docker_context_files(){ if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,} @@ -65,7 +67,7 @@ function install_airflow_and_providers_from_docker_context_files(){ if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then echo - echo Force re-installing airflow and providers from local files with eager upgrade + echo "${COLOR_BLUE}Force re-installing airflow and providers from local files with eager upgrade${COLOR_RESET}" echo # force reinstall all airflow + provider package local files with eager upgrade pip install "${pip_flags[@]}" --upgrade --upgrade-strategy eager \ @@ -73,7 +75,7 @@ function install_airflow_and_providers_from_docker_context_files(){ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} else echo - echo Force re-installing airflow and providers from local files with constraints and upgrade if needed + echo "${COLOR_BLUE}Force re-installing airflow and providers from local files with constraints and upgrade if needed${COLOR_RESET}" echo if [[ ${AIRFLOW_CONSTRAINTS_LOCATION} == "/"* ]]; then grep -ve '^apache-airflow' <"${AIRFLOW_CONSTRAINTS_LOCATION}" > /tmp/constraints.txt @@ -103,9 +105,10 @@ function install_airflow_and_providers_from_docker_context_files(){ # without dependencies. This is extremely useful in case you want to install via pip-download # method on air-gaped system where you do not want to download any dependencies from remote hosts # which is a requirement for serious installations -install_all_other_packages_from_docker_context_files() { +function install_all_other_packages_from_docker_context_files() { + echo - echo Force re-installing all other package from local files without dependencies + echo "${COLOR_BLUE}Force re-installing all other package from local files without dependencies${COLOR_RESET}" echo local reinstalling_other_packages # shellcheck disable=SC2010 @@ -118,6 +121,7 @@ install_all_other_packages_from_docker_context_files() { fi } +common::get_colors common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location diff --git a/scripts/docker/install_mssql.sh b/scripts/docker/install_mssql.sh index b5f8b5108becf..2e2693c8bacaa 100755 --- a/scripts/docker/install_mssql.sh +++ b/scripts/docker/install_mssql.sh @@ -15,13 +15,21 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -set -exuo pipefail +set -euo pipefail + +: "${INSTALL_MSSQL_CLIENT:?Should be true or false}" + +COLOR_BLUE=$'\e[34m' +readonly COLOR_BLUE +COLOR_RESET=$'\e[0m' +readonly COLOR_RESET + function install_mssql_client() { echo - echo Installing mssql client + echo "${COLOR_BLUE}Installing mssql client${COLOR_RESET}" echo - curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - - curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list + curl --silent https://packages.microsoft.com/keys/microsoft.asc | apt-key add - >/dev/null 2>&1 + curl --silent https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list apt-get update -yqq apt-get upgrade -yqq ACCEPT_EULA=Y apt-get -yqq install -y --no-install-recommends msodbcsql17 mssql-tools diff --git a/scripts/docker/install_mysql.sh b/scripts/docker/install_mysql.sh index 7983eb09c96bb..52aca493a2df5 100755 --- a/scripts/docker/install_mysql.sh +++ b/scripts/docker/install_mysql.sh @@ -15,15 +15,22 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -set -exuo pipefail +set -euo pipefail declare -a packages MYSQL_VERSION="8.0" readonly MYSQL_VERSION +COLOR_BLUE=$'\e[34m' +readonly COLOR_BLUE +COLOR_RESET=$'\e[0m' +readonly COLOR_RESET + +: "${INSTALL_MYSQL_CLIENT:?Should be true or false}" + install_mysql_client() { echo - echo Installing mysql client + echo "${COLOR_BLUE}Installing mysql client version ${MYSQL_VERSION}${COLOR_RESET}" echo if [[ "${1}" == "dev" ]]; then @@ -46,14 +53,13 @@ install_mysql_client() { for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \ keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80) do - gpg --keyserver "${keyserver}" --recv-keys "${key}" && break + gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break done set -e gpg --export "${key}" > /etc/apt/trusted.gpg.d/mysql.gpg gpgconf --kill all rm -rf "${GNUPGHOME}" unset GNUPGHOME - apt-key list > /dev/null 2>&1 echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" diff --git a/scripts/docker/install_pip_version.sh b/scripts/docker/install_pip_version.sh index 312a95470d762..68b138fecc580 100755 --- a/scripts/docker/install_pip_version.sh +++ b/scripts/docker/install_pip_version.sh @@ -16,27 +16,23 @@ # specific language governing permissions and limitations # under the License. -# Install airflow using regular 'pip install' command. This install airflow depending on the arguments: -# AIRFLOW_INSTALLATION_METHOD - determines where to install airflow form: -# "." - installs airflow from local sources -# "apache-airflow" - installs airflow from PyPI 'apache-airflow' package -# AIRFLOW_VERSION_SPECIFICATION - optional specification for Airflow version to install ( -# might be ==2.0.2 for example or <3.0.0 -# UPGRADE_TO_NEWER_DEPENDENCIES - determines whether eager-upgrade should be performed with the -# dependencies (with EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS added) -# # shellcheck disable=SC2086 # shellcheck source=scripts/docker/common.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" +: "${AIRFLOW_PIP_VERSION:?Should be set}" + function install_pip_version() { + echo + echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" + echo pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" && mkdir -p ${HOME}/.local/bin } +common::get_colors common::get_airflow_version_specification common::override_pip_version_if_needed -common::get_constraints_location common::show_pip_version_and_location install_pip_version