Skip to content

Optimize dockerfiles for local rebuilds #20238

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 1 commit into from
Jan 11, 2022
Merged

Optimize dockerfiles for local rebuilds #20238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 107 additions & 71 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,10 +166,18 @@ ARG INSTALL_PROVIDERS_FROM_SOURCES="false"
# But it also can be `.` from local installation or GitHub URL pointing to specific branch or tag
# Of Airflow. Note That for local source installation you need to have local sources of
# Airflow checked out together with the Dockerfile and AIRFLOW_SOURCES_FROM and AIRFLOW_SOURCES_TO
# set to "." and "/opt/airflow" respectively.
# set to "." and "/opt/airflow" respectively. Similarly AIRFLOW_SOURCES_WWW_FROM/TO are set to right source
# and destination
ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow"
# By default we do not upgrade to latest dependencies
ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
# By default we install latest airflow from PyPI so we do not need to copy sources of Airflow
# www to compile the assets but in case of breeze/CI builds we use latest sources and we override those
# those SOURCES_FROM/TO with "airflow/www" and "/opt/airflow/airflow/www" respectively.
# This is to rebuild the assets only when any of the www sources change
ARG AIRFLOW_SOURCES_WWW_FROM="empty"
ARG AIRFLOW_SOURCES_WWW_TO="/empty"

# By default we install latest airflow from PyPI so we do not need to copy sources of Airflow
# but in case of breeze/CI builds we use latest sources and we override those
# those SOURCES_FROM/TO with "." and "/opt/airflow" respectively
Expand All @@ -181,30 +189,13 @@ ARG AIRFLOW_USER_HOME_DIR
ARG AIRFLOW_UID

ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \
INSTALL_MSSQL_CLIENT=${INSTALL_MSSQL_CLIENT} \
AIRFLOW_REPO=${AIRFLOW_REPO} \
AIRFLOW_BRANCH=${AIRFLOW_BRANCH} \
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} \
CONSTRAINTS_GITHUB_REPOSITORY=${CONSTRAINTS_GITHUB_REPOSITORY} \
AIRFLOW_CONSTRAINTS=${AIRFLOW_CONSTRAINTS} \
AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE} \
AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \
DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \
PATH=${PATH}:${AIRFLOW_USER_HOME_DIR}/.local/bin \
AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \
AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \
AIRFLOW_HOME=${AIRFLOW_HOME} \
AIRFLOW_UID=${AIRFLOW_UID} \
AIRFLOW_INSTALL_EDITABLE_FLAG="" \
UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} \
# By default PIP installs everything to ~/.local
PIP_USER="true"
INSTALL_MSSQL_CLIENT=${INSTALL_MSSQL_CLIENT}

COPY scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh /scripts/docker/
# Only copy mysql/mssql installation scripts for now - so that changing the other
# scripts which are needed much later will not invalidate the docker layer here
COPY scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh /scripts/docker/

RUN bash -o pipefail -o errexit -o nounset -o nolog ./scripts/docker/install_mysql.sh dev \
&& bash -o pipefail -o errexit -o nounset -o nolog ./scripts/docker/install_mssql.sh
RUN /scripts/docker/install_mysql.sh dev && /scripts/docker/install_mssql.sh
ENV PATH=${PATH}:/opt/mssql-tools/bin

COPY docker-context-files /docker-context-files
Expand All @@ -220,17 +211,38 @@ RUN if [[ -f /docker-context-files/pip.conf ]]; then \
cp /docker-context-files/pip.conf "${AIRFLOW_USER_HOME_DIR}/.config/pip/pip.conf"; \
fi

# Copy all scripts required for installation - changing any of those should lead to
# rebuilding from here
COPY --chown=airflow:0 scripts/docker/* /scripts/docker/

ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \
ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \
INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES} \
AIRFLOW_VERSION=${AIRFLOW_VERSION} \
AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD} \
AIRFLOW_VERSION_SPECIFICATION=${AIRFLOW_VERSION_SPECIFICATION} \
AIRFLOW_SOURCES_FROM=${AIRFLOW_SOURCES_FROM} \
AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO}
AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO} \
AIRFLOW_REPO=${AIRFLOW_REPO} \
AIRFLOW_BRANCH=${AIRFLOW_BRANCH} \
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} \
CONSTRAINTS_GITHUB_REPOSITORY=${CONSTRAINTS_GITHUB_REPOSITORY} \
AIRFLOW_CONSTRAINTS=${AIRFLOW_CONSTRAINTS} \
AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE} \
AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \
DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \
PATH=${PATH}:${AIRFLOW_USER_HOME_DIR}/.local/bin \
AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \
AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \
AIRFLOW_HOME=${AIRFLOW_HOME} \
AIRFLOW_UID=${AIRFLOW_UID} \
AIRFLOW_INSTALL_EDITABLE_FLAG="" \
UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} \
# By default PIP installs everything to ~/.local
PIP_USER="true"

# Copy all scripts required for installation - changing any of those should lead to
# rebuilding from here
COPY --chown=airflow:0 scripts/docker/common.sh scripts/docker/install_pip_version.sh \
/scripts/docker/install_airflow_dependencies_from_branch_tip.sh \
/scripts/docker/

# In case of Production build image segment we want to pre-install main version of airflow
# dependencies from GitHub so that we do not have to always reinstall it from the scratch.
Expand All @@ -239,15 +251,33 @@ ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \
# the cache is only used when "upgrade to newer dependencies" is not set to automatically
# account for removed dependencies (we do not install them in the first place)
# Upgrade to specific PIP version
RUN bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_pip_version.sh; \
RUN /scripts/docker/install_pip_version.sh; \
if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \
${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \
bash -o pipefail -o errexit -o nounset -o nolog \
/scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
/scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
fi

COPY --chown=airflow:0 scripts/docker/compile_www_assets.sh scripts/docker/prepare_node_modules.sh /scripts/docker/
COPY --chown=airflow:0 ${AIRFLOW_SOURCES_WWW_FROM} ${AIRFLOW_SOURCES_WWW_TO}

# hadolint ignore=SC2086, SC2010
RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \
# only prepare node modules and compile assets if the prod image is build from sources
# otherwise they are already compiled-in. We should do it in one step with removing artifacts \
# as we want to keep the final image small
/scripts/docker/prepare_node_modules.sh; \
REMOVE_ARTIFACTS="true" BUILD_TYPE="prod" /scripts/docker/compile_www_assets.sh; \
# Copy generated dist folder (otherwise it will be overridden by the COPY step below)
mv -f /opt/airflow/airflow/www/static/dist /tmp/dist; \
fi;

COPY --chown=airflow:0 ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO}

# Copy back the generated dist folder
RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \
mv -f /tmp/dist /opt/airflow/airflow/www/static/dist; \
fi;

# Add extra python dependencies
ARG ADDITIONAL_PYTHON_DEPS=""
# We can set this value to true in case we want to install .whl .tar.gz packages placed in the
Expand All @@ -271,19 +301,18 @@ ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \

WORKDIR /opt/airflow

COPY --chown=airflow:0 scripts/docker/install_from_docker_context_files.sh scripts/docker/install_airflow.sh \
scripts/docker/install_additional_dependencies.sh \
/scripts/docker/

# hadolint ignore=SC2086, SC2010
RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \
# only compile assets if the prod image is build from sources
# otherwise they are already compiled-in
bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/compile_www_assets.sh; \
fi; \
if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \
bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_from_docker_context_files.sh; \
RUN if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \
/scripts/docker/install_from_docker_context_files.sh; \
elif [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \
bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_airflow.sh; \
/scripts/docker/install_airflow.sh; \
fi; \
if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_additional_dependencies.sh; \
/scripts/docker/install_additional_dependencies.sh; \
fi; \
find "${AIRFLOW_USER_HOME_DIR}/.local/" -name '*.pyc' -print0 | xargs -0 rm -f || true ; \
find "${AIRFLOW_USER_HOME_DIR}/.local/" -type d -name '__pycache__' -print0 | xargs -0 rm -rf || true ; \
Expand Down Expand Up @@ -392,11 +421,7 @@ ARG AIRFLOW_HOME
# Having the variable in final image allows to disable providers manager warnings when
# production image is prepared from sources rather than from package
ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow"
ARG BUILD_ID
ARG COMMIT_SHA
ARG AIRFLOW_IMAGE_REPOSITORY
ARG AIRFLOW_IMAGE_DATE_CREATED
ARG AIRFLOW_VERSION_SPECIFICATION

ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \
ADDITIONAL_RUNTIME_APT_DEPS=${ADDITIONAL_RUNTIME_APT_DEPS} \
Expand All @@ -412,8 +437,6 @@ ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \
GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm" \
AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD} \
AIRFLOW_VERSION_SPECIFICATION=${AIRFLOW_VERSION_SPECIFICATION} \
BUILD_ID=${BUILD_ID} \
COMMIT_SHA=${COMMIT_SHA} \
# By default PIP installs everything to ~/.local
PIP_USER="true"

Expand All @@ -434,11 +457,11 @@ RUN apt-get update \
${ADDITIONAL_RUNTIME_APT_DEPS} \
&& apt-get autoremove -yqq --purge \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /var/log/*

# Only copy install_m(y/s)sql and install_pip_version.sh. We do not need any other scripts in the final image.
COPY scripts/docker/install_mysql.sh /scripts/docker/install_mssql.sh scripts/docker/install_pip_version.sh \
scripts/docker/common.sh /scripts/docker/
# Only copy install_m(y/s)sql. We do not need any other scripts in the final image.
COPY scripts/docker/install_mysql.sh /scripts/docker/install_mssql.sh /scripts/docker/

# fix permission issue in Azure DevOps when running the scripts
RUN chmod a+x /scripts/docker/install_mysql.sh && \
Expand All @@ -453,8 +476,9 @@ RUN chmod a+x /scripts/docker/install_mysql.sh && \
mkdir -pv "${AIRFLOW_HOME}/dags"; \
mkdir -pv "${AIRFLOW_HOME}/logs"; \
chown -R airflow:0 "${AIRFLOW_USER_HOME_DIR}" "${AIRFLOW_HOME}"; \
find "${AIRFLOW_HOME}" -executable -print0 | xargs --null chmod g+x && \
find "${AIRFLOW_HOME}" -print0 | xargs --null chmod g+rw
chmod -R g+rw "${AIRFLOW_USER_HOME_DIR}" "${AIRFLOW_HOME}" ; \
find "${AIRFLOW_HOME}" -executable -print0 | xargs --null chmod g+x; \
find "${AIRFLOW_USER_HOME_DIR}" -executable -print0 | xargs --null chmod g+x

COPY --chown=airflow:0 --from=airflow-build-image \
"${AIRFLOW_USER_HOME_DIR}/.local" "${AIRFLOW_USER_HOME_DIR}/.local"
Expand All @@ -463,19 +487,45 @@ COPY --chown=airflow:0 scripts/in_container/prod/clean-logs.sh /clean-logs

# Make /etc/passwd root-group-writeable so that user can be dynamically added by OpenShift
# See https://github.com/apache/airflow/issues/9248
# Set default groups for airflow and root user

RUN chmod a+x /entrypoint /clean-logs && \
chmod g=u /etc/passwd
chmod g=u /etc/passwd && \
chmod g+w "${AIRFLOW_USER_HOME_DIR}/.local" && \
usermod -g 0 airflow -G 0

# make sure that the venv is activated for all users
# including plain sudo, sudo with --interactive flag
RUN sed --in-place=.bak "s/secure_path=\"/secure_path=\"\/.venv\/bin:/" /etc/sudoers

# See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
# to learn more about the way how signals are handled by the image
# Also set airflow as nice PROMPT message.
# LD_PRELOAD is to workaround https://github.com/apache/airflow/issues/17546
# issue with /usr/lib/x86_64-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block
# We do not yet a more "correct" solution to the problem but in order to avoid raising new issues
# by users of the prod image, we implement the workaround now.
# The side effect of this is slightly (in the range of 100s of milliseconds) slower load for any
# binary started and a little memory used for Heap allocated by initialization of libstdc++
# This overhead is not happening for binaries that already link dynamically libstdc++
ENV DUMB_INIT_SETSID="1" \
PS1="(airflow)" \
LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libstdc++.so.6"

WORKDIR ${AIRFLOW_HOME}

EXPOSE 8080

RUN usermod -g 0 airflow -G 0

USER ${AIRFLOW_UID}

RUN /scripts/docker/install_pip_version.sh
# Those should be set and used as late as possible as any change in commit/build otherwise invalidates the
# layers right after
ARG BUILD_ID
ARG COMMIT_SHA
ARG AIRFLOW_IMAGE_REPOSITORY
ARG AIRFLOW_IMAGE_DATE_CREATED

ENV BUILD_ID=${BUILD_ID} COMMIT_SHA=${COMMIT_SHA}

LABEL org.apache.airflow.distro="debian" \
org.apache.airflow.distro.version="buster" \
Expand All @@ -499,19 +549,5 @@ LABEL org.apache.airflow.distro="debian" \
org.opencontainers.image.title="Production Airflow Image" \
org.opencontainers.image.description="Reference, production-ready Apache Airflow image"


# See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
# to learn more about the way how signals are handled by the image
ENV DUMB_INIT_SETSID="1"

# This one is to workaround https://github.com/apache/airflow/issues/17546
# issue with /usr/lib/x86_64-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block
# We do not yet a more "correct" solution to the problem but in order to avoid raising new issues
# by users of the prod image, we implement the workaround now.
# The side effect of this is slightly (in the range of 100s of milliseconds) slower load for any
# binary started and a little memory used for Heap allocated by initialization of libstdc++
# This overhead is not happening for binaries that already link dynamically libstdc++
ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libstdc++.so.6"

ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]
CMD []
Loading