From a23780ac2c54c3e7e7b824bc1a5b91b0363f3a92 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 23 Oct 2021 21:27:10 +0200 Subject: [PATCH 1/2] Swtich both PROD and CI images to use virtual environments Seems that the future direction of package installation for Python is to always use virtualenvs to install dependencies. There was a heated discussion about it in the issue here: https://github.com/pypa/pip/issues/10556 and general consensus is that virtualenv building should also be used in Docker images as it can help to avoid multiple problems related to interference between distro-managed and PIP-managed files. This change implements it - both PROD and CI images are converted to use virtualenv for installation and instead of copying the `.local` directory between image segments they do the same with newly created `/.venv` virtual environment. All dependencies and shared libraries are installed there are and shared between all users using the images. --- Dockerfile | 143 ++++++++++-------- Dockerfile.ci | 54 ++++--- IMAGES.rst | 2 +- airflow/operators/python.py | 13 ++ airflow/utils/python_virtualenv.py | 39 +++-- dev/README_RELEASE_PROVIDER_PACKAGES.md | 2 +- .../installation/installing-from-pypi.rst | 2 +- docs/docker-stack/build-arg-ref.rst | 2 +- docs/docker-stack/build.rst | 18 ++- .../ci/kubernetes/ci_run_kubernetes_tests.sh | 2 +- scripts/ci/libraries/_initialization.sh | 2 +- scripts/docker/common.sh | 18 ++- scripts/docker/compile_www_assets.sh | 2 +- scripts/docker/create_venv.sh | 33 ++++ .../docker/install_additional_dependencies.sh | 19 ++- scripts/docker/install_airflow.sh | 13 +- ...ll_airflow_dependencies_from_branch_tip.sh | 5 +- .../install_from_docker_context_files.sh | 16 +- scripts/docker/install_pip_version.sh | 3 +- scripts/in_container/prod/entrypoint_prod.sh | 21 +-- setup.cfg | 1 + tests/operators/test_python.py | 60 +++++++- 22 files changed, 318 insertions(+), 152 deletions(-) create mode 100755 scripts/docker/create_venv.sh diff --git a/Dockerfile b/Dockerfile index 51874d29140f6..d26a40a25924c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,13 +21,12 @@ # # airflow-build-image - there all airflow dependencies can be installed (and # built - for those dependencies that require -# build essentials). Airflow is installed there with -# --user switch so that all the dependencies are -# installed to ${HOME}/.local +# build essentials). Python dependencies are installed +# into a virtual environment at ${HOME}/.venv # # main - this is the actual production image that is much # smaller because it does not contain all the build -# essentials. Instead the ${HOME}/.local folder +# essentials. Instead the ${HOME}/.venv folder # is copied from the build-image - this way we have # only result of installation and we do not need # all the build essentials. This makes the image @@ -43,7 +42,7 @@ ARG AIRFLOW_UID="50000" ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster" -ARG AIRFLOW_PIP_VERSION=21.2.4 +ARG AIRFLOW_PIP_VERSION=21.3.1 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" # By default PIP has progress bar but you can disable it. @@ -174,25 +173,12 @@ ARG AIRFLOW_SOURCES_FROM="empty" ARG AIRFLOW_SOURCES_TO="/empty" ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \ - INSTALL_MSSQL_CLIENT=${INSTALL_MSSQL_CLIENT} \ - AIRFLOW_REPO=${AIRFLOW_REPO} \ - AIRFLOW_BRANCH=${AIRFLOW_BRANCH} \ - AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} \ - CONSTRAINTS_GITHUB_REPOSITORY=${CONSTRAINTS_GITHUB_REPOSITORY} \ - AIRFLOW_CONSTRAINTS=${AIRFLOW_CONSTRAINTS} \ - AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE} \ - AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \ - DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \ - PATH=${PATH}:/root/.local/bin \ - AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ - PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \ - # Install Airflow with "--user" flag, so that we can copy the whole .local folder to the final image - # from the build image and always in non-editable mode - AIRFLOW_INSTALL_USER_FLAG="--user" \ - AIRFLOW_INSTALL_EDITABLE_FLAG="" \ - UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} + INSTALL_MSSQL_CLIENT=${INSTALL_MSSQL_CLIENT} + +# Only copy mysql/mssql installation scripts for now - so that changing the other +# scripts which are needed much later will not invalidate the docker layer here +COPY scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh /scripts/docker/ -COPY scripts/docker/*.sh /scripts/docker/ RUN bash ./scripts/docker/install_mysql.sh dev \ && bash ./scripts/docker/install_mssql.sh ENV PATH=${PATH}:/opt/mssql-tools/bin @@ -203,13 +189,34 @@ RUN if [[ -f /docker-context-files/.pypirc ]]; then \ cp /docker-context-files/.pypirc /root/.pypirc; \ fi -ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ +ENV PATH=/.venv/bin:${PATH} \ + PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \ + AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ + AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES} \ AIRFLOW_VERSION=${AIRFLOW_VERSION} \ AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD} \ AIRFLOW_VERSION_SPECIFICATION=${AIRFLOW_VERSION_SPECIFICATION} \ AIRFLOW_SOURCES_FROM=${AIRFLOW_SOURCES_FROM} \ - AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO} + AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO} \ + AIRFLOW_CONSTRAINTS=${AIRFLOW_CONSTRAINTS} \ + AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE} \ + AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \ + DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \ + # Install Airflow in a virtual environment, so that we can copy the whole + # .venv folder to the final image + # from the build image and always in non-editable mode + AIRFLOW_INSTALL_EDITABLE_FLAG="" \ + AIRFLOW_REPO=${AIRFLOW_REPO} \ + AIRFLOW_BRANCH=${AIRFLOW_BRANCH} \ + AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} \ + CONSTRAINTS_GITHUB_REPOSITORY=${CONSTRAINTS_GITHUB_REPOSITORY} \ + UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} + + +# Copy all scripts required for installation - changing any of those should lead to +# rebuilding from here +COPY scripts/docker/*.sh /scripts/docker/ # In case of Production build image segment we want to pre-install main version of airflow # dependencies from GitHub so that we do not have to always reinstall it from the scratch. @@ -218,7 +225,8 @@ ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \ # the cache is only used when "upgrade to newer dependencies" is not set to automatically # account for removed dependencies (we do not install them in the first place) # Upgrade to specific PIP version -RUN bash /scripts/docker/install_pip_version.sh; \ +RUN bash /scripts/docker/create_venv.sh; \ + bash /scripts/docker/install_pip_version.sh; \ if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ @@ -264,19 +272,21 @@ RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \ if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ bash /scripts/docker/install_additional_dependencies.sh; \ fi; \ - find /root/.local/ -name '*.pyc' -print0 | xargs -0 rm -r || true ; \ - find /root/.local/ -type d -name '__pycache__' -print0 | xargs -0 rm -r || true ; \ - # make sure that all directories and files in .local are also group accessible - find /root/.local -executable -print0 | xargs --null chmod g+x; \ - find /root/.local -print0 | xargs --null chmod g+rw + find /.venv/ -name '*.pyc' -print0 | xargs -0 rm -r || true ; \ + find /.venv/ -type d -name '__pycache__' -print0 | xargs -0 rm -r || true ; \ + # make sure that all directories and files in .venv are also group accessible + find /.venv -executable -print0 | xargs --null chmod g+x; \ + find /.venv -print0 | xargs --null chmod g+rw # In case there is a requirements.txt file in "docker-context-files" it will be installed # during the build additionally to whatever has been installed so far. It is recommended that # the requirements.txt contains only dependencies with == version specification RUN if [[ -f /docker-context-files/requirements.txt ]]; then \ - pip install --no-cache-dir --user -r /docker-context-files/requirements.txt; \ + pip install --no-cache-dir -r /docker-context-files/requirements.txt; \ fi +# Those should be set and used as late as possible as any change in commit/build otherwise invalidates the +# layers right after ARG BUILD_ID ARG COMMIT_SHA ARG AIRFLOW_IMAGE_REPOSITORY @@ -382,12 +392,8 @@ ARG AIRFLOW_HOME # Having the variable in final image allows to disable providers manager warnings when # production image is prepared from sources rather than from package ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow" -ARG BUILD_ID -ARG COMMIT_SHA ARG AIRFLOW_IMAGE_REPOSITORY ARG AIRFLOW_IMAGE_DATE_CREATED -# By default PIP will install everything in ~/.local -ARG PIP_USER="true" ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \ ADDITIONAL_RUNTIME_APT_DEPS=${ADDITIONAL_RUNTIME_APT_DEPS} \ @@ -399,12 +405,9 @@ ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \ AIRFLOW__CORE__LOAD_EXAMPLES="false" \ AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \ AIRFLOW_HOME=${AIRFLOW_HOME} \ - PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH}" \ + PATH="/.venv/bin:${PATH}" \ GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm" \ - AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD} \ - BUILD_ID=${BUILD_ID} \ - COMMIT_SHA=${COMMIT_SHA} \ - PIP_USER=${PIP_USER} + AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD} # Note missing man directories on debian-buster # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 @@ -422,9 +425,8 @@ RUN mkdir -pv /usr/share/man/man1 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# Only copy install_m(y/s)sql and install_pip_version.sh. We do not need any other scripts in the final image. -COPY scripts/docker/install_mysql.sh /scripts/docker/install_mssql.sh scripts/docker/install_pip_version.sh \ - /scripts/docker/ +# Only copy install_m(y/s)sql. We do not need any other scripts in the final image. +COPY scripts/docker/install_mysql.sh /scripts/docker/install_mssql.sh /scripts/docker/ # fix permission issue in Azure DevOps when running the scripts RUN chmod a+x /scripts/docker/install_mysql.sh && \ @@ -441,25 +443,54 @@ RUN chmod a+x /scripts/docker/install_mysql.sh && \ find "${AIRFLOW_HOME}" -executable -print0 | xargs --null chmod g+x && \ find "${AIRFLOW_HOME}" -print0 | xargs --null chmod g+rw -COPY --chown=airflow:root --from=airflow-build-image /root/.local "${AIRFLOW_USER_HOME_DIR}/.local" +COPY --chown=airflow:root --from=airflow-build-image /.venv /.venv COPY --chown=airflow:root scripts/in_container/prod/entrypoint_prod.sh /entrypoint COPY --chown=airflow:root scripts/in_container/prod/clean-logs.sh /clean-logs # Make /etc/passwd root-group-writeable so that user can be dynamically added by OpenShift # See https://github.com/apache/airflow/issues/9248 +# Set default groups for airflow and root user RUN chmod a+x /entrypoint /clean-logs && \ - chmod g=u /etc/passwd && \ - bash /scripts/docker/install_pip_version.sh + chmod g=u /etc/passwd && \ + usermod -g 0 airflow -G 0 + +# make sure that the venv is activated for all users +# including plain sudo, sudo with --interactive flag +RUN sed --in-place=.bak "s/secure_path=\"/secure_path=\"\/.venv\/bin:/" /etc/sudoers + +# See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation +# to learn more about the way how signals are handled by the image +# Also emulate what virtualenv does (PYTHONHOME is not set in the image so no need to unset it) +ENV DUMB_INIT_SETSID="1" \ + VIRTUAL_ENV="/.venv" \ + PATH="/.venv/bin:${PATH}" \ + PS1="(airflow)" + +# This one is to workaround https://github.com/apache/airflow/issues/17546 +# issue with /usr/lib/x86_64-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block +# We do not yet a more "correct" solution to the problem but in order to avoid raising new issues +# by users of the prod image, we implement the workaround now. +# The side effect of this is slightly (in the range of 100s of milliseconds) slower load for any +# binary started and a little memory used for Heap allocated by initialization of libstdc++ +# This overhead is not happening for binaries that already link dynamically libstdc++ +ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libstdc++.so.6" WORKDIR ${AIRFLOW_HOME} EXPOSE 8080 -RUN usermod -g 0 airflow -G 0 - USER ${AIRFLOW_UID} +# Those should be set and used as late as possible as any change in commit/build otherwise invalidates the +# layers right after +ARG BUILD_ID +ARG COMMIT_SHA +ARG AIRFLOW_IMAGE_REPOSITORY +ARG AIRFLOW_IMAGE_DATE_CREATED + +ENV BUILD_ID=${BUILD_ID} COMMIT_SHA=${COMMIT_SHA} + LABEL org.apache.airflow.distro="debian" \ org.apache.airflow.distro.version="buster" \ org.apache.airflow.module="airflow" \ @@ -482,19 +513,5 @@ LABEL org.apache.airflow.distro="debian" \ org.opencontainers.image.title="Production Airflow Image" \ org.opencontainers.image.description="Reference, production-ready Apache Airflow image" - -# See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation -# to learn more about the way how signals are handled by the image -ENV DUMB_INIT_SETSID="1" - -# This one is to workaround https://github.com/apache/airflow/issues/17546 -# issue with /usr/lib/x86_64-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block -# We do not yet a more "correct" solution to the problem but in order to avoid raising new issues -# by users of the prod image, we implement the workaround now. -# The side effect of this is slightly (in the range of 100s of milliseconds) slower load for any -# binary started and a little memory used for Heap allocated by initialization of libstdc++ -# This overhead is not happening for binaries that already link dynamically libstdc++ -ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libstdc++.so.6" - ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] CMD [] diff --git a/Dockerfile.ci b/Dockerfile.ci index b4eb4653c5db0..d972025a722f2 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -102,7 +102,10 @@ RUN mkdir -pv /usr/share/man/man1 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY scripts/docker/*.sh /scripts/docker/ +# Only copy mysql/mssql installation scripts for now - so that changing the other +# scripts which are needed much later will not invalidate the docker layer here +COPY scripts/docker/common.sh scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh \ + /scripts/docker/ RUN bash /scripts/docker/install_mysql.sh dev \ && bash /scripts/docker/install_mssql.sh \ && adduser airflow \ @@ -225,7 +228,7 @@ ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true" # By default in the image, we are installing all providers when installing from sources ARG INSTALL_PROVIDERS_FROM_SOURCES="true" ARG INSTALL_FROM_PYPI="true" -ARG AIRFLOW_PIP_VERSION=21.2.4 +ARG AIRFLOW_PIP_VERSION=21.3.1 # Setup PIP # By default PIP install run without cache to make image smaller ARG PIP_NO_CACHE_DIR="true" @@ -260,7 +263,6 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ INSTALL_MYSQL_CLIENT="true" \ INSTALL_MSSQL_CLIENT="true" \ AIRFLOW_INSTALLATION_METHOD="." \ - AIRFLOW_INSTALL_USER_FLAG="" \ AIRFLOW_INSTALL_EDITABLE_FLAG="--editable" \ AIRFLOW_VERSION_SPECIFICATION="" \ PIP_NO_CACHE_DIR=${PIP_NO_CACHE_DIR} \ @@ -280,6 +282,12 @@ ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \ UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} +# Copy all scripts required for installation - changing any of those should lead to +# rebuilding from here +COPY scripts/docker/*.sh /scripts/docker/ + +# We are first creating a venv where all python packages and .so binaries needed by those are +# installed. # In case of CI builds we want to pre-install main version of airflow dependencies so that # We do not have to always reinstall it from the scratch. # And is automatically reinstalled from the scratch every time patch release of python gets released @@ -287,12 +295,20 @@ ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENT # are uninstalled, only dependencies remain. # the cache is only used when "upgrade to newer dependencies" is not set to automatically # account for removed dependencies (we do not install them in the first place) -RUN bash /scripts/docker/install_pip_version.sh; \ +# NOTE: we use `bash` directly instead of calling the script because in case of some +# filesystem (specifically on Azure) the underlying filesystem might not have executable +# flag capabilities and the scripts copied to Dockerfile might lack the +x flag. +RUN bash /scripts/docker/create_venv.sh; \ + export PATH=/.venv/bin:${PATH}; \ + bash /scripts/docker/install_pip_version.sh ; \ if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi +# Set venv to be enabled by default +ENV PATH=/.venv/bin:${PATH} + # Generate random hex dump file so that we can determine whether it's faster to rebuild the image # using current cache (when our dump is the same as the remote onb) or better to pull # the new image (when it is different) @@ -371,11 +387,26 @@ ARG BUILD_ID ARG COMMIT_SHA ARG AIRFLOW_IMAGE_DATE_CREATED -ENV PATH="/files/bin/:/opt/airflow/scripts/in_container/bin/:${HOME}:${PATH}" \ +ENV PATH="/files/bin/:/opt/airflow/scripts/in_container/bin/:${PATH}" \ GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm/" \ BUILD_ID=${BUILD_ID} \ COMMIT_SHA=${COMMIT_SHA} +# This one is to workaround https://github.com/apache/airflow/issues/17546 +# issue with /usr/lib/x86_64-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block +# We do not yet a more "correct" solution to the problem but in order to avoid raising new issues +# by users of the prod image, we implement the workaround now. +# The side effect of this is slightly (in the range of 100s of milliseconds) slower load for any +# binary started and a little memory used for Heap allocated by initialization of libstdc++ +# This overhead is not happening for binaries that already link dynamically libstdc++ +ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libstdc++.so.6" + +# make sure that the venv is activated for root user +RUN echo ". /.venv/bin/activate" > /root/.bashrc && \ + sed --in-place=.bak "s/secure_path=\"/secure_path=\"\/.venv\/bin:/" /etc/sudoers + +EXPOSE 8080 + LABEL org.apache.airflow.distro="debian" \ org.apache.airflow.distro.version="buster" \ org.apache.airflow.module="airflow" \ @@ -400,16 +431,5 @@ LABEL org.apache.airflow.distro="debian" \ org.opencontainers.image.title="Continuous Integration Airflow Image" \ org.opencontainers.image.description="Installed Apache Airflow with Continuous Integration dependencies" -# This one is to workaround https://github.com/apache/airflow/issues/17546 -# issue with /usr/lib/x86_64-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block -# We do not yet a more "correct" solution to the problem but in order to avoid raising new issues -# by users of the prod image, we implement the workaround now. -# The side effect of this is slightly (in the range of 100s of milliseconds) slower load for any -# binary started and a little memory used for Heap allocated by initialization of libstdc++ -# This overhead is not happening for binaries that already link dynamically libstdc++ -ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libstdc++.so.6" - - -EXPOSE 8080 - ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] +CMD [] diff --git a/IMAGES.rst b/IMAGES.rst index 559ecdbd12545..6747728b33d7f 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -492,7 +492,7 @@ The following build arguments (``--build-arg`` in docker build command) can be u | ``ADDITIONAL_RUNTIME_APT_ENV`` | | Additional env variables defined | | | | when installing runtime deps | +------------------------------------------+------------------------------------------+------------------------------------------+ -| ``AIRFLOW_PIP_VERSION`` | ``21.2.4`` | PIP version used. | +| ``AIRFLOW_PIP_VERSION`` | ``21.3.1`` | PIP version used. | +------------------------------------------+------------------------------------------+------------------------------------------+ | ``PIP_PROGRESS_BAR`` | ``on`` | Progress bar for PIP installation | +------------------------------------------+------------------------------------------+------------------------------------------+ diff --git a/airflow/operators/python.py b/airflow/operators/python.py index 3e1fcbcb4242e..3dacf671f489d 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -295,6 +295,15 @@ class PythonVirtualenvOperator(PythonOperator): system_site_packages in your virtualenv. See virtualenv documentation for more information. :type system_site_packages: bool + :param clone_airflow_virtualenv: If airflow is installed via virtualenv + It will clone airflow's virtualenv when preparing the new one. By default + virtualenv does not copy packages installed in virtualenv, but in some cases + you might need to "initialize" the virtualenv with airflow and other packages + installed with airflow - for example when you want to use dill or when + you want to use airflow or custom macros in your callable. This only works when + no python version is specified (i.e. when the same python version is used for + virtualenv as used in Airflow. In case a Python version is specified, you have to + make sure that necessary packages are installed via requirements. :param op_args: A list of positional arguments to pass to python_callable. :type op_args: list :param op_kwargs: A dict of keyword arguments to pass to python_callable. @@ -355,6 +364,7 @@ def __init__( python_version: Optional[Union[str, int, float]] = None, use_dill: bool = False, system_site_packages: bool = True, + clone_airflow_virtualenv: bool = True, op_args: Optional[List] = None, op_kwargs: Optional[Dict] = None, string_args: Optional[Iterable[str]] = None, @@ -397,6 +407,8 @@ def __init__( self.requirements.append('lazy-object-proxy') if self.use_dill and 'dill' not in self.requirements: self.requirements.append('dill') + # Only allows to use clone_virtualenv_packages if no python version is specified + self.clone_virtualenv_packages = clone_airflow_virtualenv if not python_version else False self.pickling_library = dill if self.use_dill else pickle def execute(self, context: Dict): @@ -418,6 +430,7 @@ def execute_callable(self): python_bin=f'python{self.python_version}' if self.python_version else None, system_site_packages=self.system_site_packages, requirements=self.requirements, + clone_virtualenv_packages=self.clone_virtualenv_packages, ) self._write_args(input_filename) diff --git a/airflow/utils/python_virtualenv.py b/airflow/utils/python_virtualenv.py index 9344c6a3d647c..a5bb0cd98c51c 100644 --- a/airflow/utils/python_virtualenv.py +++ b/airflow/utils/python_virtualenv.py @@ -18,6 +18,7 @@ # """Utilities for creating a virtual environment""" import os +import shutil import sys from collections import deque from typing import List, Optional @@ -27,13 +28,22 @@ from airflow.utils.process_utils import execute_in_subprocess -def _generate_virtualenv_cmd(tmp_dir: str, python_bin: str, system_site_packages: bool) -> List[str]: - cmd = [sys.executable, '-m', 'virtualenv', tmp_dir] - if system_site_packages: - cmd.append('--system-site-packages') - if python_bin is not None: - cmd.append(f'--python={python_bin}') - return cmd +def _generate_virtualenv_cmd( + tmp_dir: str, python_bin: str, system_site_packages: bool, clone_virtualenv_packages: bool +) -> List[str]: + if clone_virtualenv_packages and sys.prefix != sys.base_prefix and not python_bin: + # Create virtualenv using virtualenv-clone command if we are in virtualenv and + # clone_virtualenv_packages is set and we are using same version of python + # as our virtualenv + cmd = ['virtualenv-clone', f'{sys.prefix}', tmp_dir] + return cmd + else: + cmd = [sys.executable, '-m', 'virtualenv', tmp_dir] + if system_site_packages: + cmd.append('--system-site-packages') + if python_bin is not None: + cmd.append(f'--python={python_bin}') + return cmd def _generate_pip_install_cmd(tmp_dir: str, requirements: List[str]) -> Optional[List[str]]: @@ -75,7 +85,11 @@ def remove_task_decorator(python_source: str, task_decorator_name: str) -> str: def prepare_virtualenv( - venv_directory: str, python_bin: str, system_site_packages: bool, requirements: List[str] + venv_directory: str, + python_bin: str, + system_site_packages: bool, + requirements: List[str], + clone_virtualenv_packages: bool = False, ) -> str: """ Creates a virtual environment and installs the additional python packages @@ -89,10 +103,17 @@ def prepare_virtualenv( :type system_site_packages: bool :param requirements: List of additional python packages :type requirements: List[str] + :param clone_virtualenv_packages: whether to clone aurflow virtualenv package if airflow is run + in virtualenv - default is False + :type clone_virtualenv_packages: bool :return: Path to a binary file with Python in a virtual environment. :rtype: str """ - virtualenv_cmd = _generate_virtualenv_cmd(venv_directory, python_bin, system_site_packages) + virtualenv_cmd = _generate_virtualenv_cmd( + venv_directory, python_bin, system_site_packages, clone_virtualenv_packages + ) + # Virtualenv-clone requires the directory to be non-existing + shutil.rmtree(path=venv_directory, ignore_errors=True) execute_in_subprocess(virtualenv_cmd) pip_cmd = _generate_pip_install_cmd(venv_directory, requirements) if pip_cmd: diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 9e37f819049e5..41751975a65fb 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -615,7 +615,7 @@ additional tools. Below is an example Dockerfile, which installs providers for G ```dockerfile FROM apache/airflow:2.0.0 -RUN pip install --upgrade --user apache-airflow-providers-google==2.0.0.rc1 +RUN pip install --user apache-airflow-providers-google==2.0.0.rc1 USER ${AIRFLOW_UID} ``` diff --git a/docs/apache-airflow/installation/installing-from-pypi.rst b/docs/apache-airflow/installation/installing-from-pypi.rst index 95d055210259a..33a7d7c9a4918 100644 --- a/docs/apache-airflow/installation/installing-from-pypi.rst +++ b/docs/apache-airflow/installation/installing-from-pypi.rst @@ -118,7 +118,7 @@ being installed. AIRFLOW_VERSION=|version| PYTHON_VERSION="$(python --version | cut -d " " -f 2 | cut -d "." -f 1-2)" CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt" - pip install --upgrade "apache-airflow[postgres,google]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}" + pip install "apache-airflow[postgres,google]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}" Installation and upgrading of Airflow providers separately ========================================================== diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index f142b37a94cc3..ab75d4a0cab25 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -45,7 +45,7 @@ Those are the most common arguments that you use when you want to build a custom +------------------------------------------+------------------------------------------+---------------------------------------------+ | ``AIRFLOW_USER_HOME_DIR`` | ``/home/airflow`` | Home directory of the Airflow user. | +------------------------------------------+------------------------------------------+---------------------------------------------+ -| ``AIRFLOW_PIP_VERSION`` | ``21.2.4`` | PIP version used. | +| ``AIRFLOW_PIP_VERSION`` | ``21.3.1`` | PIP version used. | +------------------------------------------+------------------------------------------+---------------------------------------------+ | ``PIP_PROGRESS_BAR`` | ``on`` | Progress bar for PIP installation | +------------------------------------------+------------------------------------------+---------------------------------------------+ diff --git a/docs/docker-stack/build.rst b/docs/docker-stack/build.rst index 335b9adc0581f..7cc454a2784f0 100644 --- a/docs/docker-stack/build.rst +++ b/docs/docker-stack/build.rst @@ -203,14 +203,22 @@ You should be aware, about a few things: `best practices of Dockerfiles `_ to make sure your image is lean and small. -* The PyPI dependencies in Apache Airflow are installed in the user library, of the "airflow" user, so - PIP packages are installed to ``~/.local`` folder as if the ``--user`` flag was specified when running PIP. - Note also that using ``--no-cache-dir`` is a good idea that can help to make your image smaller. +* Using ``--no-cache-dir`` is a good idea that can help to make your image smaller. + +* The PyPI dependencies in Apache Airflow are installed in the ``/.venv`` folder. This is virtualenv where + airflow and all dependent packages are installed, following latest ``pip`` recommendations and upcoming + :pep:`668`. The ``PATH`` inside the image is set to point first + to the ``bin`` folder of the virtualenv, to make sure that this virtualenv is used, also safe PATH for sudo + is set to include that folder. The virtualenv is also activated at the entry of interactive session by + ``.bashrc`` files placed in ``HOME`` directory of both ``airflow`` user and ``root`` user. Note that all + the arbitrary users created dynamically to follow OpenShift rules are sharing the ``airflow`` user + ``HOME`` folder, so ``.bashrc`` file is the same for those users as well. .. note:: - Only as of ``2.0.1`` image the ``--user`` flag is turned on by default by setting ``PIP_USER`` environment + As of the ``2.0.1`` image the ``--user`` flag was turned on by default by setting ``PIP_USER`` environment variable to ``true``. This can be disabled by un-setting the variable or by setting it to ``false``. In the - 2.0.0 image you had to add the ``--user`` flag as ``pip install --user`` command. + 2.0.0 image you had to add the ``--user`` flag as ``pip install --user`` command. This flag and environment + variables however were removed in ``2.2`` in favor of virtualenv in ``/.venv`` folder. * If your apt, or PyPI dependencies require some of the ``build-essential`` or other packages that need to compile your python dependencies, then your best choice is to follow the "Customize the image" route, diff --git a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh index a97f6929e1716..61cee52037eef 100755 --- a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh +++ b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh @@ -87,7 +87,7 @@ function create_virtualenv() { . "${virtualenv_path}/bin/activate" - pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}" + pip install "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}" local constraints=( --constraint diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index c819bb493d2bd..6664def21f2c2 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -418,7 +418,7 @@ function initialization::initialize_image_build_variables() { export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv" - AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.2.4"} + AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.3.1"} export AIRFLOW_PIP_VERSION # We also pin version of wheel used to get consistent builds diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh index d11715efc5509..b929d50e0c97e 100755 --- a/scripts/docker/common.sh +++ b/scripts/docker/common.sh @@ -17,13 +17,11 @@ # under the License. set -euo pipefail -test -v INSTALL_MYSQL_CLIENT -test -v INSTALL_MSSQL_CLIENT -test -v AIRFLOW_INSTALL_USER_FLAG -test -v AIRFLOW_REPO -test -v AIRFLOW_BRANCH -test -v AIRFLOW_PIP_VERSION - +: "${INSTALL_MYSQL_CLIENT:?Should be true or false}" +: "${INSTALL_MSSQL_CLIENT:?Should be true or false}" +: "${AIRFLOW_REPO:?Should be set}" +: "${AIRFLOW_BRANCH:?Should be set}" +: "${AIRFLOW_PIP_VERSION:?Should be set}" set -x function common::get_airflow_version_specification() { @@ -60,3 +58,9 @@ function common::get_constraints_location() { AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS}-${python_version}.txt" fi } + +function common::show_pip_version_and_location() { + echo "PATH=${PATH}" + echo "PIP on path: $(which pip)" + echo "Using pip: $(pip --version)" +} diff --git a/scripts/docker/compile_www_assets.sh b/scripts/docker/compile_www_assets.sh index 59a7017fd157f..42f916f3cdbf6 100755 --- a/scripts/docker/compile_www_assets.sh +++ b/scripts/docker/compile_www_assets.sh @@ -32,7 +32,7 @@ function compile_www_assets() { # In case we are building from sources in production image, we should build the assets www_dir="${AIRFLOW_SOURCES_TO}/airflow/www" else - www_dir="$(python -m site --user-site)/airflow/www" + www_dir="$(python -m site)/airflow/www" fi pushd ${www_dir} || exit 1 yarn install --frozen-lockfile --no-cache diff --git a/scripts/docker/create_venv.sh b/scripts/docker/create_venv.sh new file mode 100755 index 0000000000000..335b6ae90c422 --- /dev/null +++ b/scripts/docker/create_venv.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Sets up virtualenv for airflow installation + +# shellcheck disable=SC2086 +# shellcheck source=scripts/docker/common.sh +. "$( dirname "${BASH_SOURCE[0]}" )/common.sh" + +# the whole .venv directory and have everything installed +# there and to be sure we have a clear status of the env +# Also we explicitly name this venv `airflow` for +# command prompt +function create_venv() { + python3 -m venv "/.venv" --copies --clear --prompt airflow +} + +create_venv diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh index 4f9c05f6b7680..27fc5d387c922 100755 --- a/scripts/docker/install_additional_dependencies.sh +++ b/scripts/docker/install_additional_dependencies.sh @@ -18,11 +18,10 @@ # shellcheck disable=SC2086 set -euo pipefail -test -v UPGRADE_TO_NEWER_DEPENDENCIES -test -v ADDITIONAL_PYTHON_DEPS -test -v EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS -test -v AIRFLOW_INSTALL_USER_FLAG -test -v AIRFLOW_PIP_VERSION +: "${UPGRADE_TO_NEWER_DEPENDENCIES:?Should be true or false}" +: "${ADDITIONAL_PYTHON_DEPS:?Should be set}" +: "${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS:?Should be set}" +: "${AIRFLOW_PIP_VERSION:?Should be set}" # shellcheck source=scripts/docker/common.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" @@ -36,20 +35,19 @@ function install_additional_dependencies() { echo echo Installing additional dependencies while upgrading to newer dependencies echo - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy eager \ + pip install --upgrade --upgrade-strategy eager \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" pip check else echo echo Installing additional dependencies upgrading only if needed echo - pip install ${AIRFLOW_INSTALL_USER_FLAG} \ - --upgrade --upgrade-strategy only-if-needed \ + pip install --upgrade --upgrade-strategy only-if-needed \ ${ADDITIONAL_PYTHON_DEPS} # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" pip check fi } @@ -57,5 +55,6 @@ function install_additional_dependencies() { common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location +common::show_pip_version_and_location install_additional_dependencies diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh index 61a30c42c12d9..428eab01a0530 100755 --- a/scripts/docker/install_airflow.sh +++ b/scripts/docker/install_airflow.sh @@ -47,7 +47,7 @@ function install_airflow() { echo Installing all packages with eager upgrade echo # eager upgrade - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy eager \ + pip install --upgrade --upgrade-strategy eager \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} if [[ -n "${AIRFLOW_INSTALL_EDITABLE_FLAG}" ]]; then @@ -59,23 +59,23 @@ function install_airflow() { fi # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" pip check else \ echo echo Installing all packages with constraints and upgrade if needed echo - pip install ${AIRFLOW_INSTALL_USER_FLAG} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ + pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" # then upgrade if needed without using constraints to account for new limits in setup.py - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy only-if-needed \ + pip install --upgrade --upgrade-strategy only-if-needed \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" pip check fi @@ -84,5 +84,6 @@ function install_airflow() { common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location +common::show_pip_version_and_location install_airflow diff --git a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh index 61aaa13ef467f..b3790f002c29a 100755 --- a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh +++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -39,11 +39,11 @@ function install_airflow_dependencies_from_branch_tip() { fi # Install latest set of dependencies using constraints. In case constraints were upgraded and there # are conflicts, this might fail, but it should be fixed in the following installation steps - pip install ${AIRFLOW_INSTALL_USER_FLAG} \ + pip install \ "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes || true echo echo Uninstalling just airflow. Dependencies remain. @@ -54,5 +54,6 @@ function install_airflow_dependencies_from_branch_tip() { common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location +common::show_pip_version_and_location install_airflow_dependencies_from_branch_tip diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index d8ed6bc72bd9a..7fc0af064dc64 100755 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -34,7 +34,6 @@ function install_airflow_and_providers_from_docker_context_files(){ local pip_flags=( # Don't quote this -- if it is empty we don't want it to create an # empty array element - ${AIRFLOW_INSTALL_USER_FLAG} --find-links="file:///docker-context-files" ) @@ -88,14 +87,14 @@ function install_airflow_and_providers_from_docker_context_files(){ --constraint /tmp/constraints.txt rm /tmp/constraints.txt # make sure correct PIP version is used \ - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" # then upgrade if needed without using constraints to account for new limits in setup.py - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy only-if-needed \ + pip install --upgrade --upgrade-strategy only-if-needed \ ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} fi # make sure correct PIP version is left installed - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" pip check } @@ -104,7 +103,7 @@ function install_airflow_and_providers_from_docker_context_files(){ # without dependencies. This is extremely useful in case you want to install via pip-download # method on air-gaped system where you do not want to download any dependencies from remote hosts # which is a requirement for serious installations -install_all_other_packages_from_docker_context_files() { +function install_all_other_packages_from_docker_context_files() { echo echo Force re-installing all other package from local files without dependencies echo @@ -113,15 +112,18 @@ install_all_other_packages_from_docker_context_files() { reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz} 2>/dev/null | \ grep -v apache_airflow | grep -v apache-airflow || true) if [[ -n "${reinstalling_other_packages}" ]]; then \ - pip install ${AIRFLOW_INSTALL_USER_FLAG} --force-reinstall --no-deps --no-index ${reinstalling_other_packages} + pip install --force-reinstall --no-deps --no-index ${reinstalling_other_packages} # make sure correct PIP version is used - pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}" + pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" fi } common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location +common::show_pip_version_and_location install_airflow_and_providers_from_docker_context_files + +common::show_pip_version_and_location install_all_other_packages_from_docker_context_files diff --git a/scripts/docker/install_pip_version.sh b/scripts/docker/install_pip_version.sh index 6e0c3c1211dda..f610b24fea794 100755 --- a/scripts/docker/install_pip_version.sh +++ b/scripts/docker/install_pip_version.sh @@ -30,11 +30,12 @@ . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" function install_pip_version() { - pip install --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" && mkdir -p /root/.local/bin + pip install --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" } common::get_airflow_version_specification common::override_pip_version_if_needed common::get_constraints_location +common::show_pip_version_and_location install_pip_version diff --git a/scripts/in_container/prod/entrypoint_prod.sh b/scripts/in_container/prod/entrypoint_prod.sh index a7623b697b331..6ae2736e46a26 100755 --- a/scripts/in_container/prod/entrypoint_prod.sh +++ b/scripts/in_container/prod/entrypoint_prod.sh @@ -164,14 +164,8 @@ function create_system_user_if_missing() { fi } -function set_pythonpath_for_root_user() { - # Airflow is installed as a local user application which means that if the container is running as root - # the application is not available. because Python then only load system-wide applications. - # Now also adds applications installed as local user "airflow". +function warn_root_user() { if [[ $UID == "0" ]]; then - local python_major_minor - python_major_minor="$(python --version | cut -d " " -f 2 | cut -d "." -f 1-2)" - export PYTHONPATH="${AIRFLOW_USER_HOME_DIR}/.local/lib/python${python_major_minor}/site-packages:${PYTHONPATH:-}" >&2 echo "The container is run as root user. For security, consider using a regular user account." fi } @@ -243,12 +237,6 @@ function check_uid_gid() { fi } -# In Airflow image we are setting PIP_USER variable to true, in order to install all the packages -# by default with the ``--user`` flag. However this is a problem if a virtualenv is created later -# which happens in PythonVirtualenvOperator. We are unsetting this variable here, so that it is -# not set when PIP is run by Airflow later on -unset PIP_USER - check_uid_gid # Set umask to 0002 to make all the directories created by the current user group-writeable @@ -264,8 +252,9 @@ CONNECTION_CHECK_SLEEP_TIME=${CONNECTION_CHECK_SLEEP_TIME:=3} readonly CONNECTION_CHECK_SLEEP_TIME create_system_user_if_missing -set_pythonpath_for_root_user -if [[ "${CONNECTION_CHECK_MAX_COUNT}" -gt "0" ]]; then +warn_root_user +if [[ "${CONNECTION_CHECK_MAX_COUNT}" -gt "0" && ${AIRFLOW_COMMAND} != "" \ + && ${AIRFLOW_COMMAND} != "python" && ${AIRFLOW_COMMAND} != "bash" ]]; then wait_for_airflow_db fi @@ -290,7 +279,7 @@ if [[ -n "${_PIP_ADDITIONAL_REQUIREMENTS=}" ]] ; then >&2 echo " the container starts, so it is onlny useful for testing and trying out" >&2 echo " of adding dependencies." >&2 echo - pip install --no-cache-dir --user ${_PIP_ADDITIONAL_REQUIREMENTS} + pip install --no-cache-dir ${_PIP_ADDITIONAL_REQUIREMENTS} fi diff --git a/setup.cfg b/setup.cfg index 5f503d4586d02..5d7c6ffc128f6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -158,6 +158,7 @@ install_requires = termcolor>=1.1.0 typing-extensions>=3.7.4;python_version<"3.8" unicodecsv>=0.14.1 + virtualenv-clone werkzeug~=1.0, >=1.0.1 # SQLA still imports the compat wtforms<3.0.0 diff --git a/tests/operators/test_python.py b/tests/operators/test_python.py index dcf160109d7f9..88694cb1dc755 100644 --- a/tests/operators/test_python.py +++ b/tests/operators/test_python.py @@ -708,6 +708,10 @@ def test_clear_skipped_downstream_task(self): class TestPythonVirtualenvOperator(unittest.TestCase): + @pytest.fixture(autouse=True) + def inject_fixtures(self, caplog): + self._caplog = caplog + def setUp(self): super().setUp() self.dag = DAG( @@ -730,7 +734,6 @@ def tearDown(self): session.query(TI).delete() def _run_as_operator(self, fn, python_version=sys.version_info[0], **kwargs): - task = PythonVirtualenvOperator( python_callable=fn, python_version=python_version, task_id='task', dag=self.dag, **kwargs ) @@ -936,7 +939,60 @@ def f( ): pass - self._run_as_operator(f, use_dill=True, system_site_packages=True, requirements=None) + self._run_as_operator( + f, use_dill=True, system_site_packages=True, requirements=None, python_version=None + ) + + @pytest.mark.skipif( + sys.prefix == sys.base_prefix, reason="Should only be run if airflow installed in venv" + ) + def test_clone_virtualenv_disabled(self): + def f( + **kwargs, + ): + pass + + with pytest.raises(Exception, match="returned non-zero exit status"): + self._run_as_operator( + f, + use_dill=True, + system_site_packages=True, + clone_airflow_virtualenv=False, + python_version=None, + ) + assert "No module named 'dill'" in self._caplog.text + + @pytest.mark.skipif( + sys.prefix == sys.base_prefix, reason="Should only be run if airflow installed in venv" + ) + def test_clone_virtualenv_enabled_with_python(self): + def f( + **kwargs, + ): + pass + + with pytest.raises(Exception, match="returned non-zero exit status"): + self._run_as_operator( + f, + use_dill=True, + system_site_packages=True, + clone_airflow_virtualenv=True, + python_version=sys.version_info[0], + ) + assert "No module named 'dill'" in self._caplog.text + + # This tests might take longer than default 60 seconds as it is serializing a lot of + # context using dill (which is slow apparently). + @pytest.mark.execution_timeout(120) + def test_clone_virtualenv_enabled(self): + def f( + **kwargs, + ): + pass + + self._run_as_operator( + f, use_dill=True, system_site_packages=True, clone_airflow_virtualenv=True, python_version=None + ) def test_pendulum_context(self): def f( From 8644ea7570181982de7a83b2f27898a98825ade9 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 25 Oct 2021 16:41:01 -0100 Subject: [PATCH 2/2] Optimizes building of Production image for non-modified www files When none of the `www` files changed, we do not have to compile assets. Unfortunately the old Dockerfile run COPY for all sources before the asset compilation, so whenever any file changed in sources, the asset compilation was triggered. This change fixes it in the way that assets are compiled first, the dist files are stored on the side and copied back after all sources are copied again (this way they will not get overwritten). This is done in the "build" segment of the image so it has no impact of the size of the "final" image. It will speed up both developing changes for the image as well as building PROD image in CI when none of the www files change (which is very, very often) --- Dockerfile | 43 ++++++++++++++++++++----- breeze | 8 +++++ docs/docker-stack/build-arg-ref.rst | 10 ++++++ scripts/ci/libraries/_build_images.sh | 6 +++- scripts/ci/libraries/_initialization.sh | 9 ++++++ 5 files changed, 67 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index d26a40a25924c..6dc51b133e1f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -159,13 +159,21 @@ ARG INSTALL_PROVIDERS_FROM_SOURCES="false" # But it also can be `.` from local installation or GitHub URL pointing to specific branch or tag # Of Airflow. Note That for local source installation you need to have local sources of # Airflow checked out together with the Dockerfile and AIRFLOW_SOURCES_FROM and AIRFLOW_SOURCES_TO -# set to "." and "/opt/airflow" respectively. +# set to "." and "/opt/airflow" respectively. Similarly AIRFLOW_SOURCES_WWW_FROM/TO are set to right source +# and destination ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow" # By default latest released version of airflow is installed (when empty) but this value can be overridden # and we can install version according to specification (For example ==2.0.2 or <3.0.0). ARG AIRFLOW_VERSION_SPECIFICATION="" # By default we do not upgrade to latest dependencies ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" +# By default we install latest airflow from PyPI so we do not need to copy sources of Airflow +# www to compile the assets but in case of breeze/CI builds we use latest sources and we override those +# those SOURCES_FROM/TO with "airflow/www" and "/opt/airflow/airflow/www" respectively. +# This is to rebuild the assets only when any of the www sources change +ARG AIRFLOW_SOURCES_WWW_FROM="empty" +ARG AIRFLOW_SOURCES_WWW_TO="/empty" + # By default we install latest airflow from PyPI so we do not need to copy sources of Airflow # but in case of breeze/CI builds we use latest sources and we override those # those SOURCES_FROM/TO with "." and "/opt/airflow" respectively @@ -216,7 +224,9 @@ ENV PATH=/.venv/bin:${PATH} \ # Copy all scripts required for installation - changing any of those should lead to # rebuilding from here -COPY scripts/docker/*.sh /scripts/docker/ +COPY scripts/docker/common.sh scripts/docker/create_venv.sh \ + scripts/docker/install_pip_version.sh /scripts/docker/install_airflow_dependencies_from_branch_tip.sh \ + /scripts/docker/ # In case of Production build image segment we want to pre-install main version of airflow # dependencies from GitHub so that we do not have to always reinstall it from the scratch. @@ -232,8 +242,26 @@ RUN bash /scripts/docker/create_venv.sh; \ bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi +COPY scripts/docker/compile_www_assets.sh /scripts/docker/ + +COPY ${AIRFLOW_SOURCES_WWW_FROM} ${AIRFLOW_SOURCES_WWW_TO} + +# hadolint ignore=SC2086, SC2010 +RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \ + # only compile assets if the prod image is build from sources + # otherwise they are already compiled-in + bash /scripts/docker/compile_www_assets.sh; \ + # Copy generated dist folder (otherwise it will be overridden by the COPY step below) + mv -f /opt/airflow/airflow/www/static/dist /tmp/dist; \ + fi; + COPY ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO} +# Copy back the generated dist folder +RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \ + mv -f /tmp/dist /opt/airflow/airflow/www/static/dist; \ + fi; + # Add extra python dependencies ARG ADDITIONAL_PYTHON_DEPS="" # We can set this value to true in case we want to install .whl .tar.gz packages placed in the @@ -258,13 +286,12 @@ ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \ WORKDIR /opt/airflow +COPY scripts/docker/install_from_docker_context_files.sh scripts/docker/install_airflow.sh \ + scripts/docker/install_additional_dependencies.sh \ + /scripts/docker/ + # hadolint ignore=SC2086, SC2010 -RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \ - # only compile assets if the prod image is build from sources - # otherwise they are already compiled-in - bash /scripts/docker/compile_www_assets.sh; \ - fi; \ - if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \ +RUN if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \ bash /scripts/docker/install_from_docker_context_files.sh; \ elif [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \ bash /scripts/docker/install_airflow.sh; \ diff --git a/breeze b/breeze index 3b100db8ac9a2..4d7d8ecaf3fb1 100755 --- a/breeze +++ b/breeze @@ -149,6 +149,14 @@ function breeze::setup_default_breeze_constants() { AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO:="/opt/airflow"} export AIRFLOW_SOURCES_TO + # WWW compilation sources by default are installed from local airflow/www when using breeze + AIRFLOW_SOURCES_WWW_FROM=${AIRFLOW_SOURCES_WWW_FROM:="./airflow/www"} + export AIRFLOW_SOURCES_WWW_FROM + + # They are copied to /opt/airflow/airflow/www by default in breeze + AIRFLOW_SOURCES_WWW_TO=${AIRFLOW_SOURCES_WWW_TO:="/opt/airflow/airflow/www"} + export AIRFLOW_SOURCES_WWW_TO + # Unlike in CI scripts, in breeze by default production image is installed from sources export AIRFLOW_INSTALLATION_METHOD="." diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index ab75d4a0cab25..1c298f1b0abb3 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -198,6 +198,16 @@ You can see some examples of those in: | | | "/opt/airflow" when you install Airflow | | | | from local sources. | +------------------------------------------+------------------------------------------+------------------------------------------+ +| ``AIRFLOW_SOURCES_WWW_FROM`` | ``empty`` | Sources of Airflow WWW files used for | +| | | asset compilation. Set it to | +| | | "./airflow/www" when | +| | | you install Airflow from local sources | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``AIRFLOW_SOURCES_WWW_TO`` | ``/empty`` | Target for Airflow files used for | +| | | asset compilation. Set it to | +| | | "/opt/airflow/airflow/www" when | +| | | you install Airflow from local sources. | ++------------------------------------------+------------------------------------------+------------------------------------------+ | ``AIRFLOW_VERSION_SPECIFICATION`` | | Optional - might be used for using limit | | | | for Airflow version installation - for | | | | example ``<2.0.2`` for automated builds. | diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index dadf81fa1fcd3..ddf4452a64973 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -20,9 +20,11 @@ # pass build flags depending on the version and method of the installation (for example to # get proper requirement constraint files) function build_images::add_build_args_for_remote_install() { - # entrypoint is used as AIRFLOW_SOURCES_FROM/TO in order to avoid costly copying of all sources of + # entrypoint is used as AIRFLOW_SOURCES_(WWW)_FROM/TO in order to avoid costly copying of all sources of # Airflow - those are not needed for remote install at all. Entrypoint is later overwritten by EXTRA_DOCKER_PROD_BUILD_FLAGS+=( + "--build-arg" "AIRFLOW_SOURCES_WWW_FROM=empty" + "--build-arg" "AIRFLOW_SOURCES_WWW_TO=/empty" "--build-arg" "AIRFLOW_SOURCES_FROM=empty" "--build-arg" "AIRFLOW_SOURCES_TO=/empty" ) @@ -738,6 +740,8 @@ function build_images::prepare_prod_build() { EXTRA_DOCKER_PROD_BUILD_FLAGS=( "--build-arg" "AIRFLOW_SOURCES_FROM=${AIRFLOW_SOURCES_FROM}" "--build-arg" "AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO}" + "--build-arg" "AIRFLOW_SOURCES_WWW_FROM=${AIRFLOW_SOURCES_WWW_FROM}" + "--build-arg" "AIRFLOW_SOURCES_WWW_TO=${AIRFLOW_SOURCES_WWW_TO}" "--build-arg" "AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD}" "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH}" ) diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 6664def21f2c2..ed6925f98b224 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -436,6 +436,13 @@ function initialization::initialize_image_build_variables() { AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO:="/empty"} export AIRFLOW_SOURCES_TO + # By default no sources are copied to image + AIRFLOW_SOURCES_WWW_FROM=${AIRFLOW_SOURCES_WWW_FROM:="empty"} + export AIRFLOW_SOURCES_WWW_FROM + + AIRFLOW_SOURCES_WWW_TO=${AIRFLOW_SOURCES_WWW_TO:="/empty"} + export AIRFLOW_SOURCES_WWW_TO + # By default in scripts production docker image is installed from PyPI package export AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD:="apache-airflow"} @@ -708,6 +715,8 @@ Production image build variables: AIRFLOW_VERSION_SPECIFICATION: '${AIRFLOW_VERSION_SPECIFICATION}' AIRFLOW_SOURCES_FROM: '${AIRFLOW_SOURCES_FROM}' AIRFLOW_SOURCES_TO: '${AIRFLOW_SOURCES_TO}' + AIRFLOW_SOURCES_WWW_FROM: '${AIRFLOW_SOURCES_WWW_FROM}' + AIRFLOW_SOURCES_WWW_TO: '${AIRFLOW_SOURCES_WWW_TO}' Detected GitHub environment: