Optimizes building of Production image for non-modified www files

Dockerfile.ci

@@ -102,7 +102,10 @@ RUN mkdir -pv /usr/share/man/man1 \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-COPY scripts/docker/*.sh /scripts/docker/
+# Only copy mysql/mssql installation scripts for now - so that changing the other
+# scripts which are needed much later will not invalidate the docker layer here
+COPY scripts/docker/common.sh scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh \
+    /scripts/docker/
 RUN bash /scripts/docker/install_mysql.sh dev \
     && bash /scripts/docker/install_mssql.sh \
     && adduser airflow \
@@ -225,7 +228,7 @@ ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
 # By default in the image, we are installing all providers when installing from sources
 ARG INSTALL_PROVIDERS_FROM_SOURCES="true"
 ARG INSTALL_FROM_PYPI="true"
-ARG AIRFLOW_PIP_VERSION=21.2.4
+ARG AIRFLOW_PIP_VERSION=21.3.1
 # Setup PIP
 # By default PIP install run without cache to make image smaller
 ARG PIP_NO_CACHE_DIR="true"
@@ -260,7 +263,6 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
     INSTALL_MYSQL_CLIENT="true" \
     INSTALL_MSSQL_CLIENT="true" \
     AIRFLOW_INSTALLATION_METHOD="." \
-    AIRFLOW_INSTALL_USER_FLAG="" \
     AIRFLOW_INSTALL_EDITABLE_FLAG="--editable" \
     AIRFLOW_VERSION_SPECIFICATION="" \
     PIP_NO_CACHE_DIR=${PIP_NO_CACHE_DIR} \
@@ -280,19 +282,33 @@ ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
 ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \
     UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}
 
+# Copy all scripts required for installation - changing any of those should lead to
+# rebuilding from here
+COPY scripts/docker/*.sh /scripts/docker/
+
+# We are first creating a venv where all python packages and .so binaries needed by those are
+# installed.
 # In case of CI builds we want to pre-install main version of airflow dependencies so that
 # We do not have to always reinstall it from the scratch.
 # And is automatically reinstalled from the scratch every time patch release of python gets released
 # The Airflow (and providers in case INSTALL_PROVIDERS_FROM_SOURCES is "false")
 # are uninstalled, only dependencies remain.
 # the cache is only used when "upgrade to newer dependencies" is not set to automatically
 # account for removed dependencies (we do not install them in the first place)
-RUN bash /scripts/docker/install_pip_version.sh; \
+# NOTE: we use `bash` directly instead of calling the script because in case of some
+# filesystem (specifically on Azure) the underlying filesystem might not have executable
+# flag capabilities and the scripts copied to Dockerfile might lack the +x flag.
+RUN bash /scripts/docker/create_venv.sh; \
+    export PATH=/.venv/bin:${PATH}; \
+    bash /scripts/docker/install_pip_version.sh ; \
     if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \
           ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \
         bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
     fi
 
+# Set venv to be enabled by default
+ENV PATH=/.venv/bin:${PATH}
+
 # Generate random hex dump file so that we can determine whether it's faster to rebuild the image
 # using current cache (when our dump is the same as the remote onb) or better to pull
 # the new image (when it is different)
@@ -371,11 +387,26 @@ ARG BUILD_ID
 ARG COMMIT_SHA
 ARG AIRFLOW_IMAGE_DATE_CREATED
 
-ENV PATH="/files/bin/:/opt/airflow/scripts/in_container/bin/:${HOME}:${PATH}" \
+ENV PATH="/files/bin/:/opt/airflow/scripts/in_container/bin/:${PATH}" \
     GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm/" \
     BUILD_ID=${BUILD_ID} \
     COMMIT_SHA=${COMMIT_SHA}
 
+# This one is to workaround https://github.com/apache/airflow/issues/17546
+# issue with /usr/lib/x86_64-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block
+# We do not yet a more "correct" solution to the problem but in order to avoid raising new issues
+# by users of the prod image, we implement the workaround now.
+# The side effect of this is slightly (in the range of 100s of milliseconds) slower load for any
+# binary started and a little memory used for Heap allocated by initialization of libstdc++
+# This overhead is not happening for binaries that already link dynamically libstdc++
+ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libstdc++.so.6"
+
+# make sure that the venv is activated for root user
+RUN echo ". /.venv/bin/activate" > /root/.bashrc && \
+    sed --in-place=.bak "s/secure_path=\"/secure_path=\"\/.venv\/bin:/" /etc/sudoers
+
+EXPOSE 8080
+
 LABEL org.apache.airflow.distro="debian" \
   org.apache.airflow.distro.version="buster" \
   org.apache.airflow.module="airflow" \
@@ -400,16 +431,5 @@ LABEL org.apache.airflow.distro="debian" \
   org.opencontainers.image.title="Continuous Integration Airflow Image" \
   org.opencontainers.image.description="Installed Apache Airflow with Continuous Integration dependencies"
 
-# This one is to workaround https://github.com/apache/airflow/issues/17546
-# issue with /usr/lib/x86_64-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block
-# We do not yet a more "correct" solution to the problem but in order to avoid raising new issues
-# by users of the prod image, we implement the workaround now.
-# The side effect of this is slightly (in the range of 100s of milliseconds) slower load for any
-# binary started and a little memory used for Heap allocated by initialization of libstdc++
-# This overhead is not happening for binaries that already link dynamically libstdc++
-ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libstdc++.so.6"
-
-
-EXPOSE 8080
-
 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]
+CMD []

IMAGES.rst

@@ -492,7 +492,7 @@ The following build arguments (``--build-arg`` in docker build command) can be u
 | ``ADDITIONAL_RUNTIME_APT_ENV``           |                                          | Additional env variables defined         |
 |                                          |                                          | when installing runtime deps             |
 +------------------------------------------+------------------------------------------+------------------------------------------+
-| ``AIRFLOW_PIP_VERSION``                  | ``21.2.4``                               | PIP version used.                        |
+| ``AIRFLOW_PIP_VERSION``                  | ``21.3.1``                               | PIP version used.                        |
 +------------------------------------------+------------------------------------------+------------------------------------------+
 | ``PIP_PROGRESS_BAR``                     | ``on``                                   | Progress bar for PIP installation        |
 +------------------------------------------+------------------------------------------+------------------------------------------+

airflow/operators/python.py

@@ -295,6 +295,15 @@ class PythonVirtualenvOperator(PythonOperator):
         system_site_packages in your virtualenv.
         See virtualenv documentation for more information.
     :type system_site_packages: bool
+    :param clone_airflow_virtualenv: If airflow is installed via virtualenv
+        It will clone airflow's virtualenv when preparing the new one. By default
+        virtualenv does not copy packages installed in virtualenv, but in some cases
+        you might need to "initialize" the virtualenv with airflow and other packages
+        installed with airflow - for example when you want to use dill or when
+        you want to use airflow or custom macros in your callable. This only works when
+        no python version is specified (i.e. when the same python version is used for
+        virtualenv as used in Airflow. In case a Python version is specified, you have to
+        make sure that necessary packages are installed via requirements.
     :param op_args: A list of positional arguments to pass to python_callable.
     :type op_args: list
     :param op_kwargs: A dict of keyword arguments to pass to python_callable.
@@ -355,6 +364,7 @@ def __init__(
         python_version: Optional[Union[str, int, float]] = None,
         use_dill: bool = False,
         system_site_packages: bool = True,
+        clone_airflow_virtualenv: bool = True,
         op_args: Optional[List] = None,
         op_kwargs: Optional[Dict] = None,
         string_args: Optional[Iterable[str]] = None,
@@ -397,6 +407,8 @@ def __init__(
                 self.requirements.append('lazy-object-proxy')
             if self.use_dill and 'dill' not in self.requirements:
                 self.requirements.append('dill')
+        # Only allows to use clone_virtualenv_packages if no python version is specified
+        self.clone_virtualenv_packages = clone_airflow_virtualenv if not python_version else False
         self.pickling_library = dill if self.use_dill else pickle
 
     def execute(self, context: Dict):
@@ -418,6 +430,7 @@ def execute_callable(self):
                 python_bin=f'python{self.python_version}' if self.python_version else None,
                 system_site_packages=self.system_site_packages,
                 requirements=self.requirements,
+                clone_virtualenv_packages=self.clone_virtualenv_packages,
             )
 
             self._write_args(input_filename)

airflow/utils/python_virtualenv.py

@@ -18,6 +18,7 @@
 #
 """Utilities for creating a virtual environment"""
 import os
+import shutil
 import sys
 from collections import deque
 from typing import List, Optional
@@ -27,13 +28,22 @@
 from airflow.utils.process_utils import execute_in_subprocess
 
 
-def _generate_virtualenv_cmd(tmp_dir: str, python_bin: str, system_site_packages: bool) -> List[str]:
-    cmd = [sys.executable, '-m', 'virtualenv', tmp_dir]
-    if system_site_packages:
-        cmd.append('--system-site-packages')
-    if python_bin is not None:
-        cmd.append(f'--python={python_bin}')
-    return cmd
+def _generate_virtualenv_cmd(
+    tmp_dir: str, python_bin: str, system_site_packages: bool, clone_virtualenv_packages: bool
+) -> List[str]:
+    if clone_virtualenv_packages and sys.prefix != sys.base_prefix and not python_bin:
+        # Create virtualenv using virtualenv-clone command if we are in virtualenv and
+        # clone_virtualenv_packages is set and we are using same version of python
+        # as our virtualenv
+        cmd = ['virtualenv-clone', f'{sys.prefix}', tmp_dir]
+        return cmd
+    else:
+        cmd = [sys.executable, '-m', 'virtualenv', tmp_dir]
+        if system_site_packages:
+            cmd.append('--system-site-packages')
+        if python_bin is not None:
+            cmd.append(f'--python={python_bin}')
+        return cmd
 
 
 def _generate_pip_install_cmd(tmp_dir: str, requirements: List[str]) -> Optional[List[str]]:
@@ -75,7 +85,11 @@ def remove_task_decorator(python_source: str, task_decorator_name: str) -> str:
 
 
 def prepare_virtualenv(
-    venv_directory: str, python_bin: str, system_site_packages: bool, requirements: List[str]
+    venv_directory: str,
+    python_bin: str,
+    system_site_packages: bool,
+    requirements: List[str],
+    clone_virtualenv_packages: bool = False,
 ) -> str:
     """
     Creates a virtual environment and installs the additional python packages
@@ -89,10 +103,17 @@ def prepare_virtualenv(
     :type system_site_packages: bool
     :param requirements: List of additional python packages
     :type requirements: List[str]
+    :param clone_virtualenv_packages: whether to clone aurflow virtualenv package if airflow is run
+         in virtualenv - default is False
+    :type clone_virtualenv_packages: bool
     :return: Path to a binary file with Python in a virtual environment.
     :rtype: str
     """
-    virtualenv_cmd = _generate_virtualenv_cmd(venv_directory, python_bin, system_site_packages)
+    virtualenv_cmd = _generate_virtualenv_cmd(
+        venv_directory, python_bin, system_site_packages, clone_virtualenv_packages
+    )
+    # Virtualenv-clone requires the directory to be non-existing
+    shutil.rmtree(path=venv_directory, ignore_errors=True)
     execute_in_subprocess(virtualenv_cmd)
     pip_cmd = _generate_pip_install_cmd(venv_directory, requirements)
     if pip_cmd:

breeze

@@ -149,6 +149,14 @@ function breeze::setup_default_breeze_constants() {
     AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO:="/opt/airflow"}
     export AIRFLOW_SOURCES_TO
 
+    # WWW compilation sources by default are installed from local airflow/www when using breeze
+    AIRFLOW_SOURCES_WWW_FROM=${AIRFLOW_SOURCES_WWW_FROM:="./airflow/www"}
+    export AIRFLOW_SOURCES_WWW_FROM
+
+    # They are copied to /opt/airflow/airflow/www by default in breeze
+    AIRFLOW_SOURCES_WWW_TO=${AIRFLOW_SOURCES_WWW_TO:="/opt/airflow/airflow/www"}
+    export AIRFLOW_SOURCES_WWW_TO
+
     # Unlike in CI scripts, in breeze by default production image is installed from sources
     export AIRFLOW_INSTALLATION_METHOD="."
 

dev/README_RELEASE_PROVIDER_PACKAGES.md

@@ -615,7 +615,7 @@ additional tools. Below is an example Dockerfile, which installs providers for G
 ```dockerfile
 FROM apache/airflow:2.0.0
 
-RUN pip install --upgrade --user apache-airflow-providers-google==2.0.0.rc1
+RUN pip install  --user apache-airflow-providers-google==2.0.0.rc1
 
 USER ${AIRFLOW_UID}
 ```

docs/apache-airflow/installation/installing-from-pypi.rst

@@ -118,7 +118,7 @@ being installed.
     AIRFLOW_VERSION=|version|
     PYTHON_VERSION="$(python --version | cut -d " " -f 2 | cut -d "." -f 1-2)"
     CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt"
-    pip install --upgrade "apache-airflow[postgres,google]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"
+    pip install "apache-airflow[postgres,google]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"
 
 Installation and upgrading of Airflow providers separately
 ==========================================================

docs/docker-stack/build-arg-ref.rst

@@ -45,7 +45,7 @@ Those are the most common arguments that you use when you want to build a custom
 +------------------------------------------+------------------------------------------+---------------------------------------------+
 | ``AIRFLOW_USER_HOME_DIR``                | ``/home/airflow``                        | Home directory of the Airflow user.         |
 +------------------------------------------+------------------------------------------+---------------------------------------------+
-| ``AIRFLOW_PIP_VERSION``                  | ``21.2.4``                                | PIP version used.                          |
+| ``AIRFLOW_PIP_VERSION``                  | ``21.3.1``                               |  PIP version used.                          |
 +------------------------------------------+------------------------------------------+---------------------------------------------+
 | ``PIP_PROGRESS_BAR``                     | ``on``                                   | Progress bar for PIP installation           |
 +------------------------------------------+------------------------------------------+---------------------------------------------+
@@ -198,6 +198,16 @@ You can see some examples of those in:
 |                                          |                                          | "/opt/airflow" when you install Airflow  |
 |                                          |                                          | from local sources.                      |
 +------------------------------------------+------------------------------------------+------------------------------------------+
+| ``AIRFLOW_SOURCES_WWW_FROM``             | ``empty``                                | Sources of Airflow WWW files used for    |
+|                                          |                                          | asset compilation. Set it to             |
+|                                          |                                          | "./airflow/www" when                     |
+|                                          |                                          | you install Airflow from local sources   |
++------------------------------------------+------------------------------------------+------------------------------------------+
+| ``AIRFLOW_SOURCES_WWW_TO``               | ``/empty``                               | Target for Airflow files used for        |
+|                                          |                                          | asset compilation. Set it to             |
+|                                          |                                          | "/opt/airflow/airflow/www" when          |
+|                                          |                                          | you install Airflow from local sources.  |
++------------------------------------------+------------------------------------------+------------------------------------------+
 | ``AIRFLOW_VERSION_SPECIFICATION``        |                                          | Optional - might be used for using limit |
 |                                          |                                          | for Airflow version installation - for   |
 |                                          |                                          | example ``<2.0.2`` for automated builds. |

docs/docker-stack/build.rst

@@ -203,14 +203,22 @@ You should be aware, about a few things:
   `best practices of Dockerfiles <https://docs.docker.com/develop/develop-images/dockerfile_best-practices/>`_
   to make sure your image is lean and small.
 
-* The PyPI dependencies in Apache Airflow are installed in the user library, of the "airflow" user, so
-  PIP packages are installed to ``~/.local`` folder as if the ``--user`` flag was specified when running PIP.
-  Note also that using ``--no-cache-dir`` is a good idea that can help to make your image smaller.
+* Using ``--no-cache-dir`` is a good idea that can help to make your image smaller.
+
+* The PyPI dependencies in Apache Airflow are installed in the ``/.venv`` folder. This is virtualenv where
+  airflow and all dependent packages are installed, following latest ``pip`` recommendations and upcoming
+  :pep:`668`. The ``PATH`` inside the image is set to point first
+  to the ``bin`` folder of the virtualenv, to make sure that this virtualenv is used, also safe PATH for sudo
+  is set to include that folder. The virtualenv is also activated at the entry of interactive session by
+  ``.bashrc`` files placed in ``HOME`` directory of both ``airflow`` user and ``root`` user. Note that all
+  the arbitrary users created dynamically to follow OpenShift rules are sharing the ``airflow`` user
+  ``HOME`` folder, so ``.bashrc`` file is the same for those users as well.
 
 .. note::
-  Only as of ``2.0.1`` image the ``--user`` flag is turned on by default by setting ``PIP_USER`` environment
+  As of the ``2.0.1`` image the ``--user`` flag was turned on by default by setting ``PIP_USER`` environment
   variable to ``true``. This can be disabled by un-setting the variable or by setting it to ``false``. In the
-  2.0.0 image you had to add the ``--user`` flag as ``pip install --user`` command.
+  2.0.0 image you had to add the ``--user`` flag as ``pip install --user`` command. This flag and environment
+  variables however were removed in ``2.2`` in favor of virtualenv in ``/.venv`` folder.
 
 * If your apt, or PyPI dependencies require some of the ``build-essential`` or other packages that need
   to compile your python dependencies, then your best choice is to follow the "Customize the image" route,

scripts/ci/kubernetes/ci_run_kubernetes_tests.sh

@@ -87,7 +87,7 @@ function create_virtualenv() {
 
     . "${virtualenv_path}/bin/activate"
 
-    pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}"
+    pip install "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}"
 
     local constraints=(
         --constraint