From eedec678023ffae1ca3aac1cf4e0f7c8b0eed79e Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 4 Aug 2021 12:50:10 +0200 Subject: [PATCH 1/7] Improve image building documentation for new users This PR improves documentation for building images of airflow, specifically targetting users who do not have big experience with building the images. It shows examples on how custom image building can be easily used to upgrade provider packages as well as how image building can be easily integrated in quick-start using docker-compose. --- docs/apache-airflow-providers/index.rst | 5 +++++ docs/apache-airflow/start/docker-compose.yaml | 6 +++++- docs/apache-airflow/start/docker.rst | 12 +++++++++++ docs/docker-stack/build.rst | 13 ++++++++++++ .../extending/add-apt-packages/Dockerfile | 2 +- .../add-build-essential-extend/Dockerfile | 2 +- .../extending/add-providers/Dockerfile | 21 +++++++++++++++++++ .../extending/add-pypi-packages/Dockerfile | 2 +- .../extending/embedding-dags/Dockerfile | 2 +- .../extending/writable-directory/Dockerfile | 2 +- 10 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 docs/docker-stack/docker-examples/extending/add-providers/Dockerfile diff --git a/docs/apache-airflow-providers/index.rst b/docs/apache-airflow-providers/index.rst index 7329b7fd780a4..71c5132acdec8 100644 --- a/docs/apache-airflow-providers/index.rst +++ b/docs/apache-airflow-providers/index.rst @@ -21,6 +21,8 @@ Provider packages .. contents:: :local: +.. _providers:community-maintained-providers: + Community maintained providers '''''''''''''''''''''''''''''' @@ -31,6 +33,9 @@ Those provider packages are separated per-provider (for example ``amazon``, ``go etc.). Those packages are available as ``apache-airflow-providers`` packages - separately per each provider (for example there is an ``apache-airflow-providers-amazon`` or ``apache-airflow-providers-google`` package). +The full list of community managed providers is available at +`Providers Index `_. + You can install those provider packages separately in order to interface with a given service. For those providers that have corresponding extras, the provider packages (latest version from PyPI) are installed automatically when Airflow is installed with the extra. diff --git a/docs/apache-airflow/start/docker-compose.yaml b/docs/apache-airflow/start/docker-compose.yaml index 95796715fcc85..832092eec9e95 100644 --- a/docs/apache-airflow/start/docker-compose.yaml +++ b/docs/apache-airflow/start/docker-compose.yaml @@ -44,7 +44,11 @@ version: '3' x-airflow-common: &airflow-common + # In order to add custom dependencies or upgrade provider packages you can use your extended image. + # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml + # and uncomment the "build" line below, Then run `docker-compose build` to build the images. image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:|version|} + # build: . environment: &airflow-common-env AIRFLOW__CORE__EXECUTOR: CeleryExecutor @@ -60,7 +64,7 @@ x-airflow-common: - ./dags:/opt/airflow/dags - ./logs:/opt/airflow/logs - ./plugins:/opt/airflow/plugins - user: "${AIRFLOW_UID:-50000}:${AIRFLOW_GID:-50000}" + user: "${AIRFLOW_UID:-50000}:${AIRFLOW_GID:-0}" depends_on: &airflow-common-depends-on redis: diff --git a/docs/apache-airflow/start/docker.rst b/docs/apache-airflow/start/docker.rst index baea8ab2763ee..88c004e273592 100644 --- a/docs/apache-airflow/start/docker.rst +++ b/docs/apache-airflow/start/docker.rst @@ -81,6 +81,18 @@ If you need install a new Python library or system library, you can :doc:`build .. _initializing_docker_compose_environment: +Using custom images +=================== + +When you want to run Airflow locally, you might want to use extended image, containing some additional dependencies - for +example you might add new python packages, or upgrade airflow providers to a later version. This can be done very easily +by placing your custom Dockerfile. Then you can use `docker-compose build` command to build your image (you need to +do it only once) and you can also add `--build` to your `docker-compose` commands with flag to rebuild the images +on-the-fly when you run other `docker-compose` commands. + +The examples of how you can extend the image with custom providers, python packages, +apt packages and more can be found in :doc:`Building the image `. + Initializing Environment ======================== diff --git a/docs/docker-stack/build.rst b/docs/docker-stack/build.rst index 7d89f2fa23adf..b4a3745adb652 100644 --- a/docs/docker-stack/build.rst +++ b/docs/docker-stack/build.rst @@ -250,6 +250,19 @@ You should be aware, about a few things: Examples of image extending --------------------------- +Example of upgrading Airflow Provider packages +.............................................. + +The :ref:`Airflow Providers ` are released independently of the main +Airflow and sometimes you might want to upgrade specific providers only to fix some problems or +use features available in that provider. Here is an example of how you can do it + +.. exampleinclude:: docker-examples/extending/add-providers/Dockerfile + :language: Dockerfile + :start-after: [START Dockerfile] + :end-before: [END Dockerfile] + + Example of adding ``apt`` package ................................. diff --git a/docs/docker-stack/docker-examples/extending/add-apt-packages/Dockerfile b/docs/docker-stack/docker-examples/extending/add-apt-packages/Dockerfile index 62de197973833..f11e87adc666d 100644 --- a/docs/docker-stack/docker-examples/extending/add-apt-packages/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/add-apt-packages/Dockerfile @@ -15,7 +15,7 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow +FROM apache/airflow:2.1.2 USER root RUN apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/docs/docker-stack/docker-examples/extending/add-build-essential-extend/Dockerfile b/docs/docker-stack/docker-examples/extending/add-build-essential-extend/Dockerfile index b34fdc9ab3cf8..47ac51ffbe07c 100644 --- a/docs/docker-stack/docker-examples/extending/add-build-essential-extend/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/add-build-essential-extend/Dockerfile @@ -15,7 +15,7 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow +FROM apache/airflow:2.1.2 USER root RUN apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/docs/docker-stack/docker-examples/extending/add-providers/Dockerfile b/docs/docker-stack/docker-examples/extending/add-providers/Dockerfile new file mode 100644 index 0000000000000..cdf7a42b0e41b --- /dev/null +++ b/docs/docker-stack/docker-examples/extending/add-providers/Dockerfile @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example Dockerfile. It is not intended for PRODUCTION use +# [START Dockerfile] +FROM apache/airflow:2.1.2 +RUN pip install --no-cache-dir apache-airflow-providers-docker==2.1.0 +# [END Dockerfile] + diff --git a/docs/docker-stack/docker-examples/extending/add-pypi-packages/Dockerfile b/docs/docker-stack/docker-examples/extending/add-pypi-packages/Dockerfile index cc2559f79d062..310f84cf68dc8 100644 --- a/docs/docker-stack/docker-examples/extending/add-pypi-packages/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/add-pypi-packages/Dockerfile @@ -15,6 +15,6 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow +FROM apache/airflow:2.1.2 RUN pip install --no-cache-dir lxml # [END Dockerfile] diff --git a/docs/docker-stack/docker-examples/extending/embedding-dags/Dockerfile b/docs/docker-stack/docker-examples/extending/embedding-dags/Dockerfile index c849697859fb2..48701aae23d87 100644 --- a/docs/docker-stack/docker-examples/extending/embedding-dags/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/embedding-dags/Dockerfile @@ -15,7 +15,7 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow +FROM apache/airflow:2.1.2 COPY --chown=airflow:root test_dag.py /opt/airflow/dags diff --git a/docs/docker-stack/docker-examples/extending/writable-directory/Dockerfile b/docs/docker-stack/docker-examples/extending/writable-directory/Dockerfile index ba07f6816888f..8fbb98dfef00c 100644 --- a/docs/docker-stack/docker-examples/extending/writable-directory/Dockerfile +++ b/docs/docker-stack/docker-examples/extending/writable-directory/Dockerfile @@ -15,7 +15,7 @@ # This is an example Dockerfile. It is not intended for PRODUCTION use # [START Dockerfile] -FROM apache/airflow +FROM apache/airflow:2.1.2 RUN umask 0002; \ mkdir -p ~/writeable-directory # [END Dockerfile] From ef68c19921454890343f54ef424b38115608be3a Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 4 Aug 2021 22:03:28 +0200 Subject: [PATCH 2/7] Update docs/docker-stack/build.rst Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- docs/docker-stack/build.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docker-stack/build.rst b/docs/docker-stack/build.rst index b4a3745adb652..0b7441d4aae5f 100644 --- a/docs/docker-stack/build.rst +++ b/docs/docker-stack/build.rst @@ -253,7 +253,7 @@ Examples of image extending Example of upgrading Airflow Provider packages .............................................. -The :ref:`Airflow Providers ` are released independently of the main +The :ref:`Airflow Providers ` are released independently of core Airflow and sometimes you might want to upgrade specific providers only to fix some problems or use features available in that provider. Here is an example of how you can do it From 32689ea934ead7d51be1af39941688139b07af3f Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 4 Aug 2021 22:03:41 +0200 Subject: [PATCH 3/7] Update docs/apache-airflow/start/docker.rst Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- docs/apache-airflow/start/docker.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/start/docker.rst b/docs/apache-airflow/start/docker.rst index 88c004e273592..3f471900f2728 100644 --- a/docs/apache-airflow/start/docker.rst +++ b/docs/apache-airflow/start/docker.rst @@ -84,7 +84,7 @@ If you need install a new Python library or system library, you can :doc:`build Using custom images =================== -When you want to run Airflow locally, you might want to use extended image, containing some additional dependencies - for +When you want to run Airflow locally, you might want to use an extended image, containing some additional dependencies - for example you might add new python packages, or upgrade airflow providers to a later version. This can be done very easily by placing your custom Dockerfile. Then you can use `docker-compose build` command to build your image (you need to do it only once) and you can also add `--build` to your `docker-compose` commands with flag to rebuild the images From f311f2d227a0c21964b7529fefa86eb1c2d83290 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 4 Aug 2021 22:03:54 +0200 Subject: [PATCH 4/7] Update docs/apache-airflow/start/docker.rst Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- docs/apache-airflow/start/docker.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/start/docker.rst b/docs/apache-airflow/start/docker.rst index 3f471900f2728..323763352824a 100644 --- a/docs/apache-airflow/start/docker.rst +++ b/docs/apache-airflow/start/docker.rst @@ -86,7 +86,7 @@ Using custom images When you want to run Airflow locally, you might want to use an extended image, containing some additional dependencies - for example you might add new python packages, or upgrade airflow providers to a later version. This can be done very easily -by placing your custom Dockerfile. Then you can use `docker-compose build` command to build your image (you need to +by placing a custom Dockerfile alongside your `docker-compose.yaml`. Then you can use `docker-compose build` command to build your image (you need to do it only once) and you can also add `--build` to your `docker-compose` commands with flag to rebuild the images on-the-fly when you run other `docker-compose` commands. From bc18e918cc221c52afb8a94ebcec0cb48ca52dc5 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 4 Aug 2021 22:04:04 +0200 Subject: [PATCH 5/7] Update docs/apache-airflow/start/docker.rst Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- docs/apache-airflow/start/docker.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/start/docker.rst b/docs/apache-airflow/start/docker.rst index 323763352824a..180ccb879ea69 100644 --- a/docs/apache-airflow/start/docker.rst +++ b/docs/apache-airflow/start/docker.rst @@ -87,7 +87,7 @@ Using custom images When you want to run Airflow locally, you might want to use an extended image, containing some additional dependencies - for example you might add new python packages, or upgrade airflow providers to a later version. This can be done very easily by placing a custom Dockerfile alongside your `docker-compose.yaml`. Then you can use `docker-compose build` command to build your image (you need to -do it only once) and you can also add `--build` to your `docker-compose` commands with flag to rebuild the images +do it only once). You can also add the `--build` flag to your `docker-compose` commands to rebuild the images on-the-fly when you run other `docker-compose` commands. The examples of how you can extend the image with custom providers, python packages, From 0e63b34d49195d75efa6588f8530c14d85ef6a95 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 4 Aug 2021 22:04:16 +0200 Subject: [PATCH 6/7] Update docs/docker-stack/build.rst Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- docs/docker-stack/build.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docker-stack/build.rst b/docs/docker-stack/build.rst index 0b7441d4aae5f..c469a35dfc653 100644 --- a/docs/docker-stack/build.rst +++ b/docs/docker-stack/build.rst @@ -255,7 +255,7 @@ Example of upgrading Airflow Provider packages The :ref:`Airflow Providers ` are released independently of core Airflow and sometimes you might want to upgrade specific providers only to fix some problems or -use features available in that provider. Here is an example of how you can do it +use features available in that provider version. Here is an example of how you can do it .. exampleinclude:: docker-examples/extending/add-providers/Dockerfile :language: Dockerfile From 7a6b09a2eede0255d5087a0712462454ff564273 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 4 Aug 2021 22:04:28 +0200 Subject: [PATCH 7/7] Update docs/apache-airflow/start/docker.rst Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- docs/apache-airflow/start/docker.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow/start/docker.rst b/docs/apache-airflow/start/docker.rst index 180ccb879ea69..18fba4314bcef 100644 --- a/docs/apache-airflow/start/docker.rst +++ b/docs/apache-airflow/start/docker.rst @@ -90,7 +90,7 @@ by placing a custom Dockerfile alongside your `docker-compose.yaml`. Then you ca do it only once). You can also add the `--build` flag to your `docker-compose` commands to rebuild the images on-the-fly when you run other `docker-compose` commands. -The examples of how you can extend the image with custom providers, python packages, +Examples of how you can extend the image with custom providers, python packages, apt packages and more can be found in :doc:`Building the image `. Initializing Environment