From f3fa3fcc3e5fca0d7d7e9bda8646bd2a6916d09f Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 20 Oct 2023 14:24:21 +0200 Subject: [PATCH] Move installed_providers.txt to airflow/providers. The CI scripts in `build-image` workflow are taken from main so when you add new (preinstalled) provider to be installed for PROD image, the provider added in the PR in the file is not present in the build-image workflow. Moving the file to airflow/providers is safe because it does not contain any code - just list of packages that breeze command will use when building the PROD image. Related to #34729 --- .github/actions/build-prod-images/action.yml | 2 +- .github/workflows/ci.yml | 2 +- .pre-commit-config.yaml | 2 +- .rat-excludes | 2 +- {scripts/ci => airflow/providers}/installed_providers.txt | 0 dev/refresh_images.sh | 2 +- docker_tests/test_prod_image.py | 2 +- scripts/ci/pre_commit/pre_commit_sort_installed_providers.py | 2 +- 8 files changed, 7 insertions(+), 7 deletions(-) rename {scripts/ci => airflow/providers}/installed_providers.txt (100%) diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml index 0da3568680e70..c42c170f6c1e5 100644 --- a/.github/actions/build-prod-images/action.yml +++ b/.github/actions/build-prod-images/action.yml @@ -45,7 +45,7 @@ runs: shell: bash run: > breeze release-management prepare-provider-packages - --package-list-file ./scripts/ci/installed_providers.txt + --package-list-file ./airflow/providers/installed_providers.txt --package-format wheel --version-suffix-for-pypi dev0 if: ${{ inputs.build-provider-packages == 'true' }} - name: "Prepare airflow package" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b56b096249533..d9c136ba2444c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1805,7 +1805,7 @@ jobs: - name: "Prepare providers packages for PROD build" run: > breeze release-management prepare-provider-packages - --package-list-file ./scripts/ci/installed_providers.txt + --package-list-file ./airflow/providers/installed_providers.txt --package-format wheel env: VERSION_SUFFIX_FOR_PYPI: "dev0" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 42b67dbd662bf..85cfc050c140b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -669,7 +669,7 @@ repos: name: Sort alphabetically and uniquify installed_providers.txt entry: ./scripts/ci/pre_commit/pre_commit_sort_installed_providers.py language: python - files: ^\.pre-commit-config\.yaml$|^scripts/ci/installed_providers\.txt$ + files: ^\.pre-commit-config\.yaml$|^airflow/providers/installed_providers\.txt$ pass_filenames: false require_serial: true - id: update-spelling-wordlist-to-be-sorted diff --git a/.rat-excludes b/.rat-excludes index 8d25588e9ffcf..ce2cae7dd92ac 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -43,7 +43,7 @@ venv files airflow.iml .gitmodules -scripts/ci/installed_providers.txt +installed_providers.txt # Generated doc files .*html diff --git a/scripts/ci/installed_providers.txt b/airflow/providers/installed_providers.txt similarity index 100% rename from scripts/ci/installed_providers.txt rename to airflow/providers/installed_providers.txt diff --git a/dev/refresh_images.sh b/dev/refresh_images.sh index 594db2cc36266..638e86930cc1f 100755 --- a/dev/refresh_images.sh +++ b/dev/refresh_images.sh @@ -36,7 +36,7 @@ breeze ci-image build \ rm -fv ./dist/* ./docker-context-files/* breeze release-management prepare-provider-packages \ - --package-list-file ./scripts/ci/installed_providers.txt \ + --package-list-file ./airflow/providers/installed_providers.txt \ --package-format wheel \ --version-suffix-for-pypi dev0 diff --git a/docker_tests/test_prod_image.py b/docker_tests/test_prod_image.py index 8fd467be5812a..2b83bd91799cd 100644 --- a/docker_tests/test_prod_image.py +++ b/docker_tests/test_prod_image.py @@ -36,7 +36,7 @@ # isort:on (needed to workaround isort bug) from setup import PREINSTALLED_PROVIDERS -INSTALLED_PROVIDER_PATH = SOURCE_ROOT / "scripts" / "ci" / "installed_providers.txt" +INSTALLED_PROVIDER_PATH = SOURCE_ROOT / "airflow" / "providers" / "installed_providers.txt" class TestCommands: diff --git a/scripts/ci/pre_commit/pre_commit_sort_installed_providers.py b/scripts/ci/pre_commit/pre_commit_sort_installed_providers.py index 7ab17c5dd80e5..df53dea94dc3e 100755 --- a/scripts/ci/pre_commit/pre_commit_sort_installed_providers.py +++ b/scripts/ci/pre_commit/pre_commit_sort_installed_providers.py @@ -38,7 +38,7 @@ def sort_uniq(sequence): if __name__ == "__main__": - installed_providers_path = Path(AIRFLOW_SOURCES) / "scripts" / "ci" / "installed_providers.txt" + installed_providers_path = Path(AIRFLOW_SOURCES) / "airflow" / "providers" / "installed_providers.txt" content = installed_providers_path.read_text().splitlines(keepends=True) sorted_content = sort_uniq(content) installed_providers_path.write_text("".join(sorted_content))