diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 6ef786e8a8628..296e58fa23ad2 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -20,12 +20,12 @@ name: Additional CI image checks on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: - description: "The array of labels (in json form) determining AMD public runners." + runners: + description: "The array of labels (in json form) determining runners." required: true type: string - arm-runners: - description: "The array of labels (in json form) determining ARM public runners." + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string python-versions: @@ -99,12 +99,11 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - amd-runners: ${{ inputs.amd-runners }} - arm-runners: ${{ inputs.arm-runners }} + runners: ${{ inputs.runners }} cache-type: "Early" include-prod-images: "false" push-latest-images: "false" - platform: "linux/amd64" + platform: ${{ inputs.platform }} python-versions: ${{ inputs.python-versions }} branch: ${{ inputs.branch }} constraints-branch: ${{ inputs.constraints-branch }} @@ -120,7 +119,7 @@ jobs: check-that-image-builds-quickly: timeout-minutes: 11 name: Check that image builds quickly - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: UPGRADE_TO_NEWER_DEPENDENCIES: false PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }} @@ -129,6 +128,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} VERBOSE: "true" + PLATFORM: ${{ inputs.platform }} if: inputs.branch == 'main' steps: - name: "Cleanup repo" @@ -150,29 +150,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$actor" --password-stdin - name: "Check that image builds quickly" - run: breeze shell --max-time 600 --platform "linux/amd64" - -# This is only a check if ARM images are successfully building when committer runs PR from -# Apache repository. This is needed in case you want to fix failing cache job in "canary" run -# There is no point in running this one in "canary" run, because the above step is doing the -# same build anyway. -# build-ci-arm-images: -# name: Build CI ARM images -# uses: ./.github/workflows/ci-image-build.yml -# permissions: -# contents: read -# packages: write -# with: -# amd-runners: ${{ inputs.amd-runners }} -# arm-runners: ${{ inputs.arm-runners }} -# platform: "linux/arm64" -# push-image: "false" -# upload-image-artifact: "true" -# upload-mount-cache-artifact: ${{ inputs.canary-run }} -# python-versions: ${{ inputs.python-versions }} -# branch: ${{ inputs.branch }} -# constraints-branch: ${{ inputs.constraints-branch }} -# use-uv: ${{ inputs.use-uv }} -# upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} -# docker-cache: ${{ inputs.docker-cache }} -# disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }} + run: breeze shell --max-time 600 --platform "${PLATFORM}" diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index 3834efad76e3e..489b2967af3d1 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -20,8 +20,12 @@ name: Additional PROD image tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: - description: "The array of labels (in json form) determining AMD public runners." + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string default-branch: @@ -63,7 +67,8 @@ jobs: name: PROD image extra checks (main) uses: ./.github/workflows/prod-image-extra-checks.yml with: - amd-runners: ${{ inputs.amd-runners }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} python-versions: "[ '${{ inputs.default-python-version }}' ]" default-python-version: ${{ inputs.default-python-version }} branch: ${{ inputs.default-branch }} @@ -78,7 +83,8 @@ jobs: name: PROD image extra checks (release) uses: ./.github/workflows/prod-image-extra-checks.yml with: - amd-runners: ${{ inputs.amd-runners }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} python-versions: "[ '${{ inputs.default-python-version }}' ]" default-python-version: ${{ inputs.default-python-version }} branch: ${{ inputs.default-branch }} @@ -92,7 +98,7 @@ jobs: test-examples-of-prod-image-building: timeout-minutes: 60 name: "Test examples of PROD image building" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -112,7 +118,7 @@ jobs: - name: "Prepare breeze & PROD image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} image-type: "prod" python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} @@ -130,7 +136,7 @@ jobs: test-docker-compose-quick-start: timeout-minutes: 60 name: "Docker Compose quick start with PROD image verifying" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} @@ -149,7 +155,7 @@ jobs: - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} image-type: "prod" python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} use-uv: ${{ inputs.use-uv }} diff --git a/.github/workflows/airflow-distributions-tests.yml b/.github/workflows/airflow-distributions-tests.yml index 77b0e76462a0b..a7156ef98dfb7 100644 --- a/.github/workflows/airflow-distributions-tests.yml +++ b/.github/workflows/airflow-distributions-tests.yml @@ -21,8 +21,12 @@ on: # yamllint disable-line rule:truthy workflow_call: inputs: # Static inputs defined to choose which distribution to test to run - amd-runners: - description: "The array of labels (in json form) determining public AMD runners." + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string distribution-name: @@ -59,7 +63,7 @@ jobs: distributions-tests: timeout-minutes: 80 name: ${{ inputs.distribution-name }}:P${{ matrix.python-version }} tests - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: fail-fast: false matrix: @@ -82,7 +86,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ matrix.python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ matrix.python-version }} use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist files" diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index cdd619c9f4c47..5d931b001865a 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -20,8 +20,8 @@ name: Basic tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: - description: "The array of labels (in json form) determining public runners." + runners: + description: "The array of labels (in json form) determining runners." required: true type: string run-ui-tests: @@ -66,7 +66,7 @@ jobs: run-breeze-tests: timeout-minutes: 10 name: Breeze unit tests - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} steps: - name: "Cleanup repo" shell: bash @@ -87,7 +87,7 @@ jobs: tests-ui: timeout-minutes: 15 name: React UI tests - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} if: inputs.run-ui-tests == 'true' steps: - name: "Cleanup repo" @@ -155,7 +155,7 @@ jobs: install-pre-commit: timeout-minutes: 5 name: "Install pre-commit for cache" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" steps: @@ -183,7 +183,7 @@ jobs: static-checks-basic-checks-only: timeout-minutes: 30 name: "Static checks: basic checks only" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} needs: install-pre-commit if: inputs.basic-checks-only == 'true' steps: @@ -236,7 +236,7 @@ jobs: upgrade-check: timeout-minutes: 45 name: "Upgrade checks" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} needs: install-pre-commit env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" @@ -306,7 +306,7 @@ jobs: test-airflow-release-commands: timeout-minutes: 80 name: "Test Airflow release commands" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci-amd.yml similarity index 90% rename from .github/workflows/ci.yml rename to .github/workflows/ci-amd.yml index ab0e577e94416..a788319e496b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci-amd.yml @@ -16,7 +16,7 @@ # under the License. # --- -name: Tests +name: Tests AMD on: # yamllint disable-line rule:truthy schedule: - cron: '28 1,7,13,19 * * *' @@ -43,7 +43,7 @@ env: VERBOSE: "true" concurrency: - group: ci-${{ github.event.pull_request.number || github.ref }} + group: ci-amd-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: @@ -173,7 +173,7 @@ jobs: needs: [build-info] uses: ./.github/workflows/basic-tests.yml with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}} run-www-tests: ${{needs.build-info.outputs.run-www-tests}} needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} @@ -194,8 +194,7 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} - arm-runners: ${{ needs.build-info.outputs.arm-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} platform: "linux/amd64" push-image: "false" upload-image-artifact: "true" @@ -219,8 +218,8 @@ jobs: id-token: write if: needs.build-info.outputs.canary-run == 'true' with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} - arm-runners: ${{ needs.build-info.outputs.arm-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} @@ -241,7 +240,8 @@ jobs: uses: ./.github/workflows/generate-constraints.yml if: needs.build-info.outputs.ci-image-build == 'true' with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} # generate no providers constraints only in canary builds - they take quite some time to generate # they are not needed for regular builds, they are only needed to update constraints in canaries @@ -257,7 +257,8 @@ jobs: id-token: write contents: read with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" needs-mypy: ${{ needs.build-info.outputs.needs-mypy }} mypy-checks: ${{ needs.build-info.outputs.mypy-checks }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} @@ -293,7 +294,8 @@ jobs: needs.build-info.outputs.latest-versions-only != 'true' && needs.build-info.outputs.run-tests == 'true' with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" canary-run: ${{ needs.build-info.outputs.canary-run }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} @@ -315,7 +317,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -332,7 +335,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "postgres" test-name: "Postgres" test-scope: "DB" @@ -359,7 +363,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "postgres" test-name: "Postgres" test-scope: "DB" @@ -386,7 +391,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "mysql" test-name: "MySQL" test-scope: "DB" @@ -413,7 +419,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "mysql" test-name: "MySQL" test-scope: "DB" @@ -441,7 +448,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "sqlite" test-name: "Sqlite" test-name-separator: "" @@ -470,7 +478,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "sqlite" test-name: "Sqlite" test-name-separator: "" @@ -500,7 +509,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "sqlite" test-name: "" test-name-separator: "" @@ -528,7 +538,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" backend: "sqlite" test-name: "" test-name-separator: "" @@ -562,7 +573,8 @@ jobs: needs.build-info.outputs.full-tests-needed == 'true') with: default-branch: ${{ needs.build-info.outputs.default-branch }} - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" core-test-types-list-as-strings-in-json: > ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} providers-test-types-list-as-strings-in-json: > @@ -587,7 +599,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }} testable-providers-integrations: ${{ needs.build-info.outputs.testable-providers-integrations }} run-system-tests: ${{ needs.build-info.outputs.run-tests }} @@ -610,7 +623,8 @@ jobs: if: > needs.build-info.outputs.run-tests == 'true' with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" test-name: "LowestDeps" force-lowest-dependencies: "true" test-scope: "All" @@ -638,7 +652,8 @@ jobs: packages: read if: needs.build-info.outputs.run-tests == 'true' with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" test-name: "LowestDeps" force-lowest-dependencies: "true" test-scope: "All" @@ -667,9 +682,9 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} - build-type: "Regular" + runners: ${{ needs.build-info.outputs.amd-runners }} platform: "linux/amd64" + build-type: "Regular" push-image: "false" upload-image-artifact: "true" upload-package-artifact: "true" @@ -688,7 +703,8 @@ jobs: needs: [build-info, build-prod-images, generate-constraints] uses: ./.github/workflows/additional-prod-image-tests.yml with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" default-branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} @@ -707,8 +723,8 @@ jobs: contents: read packages: read with: + runners: ${{ needs.build-info.outputs.amd-runners }} platform: "linux/amd64" - amd-runners: ${{ needs.build-info.outputs.amd-runners }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -726,7 +742,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" default-python-version: ${{ needs.build-info.outputs.default-python-version }} python-versions: ${{ needs.build-info.outputs.python-versions }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -747,7 +764,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" default-python-version: ${{ needs.build-info.outputs.default-python-version }} python-versions: ${{ needs.build-info.outputs.python-versions }} use-uv: ${{ needs.build-info.outputs.use-uv }} @@ -794,8 +812,8 @@ jobs: - tests-with-lowest-direct-resolution-providers uses: ./.github/workflows/finalize-tests.yml with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} - arm-runners: ${{ needs.build-info.outputs.arm-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} + platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} branch: ${{ needs.build-info.outputs.default-branch }} @@ -825,10 +843,51 @@ jobs: # yamllint disable rule:line-length payload: | channel: "internal-airflow-ci-cd" - text: "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* " + text: "🚨🕒 Scheduled CI Failure Alert (AMD) 🕒🚨\n\n*Details:* " blocks: - type: "section" text: type: "mrkdwn" - text: "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* " + text: "🚨🕒 Scheduled CI Failure Alert (AMD) 🕒🚨\n\n*Details:* " # yamllint enable rule:line-length + + summarize-warnings: + timeout-minutes: 15 + name: "Summarize warnings" + runs-on: ${{ fromJSON(needs.build-info.outputs.amd-runners) }} + if: needs.build-info.outputs.run-tests == 'true' + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Cleanup docker" + run: ./scripts/ci/cleanup_docker.sh + - name: "Free up disk space" + shell: bash + run: ./scripts/tools/free_up_disk_space.sh + - name: "Download all test warning artifacts from the current build" + uses: actions/download-artifact@v4 + with: + path: ./artifacts + pattern: test-warnings-* + - name: "Setup python" + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.default-python-version }} + - name: "Summarize all warnings" + run: | + ./scripts/ci/testing/summarize_captured_warnings.py ./artifacts \ + --pattern "**/warnings-*.txt" \ + --output ./files + - name: "Upload artifact for summarized warnings" + uses: actions/upload-artifact@v4 + with: + name: test-summarized-amd-runner-warnings + path: ./files/warn-summary-*.txt + retention-days: 7 + if-no-files-found: ignore + overwrite: true diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml new file mode 100644 index 0000000000000..5b8cee0a432ca --- /dev/null +++ b/.github/workflows/ci-arm.yml @@ -0,0 +1,572 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Tests ARM +on: # yamllint disable-line rule:truthy + schedule: + - cron: '28 3,9,15,21 * * *' + push: + branches: + - v[0-9]+-[0-9]+-test + - providers-[a-z]+-?[a-z]*/v[0-9]+-[0-9]+ + workflow_dispatch: +permissions: + # All other permissions are set to none by default + contents: read +env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + VERBOSE: "true" + +concurrency: + group: ci-arm-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + + build-info: + name: "Build info" + # At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners + runs-on: ["ubuntu-22.04"] + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + outputs: + all-python-versions-list-as-string: >- + ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} + basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} + canary-run: ${{ steps.source-run-info.outputs.canary-run }} + ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} + core-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.core-test-types-list-as-strings-in-json }} + debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} + default-branch: ${{ steps.selective-checks.outputs.default-branch }} + default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} + default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} + default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} + default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} + default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }} + default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} + default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} + disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }} + docker-cache: ${{ steps.selective-checks.outputs.docker-cache }} + docs-build: ${{ steps.selective-checks.outputs.docs-build }} + docs-list-as-string: ${{ steps.selective-checks.outputs.docs-list-as-string }} + excluded-providers-as-string: ${{ steps.selective-checks.outputs.excluded-providers-as-string }} + force-pip: ${{ steps.selective-checks.outputs.force-pip }} + full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} + has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} + helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} + include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} + individual-providers-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-strings-in-json }} + kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }} + kubernetes-combos-list-as-string: >- + ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} + kubernetes-versions-list-as-string: >- + ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} + latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} + mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }} + mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} + mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} + needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} + needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} + needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} + needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }} + only-new-ui-files: ${{ steps.selective-checks.outputs.only-new-ui-files }} + postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} + postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} + prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} + # yamllint disable rule:line-length + providers-compatibility-tests-matrix: > + ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }} + providers-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.providers-test-types-list-as-strings-in-json }} + pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} + python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} + python-versions: ${{ steps.selective-checks.outputs.python-versions }} + run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }} + run-airflow-ctl-tests: ${{ steps.selective-checks.outputs.run-airflow-ctl-tests }} + run-coverage: ${{ steps.source-run-info.outputs.run-coverage }} + run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} + run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }} + run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }} + run-tests: ${{ steps.selective-checks.outputs.run-tests }} + run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} + run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} + amd-runners: ${{ steps.selective-checks.outputs.amd-runners }} + arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} + selected-providers-list-as-string: >- + ${{ steps.selective-checks.outputs.selected-providers-list-as-string }} + skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} + skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }} + source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} + sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} + testable-core-integrations: ${{ steps.selective-checks.outputs.testable-core-integrations }} + testable-providers-integrations: ${{ steps.selective-checks.outputs.testable-providers-integrations }} + use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} + upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Cleanup docker" + run: ./scripts/ci/cleanup_docker.sh + - name: Fetch incoming commit ${{ github.sha }} with its parent + uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + fetch-depth: 2 + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + use-uv: ${{ inputs.use-uv }} + id: breeze + - name: "Get information about the Workflow" + id: source-run-info + run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} + env: + SKIP_BREEZE_SELF_UPGRADE_CHECK: "true" + - name: Selective checks + id: selective-checks + env: + PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" + COMMIT_REF: "${{ github.sha }}" + VERBOSE: "false" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} + - name: env + run: printenv + env: + PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} + GITHUB_CONTEXT: ${{ toJson(github) }} + + basic-tests: + name: "Basic tests" + needs: [build-info] + uses: ./.github/workflows/basic-tests.yml + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}} + run-www-tests: ${{needs.build-info.outputs.run-www-tests}} + needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} + default-python-version: ${{needs.build-info.outputs.default-python-version}} + basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}} + skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}} + canary-run: ${{needs.build-info.outputs.canary-run}} + latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} + use-uv: ${{needs.build-info.outputs.use-uv}} + + build-ci-images: + name: Build CI images + needs: [build-info] + uses: ./.github/workflows/ci-image-build.yml + permissions: + contents: read + # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs + # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. + packages: write + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + push-image: "false" + upload-image-artifact: "true" + upload-mount-cache-artifact: ${{ needs.build-info.outputs.canary-run }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + if: needs.build-info.outputs.ci-image-build == 'true' + + additional-ci-image-checks: + name: "Additional CI image checks" + needs: [build-info, build-ci-images] + uses: ./.github/workflows/additional-ci-image-checks.yml + permissions: + contents: read + packages: write + id-token: write + if: needs.build-info.outputs.canary-run == 'true' + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + default-python-version: ${{ needs.build-info.outputs.default-python-version }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + + generate-constraints: + name: "Generate constraints" + needs: [build-info, build-ci-images] + uses: ./.github/workflows/generate-constraints.yml + if: needs.build-info.outputs.ci-image-build == 'true' + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + # generate no providers constraints only in canary builds - they take quite some time to generate + # they are not needed for regular builds, they are only needed to update constraints in canaries + generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + + + providers: + name: "provider distributions tests" + uses: ./.github/workflows/test-providers.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + if: > + needs.build-info.outputs.skip-providers-tests != 'true' && + needs.build-info.outputs.latest-versions-only != 'true' + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + canary-run: ${{ needs.build-info.outputs.canary-run }} + default-python-version: ${{ needs.build-info.outputs.default-python-version }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + selected-providers-list-as-string: ${{ needs.build-info.outputs.selected-providers-list-as-string }} + # yamllint disable rule:line-length + providers-compatibility-tests-matrix: > + ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + providers-test-types-list-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + + tests-helm: + name: "Helm tests" + uses: ./.github/workflows/helm-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} + default-python-version: ${{ needs.build-info.outputs.default-python-version }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: > + needs.build-info.outputs.needs-helm-tests == 'true' && + needs.build-info.outputs.default-branch == 'main' && + needs.build-info.outputs.latest-versions-only != 'true' + + tests-postgres-core: + name: "Postgres tests: core" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + backend: "postgres" + test-name: "Postgres" + test-scope: "DB" + test-group: "core" + python-versions: ${{ needs.build-info.outputs.python-versions }} + backend-versions: ${{ needs.build-info.outputs.postgres-versions }} + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.postgres-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-migration-tests: "true" + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: needs.build-info.outputs.run-tests == 'true' + + tests-postgres-providers: + name: "Postgres tests: providers" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + backend: "postgres" + test-name: "Postgres" + test-scope: "DB" + test-group: "providers" + python-versions: ${{ needs.build-info.outputs.python-versions }} + backend-versions: ${{ needs.build-info.outputs.postgres-versions }} + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.postgres-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-migration-tests: "true" + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: needs.build-info.outputs.run-tests == 'true' + + tests-sqlite-core: + name: "Sqlite tests: core" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + backend: "sqlite" + test-name: "Sqlite" + test-name-separator: "" + test-scope: "DB" + test-group: "core" + python-versions: ${{ needs.build-info.outputs.python-versions }} + # No versions for sqlite + backend-versions: "['']" + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.sqlite-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + run-migration-tests: "true" + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: needs.build-info.outputs.run-tests == 'true' + + tests-sqlite-providers: + name: "Sqlite tests: providers" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + backend: "sqlite" + test-name: "Sqlite" + test-name-separator: "" + test-scope: "DB" + test-group: "providers" + python-versions: ${{ needs.build-info.outputs.python-versions }} + # No versions for sqlite + backend-versions: "['']" + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.sqlite-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + run-migration-tests: "true" + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: needs.build-info.outputs.run-tests == 'true' + + + tests-non-db-core: + name: "Non-DB tests: core" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + backend: "sqlite" + test-name: "" + test-name-separator: "" + test-scope: "Non-DB" + test-group: "core" + python-versions: ${{ needs.build-info.outputs.python-versions }} + # No versions for non-db + backend-versions: "['']" + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.sqlite-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: needs.build-info.outputs.run-tests == 'true' + + tests-non-db-providers: + name: "Non-DB tests: providers" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + backend: "sqlite" + test-name: "" + test-name-separator: "" + test-scope: "Non-DB" + test-group: "providers" + python-versions: ${{ needs.build-info.outputs.python-versions }} + # No versions for non-db + backend-versions: "['']" + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.sqlite-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: needs.build-info.outputs.run-tests == 'true' + + build-prod-images: + name: Build PROD images + needs: [build-info, build-ci-images, generate-constraints] + uses: ./.github/workflows/prod-image-build.yml + permissions: + contents: read + # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs + # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. + packages: write + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + build-type: "Regular" + push-image: "false" + upload-image-artifact: "true" + upload-package-artifact: "true" + python-versions: ${{ needs.build-info.outputs.python-versions }} + default-python-version: ${{ needs.build-info.outputs.default-python-version }} + branch: ${{ needs.build-info.outputs.default-branch }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + prod-image-build: ${{ needs.build-info.outputs.prod-image-build }} + + tests-kubernetes: + name: "Kubernetes tests" + uses: ./.github/workflows/k8s-tests.yml + needs: [build-info, build-prod-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + kubernetes-combos: ${{ needs.build-info.outputs.kubernetes-combos }} + if: > + ( needs.build-info.outputs.run-kubernetes-tests == 'true' || + needs.build-info.outputs.needs-helm-tests == 'true') + + finalize-tests: + name: Finalize tests + permissions: + contents: write + packages: write + # This will fire when all the jobs from "needs" are either successful or skipped + if: always() && !failure() && !cancelled() + needs: + - additional-ci-image-checks + - basic-tests + - build-info + - basic-tests + - generate-constraints + - build-prod-images + - providers + - tests-helm + - tests-kubernetes + - tests-non-db-core + - tests-non-db-providers + - tests-postgres-core + - tests-postgres-providers + - tests-sqlite-core + - tests-sqlite-providers + uses: ./.github/workflows/finalize-tests.yml + with: + runners: ${{ needs.build-info.outputs.arm-runners }} + platform: "linux/arm64" + python-versions: ${{ needs.build-info.outputs.python-versions }} + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + default-python-version: ${{ needs.build-info.outputs.default-python-version }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + + notify-slack-failure: + name: "Notify Slack on Failure" + needs: + - finalize-tests + if: github.event_name == 'schedule' && failure() && github.run_attempt == 1 + runs-on: ["ubuntu-22.04"] + steps: + - name: Notify Slack + id: slack + uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 + with: + method: chat.postMessage + token: ${{ env.SLACK_BOT_TOKEN }} + # yamllint disable rule:line-length + payload: | + channel: "internal-airflow-ci-cd" + text: "🚨🕒 Scheduled CI Failure Alert (ARM) 🕒🚨\n\n*Details:* " + blocks: + - type: "section" + text: + type: "mrkdwn" + text: "🚨🕒 Scheduled CI Failure Alert (ARM) 🕒🚨\n\n*Details:* " + # yamllint enable rule:line-length diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 9c032fbff374f..d531ffebebf2d 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -20,12 +20,8 @@ name: Build CI images on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: - description: "The array of labels (in json form) determining public AMD runners." - required: true - type: string - arm-runners: - description: "The array of labels (in json form) determining public ARM runners." + runners: + description: "The array of labels (in json form) determining runners." required: true type: string target-commit-sha: @@ -106,11 +102,7 @@ jobs: python-version: ${{ fromJSON(inputs.python-versions) || fromJSON('[""]') }} timeout-minutes: 110 name: "Build CI ${{ inputs.platform }} image ${{ matrix.python-version }}" - # NOTE!!!!! This has to be put in one line for runs-on to recognize the "fromJSON" properly !!!! - # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array" - # instead of an array of strings. - # yamllint disable-line rule:line-length - runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.amd-runners) || fromJSON(inputs.arm-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: BACKEND: sqlite PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml index 70d1e2b847069..7e5a702911794 100644 --- a/.github/workflows/ci-image-checks.yml +++ b/.github/workflows/ci-image-checks.yml @@ -20,8 +20,12 @@ name: CI Image Checks on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: - description: "The array of labels (in json form) determining public AMD runners." + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string needs-mypy: @@ -113,7 +117,7 @@ jobs: install-pre-commit: timeout-minutes: 5 name: "Install pre-commit for cache (only canary runs)" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" if: inputs.basic-checks-only == 'false' @@ -157,7 +161,7 @@ jobs: static-checks: timeout-minutes: 45 name: "Static checks" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} needs: install-pre-commit env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" @@ -175,7 +179,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} id: breeze @@ -197,7 +201,7 @@ jobs: mypy: timeout-minutes: 45 name: "MyPy checks" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} needs: install-pre-commit if: inputs.needs-mypy == 'true' strategy: @@ -218,7 +222,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} id: breeze @@ -241,7 +245,7 @@ jobs: build-docs: timeout-minutes: 150 name: "Build documentation" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} if: inputs.docs-build == 'true' strategy: fail-fast: false @@ -266,7 +270,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "Restore docs inventory cache" @@ -344,7 +348,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "Publish docs" @@ -382,7 +386,7 @@ jobs: test-python-api-client: timeout-minutes: 60 name: "Test Python API client" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} if: inputs.needs-api-codegen == 'true' env: BACKEND: "postgres" @@ -416,7 +420,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "Generate airflow python client" diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index ae1d77e074164..082744fd6faea 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -20,12 +20,12 @@ name: Finalize tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: - description: "The array of labels (in json form) determining public AMD runners." + runners: + description: "The array of labels (in json form) determining runners." required: true type: string - arm-runners: - description: "The array of labels (in json form) determining public ARM runners." + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string python-versions: @@ -80,7 +80,7 @@ permissions: contents: read jobs: update-constraints: - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} timeout-minutes: 80 name: "Update constraints" permissions: @@ -93,7 +93,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} VERBOSE: "true" - if: inputs.upgrade-to-newer-dependencies != 'false' + if: inputs.upgrade-to-newer-dependencies != 'false' && inputs.platform == 'linux/amd64' steps: - name: "Cleanup repo" shell: bash @@ -132,8 +132,8 @@ jobs: run: git push - push-buildx-cache-to-github-registry-amd: - name: Push Regular AMD Image Cache + push-buildx-cache-to-github-registry: + name: Push Regular Image Cache ${{ inputs.platform }} needs: [update-constraints] uses: ./.github/workflows/push-image-cache.yml permissions: @@ -142,12 +142,11 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - amd-runners: ${{ inputs.amd-runners }} - arm-runners: ${{ inputs.arm-runners }} + runners: ${{ inputs.runners }} + platform: ${ { inputs.platform }} cache-type: "Regular AMD" include-prod-images: "true" push-latest-images: "true" - platform: "linux/amd64" python-versions: ${{ inputs.python-versions }} branch: ${{ inputs.branch }} constraints-branch: ${{ inputs.constraints-branch }} @@ -158,67 +157,3 @@ jobs: if: > inputs.canary-run == 'true' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') - - # push-buildx-cache-to-github-registry-arm: - # name: Push Regular ARM Image Cache - # needs: [update-constraints] - # uses: ./.github/workflows/push-image-cache.yml - # permissions: - # contents: read - # packages: write - # with: - # amd-runners: ${{ inputs.amd-runners }} - # arm-runners: ${{ inputs.arm-runners }} - # cache-type: "Regular ARM" - # include-prod-images: "true" - # push-latest-images: "true" - # platform: "linux/arm64" - # python-versions: ${{ inputs.python-versions }} - # branch: ${{ inputs.branch }} - # constraints-branch: ${{ inputs.constraints-branch }} - # use-uv: "true" - # include-success-outputs: ${{ inputs.include-success-outputs }} - # docker-cache: ${{ inputs.docker-cache }} - # if: > - # inputs.canary-run == 'true' && - # (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') - - summarize-warnings: - timeout-minutes: 15 - name: "Summarize warnings" - runs-on: ${{ fromJSON(inputs.amd-runners) }} - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh - - name: "Free up disk space" - shell: bash - run: ./scripts/tools/free_up_disk_space.sh - - name: "Download all test warning artifacts from the current build" - uses: actions/download-artifact@v4 - with: - path: ./artifacts - pattern: test-warnings-* - - name: "Setup python" - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.default-python-version }} - - name: "Summarize all warnings" - run: | - ./scripts/ci/testing/summarize_captured_warnings.py ./artifacts \ - --pattern "**/warnings-*.txt" \ - --output ./files - - name: "Upload artifact for summarized warnings" - uses: actions/upload-artifact@v4 - with: - name: test-summarized-warnings - path: ./files/warn-summary-*.txt - retention-days: 7 - if-no-files-found: ignore - overwrite: true diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 91b569424545b..363888ff29718 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -20,8 +20,12 @@ name: Generate constraints on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: - description: "The array of labels (in json form) determining public AMD runners." + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string python-versions-list-as-string: @@ -46,7 +50,7 @@ jobs: contents: read timeout-minutes: 70 name: Generate constraints ${{ inputs.python-versions-list-as-string }} - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: DEBUG_RESOURCES: ${{ inputs.debug-resources }} GITHUB_REPOSITORY: ${{ github.repository }} @@ -74,7 +78,7 @@ jobs: - name: "Prepare all CI images: ${{ inputs.python-versions-list-as-string}}" uses: ./.github/actions/prepare_all_ci_images with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }} docker-volume-location: "" # TODO(jscheffl): Understand why it fails here and fix it - name: "Verify all CI images ${{ inputs.python-versions-list-as-string }}" diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 21c7f01a528d4..e00ac14506ab4 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -20,8 +20,12 @@ name: Helm tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: - description: "The array of labels (in json form) determining public AMD runners." + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string helm-test-packages: @@ -42,7 +46,7 @@ jobs: tests-helm: timeout-minutes: 80 name: "Unit tests Helm: ${{ matrix.helm-test-package }}" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: fail-fast: false matrix: @@ -70,7 +74,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "Helm Unit Tests: ${{ matrix.helm-test-package }}" @@ -81,7 +85,7 @@ jobs: tests-helm-release: timeout-minutes: 80 name: "Release Helm" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}" steps: diff --git a/.github/workflows/integration-system-tests.yml b/.github/workflows/integration-system-tests.yml index 9c3480432b59c..bf75cc87f31a3 100644 --- a/.github/workflows/integration-system-tests.yml +++ b/.github/workflows/integration-system-tests.yml @@ -20,10 +20,14 @@ name: Integration and system tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: + runners: description: "The array of labels (in json form) determining public runners." required: true type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" + required: true + type: string testable-core-integrations: description: "The list of testable core integrations as JSON array." required: true @@ -71,7 +75,7 @@ jobs: timeout-minutes: 30 if: inputs.testable-core-integrations != '[]' name: "Integration core ${{ matrix.integration }}" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: fail-fast: false matrix: @@ -99,7 +103,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "Integration: core ${{ matrix.integration }}" @@ -120,7 +124,7 @@ jobs: timeout-minutes: 30 if: inputs.testable-providers-integrations != '[]' && inputs.skip-providers-tests != 'true' name: "Integration: providers ${{ matrix.integration }}" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: fail-fast: false matrix: @@ -148,7 +152,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "Integration: providers ${{ matrix.integration }}" @@ -168,7 +172,7 @@ jobs: timeout-minutes: 30 if: inputs.run-system-tests == 'true' name: "System Tests" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} env: BACKEND: "postgres" BACKEND_VERSION: ${{ inputs.default-postgres-version }}" @@ -192,7 +196,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "System Tests" diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index 43b30e40a9721..6ed0c79d187c5 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -20,12 +20,12 @@ name: K8s tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - platform: - description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" + runners: + description: "The array of labels (in json form) determining runners." required: true type: string - amd-runners: - description: "The array of labels (in json form) determining public AMD runners." + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string python-versions-list-as-string: @@ -54,7 +54,7 @@ jobs: tests-kubernetes: timeout-minutes: 60 name: "K8S System:${{ matrix.executor }}-${{ matrix.kubernetes-combo }}-${{ matrix.use-standard-naming }}" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: matrix: executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor] diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index bf4ead70addf9..14d836c094ffc 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -20,8 +20,8 @@ name: Build PROD images on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: - description: "The array of labels (in json form) determining default runner for AMD build." + runners: + description: "The array of labels (in json form) determining runners." required: true type: string build-type: @@ -114,7 +114,7 @@ jobs: build-prod-packages: name: "Build Airflow and provider distributions" timeout-minutes: 10 - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} if: inputs.prod-image-build == 'true' env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" @@ -186,7 +186,7 @@ jobs: python-version: ${{ fromJSON(inputs.python-versions) || fromJSON('[""]') }} timeout-minutes: 80 name: "Build PROD ${{ inputs.build-type }} image ${{ matrix.python-version }}" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} needs: - build-prod-packages env: diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml index 640d924d87339..4f6ce6c762964 100644 --- a/.github/workflows/prod-image-extra-checks.yml +++ b/.github/workflows/prod-image-extra-checks.yml @@ -20,8 +20,12 @@ name: PROD images extra checks on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: - description: "The array of labels (in json form) determining public AMD runners." + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string python-versions: @@ -62,14 +66,14 @@ jobs: myssql-client-image: uses: ./.github/workflows/prod-image-build.yml with: - amd-runners: ${{ inputs.amd-runners }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} build-type: "MySQL Client" upload-image-artifact: "false" upload-package-artifact: "false" install-mysql-client-type: "mysql" python-versions: ${{ inputs.python-versions }} default-python-version: ${{ inputs.default-python-version }} - platform: "linux/amd64" branch: ${{ inputs.branch }} # Always build images during the extra checks and never push them push-image: "false" @@ -83,14 +87,14 @@ jobs: pip-image: uses: ./.github/workflows/prod-image-build.yml with: - amd-runners: ${{ inputs.amd-runners }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} build-type: "pip" upload-image-artifact: "false" upload-package-artifact: "false" install-mysql-client-type: "mysql" python-versions: ${{ inputs.python-versions }} default-python-version: ${{ inputs.default-python-version }} - platform: "linux/amd64" branch: ${{ inputs.branch }} # Always build images during the extra checks and never push them push-image: "false" diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml index 71cd4e2d2903d..23e644a286de6 100644 --- a/.github/workflows/publish-docs-to-s3.yml +++ b/.github/workflows/publish-docs-to-s3.yml @@ -49,8 +49,7 @@ jobs: name: "Build Info" runs-on: ["ubuntu-24.04"] outputs: - amd-runners: ${{ steps.selective-checks.outputs.amd-runners }} - arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} + runners: ${{ steps.selective-checks.outputs.amd-runners }} env: GITHUB_CONTEXT: ${{ toJson(github) }} VERBOSE: true @@ -109,8 +108,7 @@ jobs: # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. packages: write with: - amd-runners: ${{ needs.build-info.outputs.amd-runners }} - arm-runners: ${{ needs.build-info.outputs.arm-runners }} + runners: ${{ needs.build-info.outputs.amd-runners }} platform: "linux/amd64" push-image: "false" upload-image-artifact: "true" diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index 2dc539956d7cc..d562f798c7791 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -20,12 +20,8 @@ name: Push image cache on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: - description: "The array of labels (in json form) determining public AMD runners." - required: true - type: string - arm-runners: - description: "The array of labels (in json form) determining public ARM runners." + runners: + description: "The array of labels (in json form) determining runners." required: true type: string cache-type: @@ -83,11 +79,7 @@ on: # yamllint disable-line rule:truthy jobs: push-ci-image-cache: name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache " - # NOTE!!!!! This has to be put in one line for runs-on to recognize the "fromJSON" properly !!!! - # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array" - # instead of an array of strings. - # yamllint disable-line rule:line-length - runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.amd-runners) || fromJSON(inputs.arm-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} permissions: contents: read packages: write @@ -158,11 +150,7 @@ jobs: push-prod-image-cache: name: "Push PROD ${{ inputs.cache-type }}:${{ matrix.python }} image cache" - # NOTE!!!!! This has to be put in one line for runs-on to recognize the "fromJSON" properly !!!! - # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array" - # instead of an array of strings. - # yamllint disable-line rule:line-length - runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.amd-runners) || fromJSON(inputs.arm-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} permissions: contents: read packages: write diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index cdfb7b58f0db8..2fbdb568cdde2 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -20,10 +20,14 @@ name: Unit tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: + runners: description: "The array of labels (in json form) determining public AMD runners." required: true type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" + required: true + type: string test-group: description: "Test group to run: ('core', 'providers')" required: true @@ -37,7 +41,7 @@ on: # yamllint disable-line rule:truthy required: true type: string test-scope: - description: "The scope of the test to run: ('DB', 'Non-DB', 'All', 'ARM collection')" + description: "The scope of the test to run: ('DB', 'Non-DB', 'All')" required: true type: string test-name: @@ -125,7 +129,7 @@ jobs: ${{ inputs.test-scope }}-${{ inputs.test-group }}:\ ${{ inputs.test-name }}${{ inputs.test-name-separator }}${{ matrix.backend-version }}:\ ${{ matrix.python-version}}:${{ matrix.test-types.description }}" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: fail-fast: false matrix: @@ -168,7 +172,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ matrix.python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ matrix.python-version }} use-uv: ${{ inputs.use-uv }} - name: > diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 7f20ce1111194..a41ab1693c830 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -20,8 +20,12 @@ name: Special tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: - description: "The array of labels (in json form) determining public AMD runners." + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" required: true type: string default-branch: @@ -90,7 +94,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ inputs.amd-runners }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} downgrade-sqlalchemy: "true" test-name: "MinSQLAlchemy-Postgres" test-scope: "DB" @@ -113,7 +118,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ inputs.amd-runners }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} downgrade-sqlalchemy: "true" test-name: "MinSQLAlchemy-Postgres" test-scope: "DB" @@ -136,7 +142,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ inputs.amd-runners }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} upgrade-boto: "true" test-name: "LatestBoto-Postgres" test-scope: "All" @@ -160,7 +167,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ inputs.amd-runners }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} upgrade-boto: "true" test-name: "LatestBoto-Postgres" test-scope: "All" @@ -185,7 +193,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ inputs.amd-runners }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} downgrade-pendulum: "true" test-name: "Pendulum2-Postgres" test-scope: "All" @@ -209,7 +218,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ inputs.amd-runners }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} downgrade-pendulum: "true" test-name: "Pendulum2-Postgres" test-scope: "All" @@ -233,7 +243,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ inputs.amd-runners }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} test-name: "Postgres" test-scope: "Quarantined" test-group: "core" @@ -256,7 +267,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ inputs.amd-runners }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} test-name: "Postgres" test-scope: "Quarantined" test-group: "providers" @@ -272,56 +284,6 @@ jobs: skip-providers-tests: ${{ inputs.skip-providers-tests }} use-uv: ${{ inputs.use-uv }} - - tests-arm-collection-core: - name: "ARM Collection test: core" - uses: ./.github/workflows/run-unit-tests.yml - permissions: - contents: read - packages: read - with: - amd-runners: ${{ inputs.amd-runners }} - test-name: "Postgres" - test-scope: "ARM collection" - test-group: "core" - backend: "postgres" - python-versions: "['${{ inputs.default-python-version }}']" - backend-versions: "['${{ inputs.default-postgres-version }}']" - excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} - excludes: "[]" - test-types-as-strings-in-json: ${{ inputs.core-test-types-list-as-strings-in-json }} - include-success-outputs: ${{ inputs.include-success-outputs }} - run-coverage: ${{ inputs.run-coverage }} - debug-resources: ${{ inputs.debug-resources }} - skip-providers-tests: ${{ inputs.skip-providers-tests }} - use-uv: ${{ inputs.use-uv }} - if: ${{ inputs.default-branch == 'main' }} - - tests-arm-collection-providers: - name: "ARM Collection test: providers" - uses: ./.github/workflows/run-unit-tests.yml - permissions: - contents: read - packages: read - with: - amd-runners: ${{ inputs.amd-runners }} - test-name: "Postgres" - test-scope: "ARM collection" - test-group: "providers" - backend: "postgres" - python-versions: "['${{ inputs.default-python-version }}']" - backend-versions: "['${{ inputs.default-postgres-version }}']" - excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} - excludes: "[]" - test-types-as-strings-in-json: ${{ inputs.core-test-types-list-as-strings-in-json }} - include-success-outputs: ${{ inputs.include-success-outputs }} - run-coverage: ${{ inputs.run-coverage }} - debug-resources: ${{ inputs.debug-resources }} - skip-providers-tests: ${{ inputs.skip-providers-tests }} - use-uv: ${{ inputs.use-uv }} - if: ${{ inputs.default-branch == 'main' }} - - tests-system-core: name: "System test: ${{ matrix.test-group }}" uses: ./.github/workflows/run-unit-tests.yml @@ -329,7 +291,8 @@ jobs: contents: read packages: read with: - amd-runners: ${{ inputs.amd-runners }} + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} test-name: "SystemTest" test-scope: "System" test-group: "core" diff --git a/.github/workflows/test-providers.yml b/.github/workflows/test-providers.yml index 0fbeaaee052c5..411b18daaa7b4 100644 --- a/.github/workflows/test-providers.yml +++ b/.github/workflows/test-providers.yml @@ -20,10 +20,14 @@ name: Provider tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - amd-runners: + runners: description: "The array of labels (in json form) determining public AMD runners." required: true type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" + required: true + type: string canary-run: description: "Whether this is a canary run" required: true @@ -68,7 +72,7 @@ jobs: prepare-install-verify-provider-distributions: timeout-minutes: 80 name: "Providers ${{ matrix.package-format }} tests" - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: fail-fast: false matrix: @@ -91,7 +95,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ inputs.default-python-version }} use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist files" @@ -163,7 +167,7 @@ jobs: timeout-minutes: 80 # yamllint disable rule:line-length name: Compat ${{ matrix.compat.airflow-version }}:P${{ matrix.compat.python-version }}:${{ matrix.compat.test-types.description }} - runs-on: ${{ fromJSON(inputs.amd-runners) }} + runs-on: ${{ fromJSON(inputs.runners) }} strategy: fail-fast: false matrix: @@ -189,7 +193,7 @@ jobs: - name: "Prepare breeze & CI image: ${{ matrix.compat.python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: - platform: "linux/amd64" + platform: ${{ inputs.platform }} python: ${{ matrix.compat.python-version }} use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist files" diff --git a/Dockerfile.ci b/Dockerfile.ci index 343a65d355dc4..689c5ab525628 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1071,13 +1071,6 @@ function check_run_tests() { return fi - if [[ ${REMOVE_ARM_PACKAGES:="false"} == "true" ]]; then - # Test what happens if we do not have ARM packages installed. - # This is useful to see if pytest collection works without ARM packages which is important - # for the MacOS M1 users running tests in their ARM machines with `breeze testing *-tests` command - python "${IN_CONTAINER_DIR}/remove_arm_packages.py" - fi - if [[ ${TEST_GROUP:=""} == "system" ]]; then exec "${IN_CONTAINER_DIR}/run_system_tests.sh" "${@}" else diff --git a/dev/breeze/doc/images/output_testing_core-tests.svg b/dev/breeze/doc/images/output_testing_core-tests.svg index c148fbe97129d..987c862dd1f5d 100644 --- a/dev/breeze/doc/images/output_testing_core-tests.svg +++ b/dev/breeze/doc/images/output_testing_core-tests.svg @@ -1,4 +1,4 @@ - +