diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 28c078a356347..8178cde09e093 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -44,6 +44,15 @@ on: # yamllint disable-line rule:truthy description: "Whether to generate PyPI constraints (true/false)" required: true type: string + allow-pre-releases: + description: > + Whether the PyPI constraints may pin pre-release providers (true/false). Set when + constraints are cut for a release candidate, whose providers are on PyPI only as rc + versions. Scoped to apache-airflow-providers-* - no other package can resolve to a + pre-release. + required: false + default: "false" + type: string debug-resources: description: "Whether to run in debug mode (true/false)" required: true @@ -137,9 +146,14 @@ jobs: - name: "PyPI constraints" shell: bash timeout-minutes: 25 + env: + # Deliberately not named ALLOW_PRE_RELEASES: that is the option's own envvar, and having + # both paths set it would make it unclear which one is in force. Empty on a normal run, + # so the unquoted expansion below contributes no argument at all. + PRE_RELEASE_FLAG: ${{ inputs.allow-pre-releases == 'true' && '--allow-pre-releases' || '' }} run: | breeze release-management generate-constraints --airflow-constraints-mode constraints \ - --answer yes --python "${PYTHON_VERSION}" + --answer yes --python "${PYTHON_VERSION}" ${PRE_RELEASE_FLAG} if: inputs.generate-pypi-constraints == 'true' - name: "Check for provider downgrade alert" id: downgrade-alert diff --git a/.github/workflows/release-constraints.yml b/.github/workflows/release-constraints.yml new file mode 100644 index 0000000000000..268abf8334131 --- /dev/null +++ b/.github/workflows/release-constraints.yml @@ -0,0 +1,248 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Resolves the constraints that a release ships, rather than tagging whatever the +# `constraints-X-Y` branch happened to hold when the release ran. +# +# The stage is derived from the version, so the two cannot be mismatched by hand: +# +# * a candidate (`3.1.3rc1`) resolves with pre-releases allowed - the providers of the wave +# being voted on exist on PyPI only as rc versions - and lands on a branch of its own, so a +# candidate never moves the branch every other build reads; +# * a final (`3.1.3`) resolves without them, against the providers now published as finals, and +# commits onto `constraints-X-Y` itself, which is what makes the released constraints the +# baseline everything downstream reads. +# +# A final therefore cannot be produced by retagging a candidate: a released version must never +# pin an rc. +--- +name: Release constraints +on: # yamllint disable-line rule:truthy + workflow_dispatch: + inputs: + version: + description: "Version the constraints belong to, e.g. 3.1.3rc1 or 3.1.3" + required: true + type: string + ref: + description: "Ref the constraints are resolved from, e.g. v3-1-stable or the release tag" + required: true + type: string +permissions: + contents: read +concurrency: + group: release-constraints-${{ inputs.version }} + cancel-in-progress: false +jobs: + build-info: + timeout-minutes: 10 + name: "Build info" + runs-on: ["ubuntu-22.04"] + if: contains(fromJSON('[ + "ashb", + "eladkal", + "ephraimbuddy", + "jedcunningham", + "kaxil", + "pierrejeambrun", + "potiuk", + "utkarsharma2", + "vincbeck", + ]'), github.event.sender.login) + outputs: + python-versions: ${{ steps.selective-checks.outputs.python-versions }} + python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} + default-branch: ${{ steps.selective-checks.outputs.default-branch }} + default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} + constraints-branch: ${{ steps.stage.outputs.constraints-branch }} + target-branch: ${{ steps.stage.outputs.target-branch }} + allow-pre-releases: ${{ steps.stage.outputs.allow-pre-releases }} + steps: + - name: "Cleanup repo" + shell: bash + run: sudo rm -rf ${GITHUB_WORKSPACE}/* + - name: "Checkout ${{ inputs.ref }}" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref }} + fetch-depth: 2 + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + id: breeze + - name: "Save github context to file" + # See ci-amd.yml for the full rationale: avoids ARG_MAX on big PRs by writing the + # github context to a file, and the single-quoted heredoc makes the zizmor + # template-injection finding a false positive (no bash expansion happens inside it). + shell: bash + run: | # zizmor: ignore[template-injection] + cat > "${RUNNER_TEMP}/github_context.json" << '__GITHUB_CONTEXT_END__' + ${{ toJson(github) }} + __GITHUB_CONTEXT_END__ + - name: Selective checks + id: selective-checks + env: + PR_LABELS: "[]" + COMMIT_REF: "${{ inputs.ref }}" + VERBOSE: "false" + GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} + - name: "Derive the release stage from the version" + id: stage + shell: bash + env: + VERSION: ${{ inputs.version }} + run: | + if [[ ! "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]+)?$ ]]; then + echo "'${VERSION}' is not a release version - expected X.Y.Z or X.Y.ZrcN." >&2 + exit 1 + fi + major="$(echo "${VERSION}" | cut -d. -f1)" + minor="$(echo "${VERSION}" | cut -d. -f2)" + constraints_branch="constraints-${major}-${minor}" + echo "constraints-branch=${constraints_branch}" >> "${GITHUB_OUTPUT}" + if [[ "${VERSION}" == *rc* ]]; then + echo "allow-pre-releases=true" >> "${GITHUB_OUTPUT}" + echo "target-branch=constraints-${VERSION}" >> "${GITHUB_OUTPUT}" + else + echo "allow-pre-releases=false" >> "${GITHUB_OUTPUT}" + echo "target-branch=${constraints_branch}" >> "${GITHUB_OUTPUT}" + fi + - name: "Parameters summary" + shell: bash + env: + VERSION: ${{ inputs.version }} + REF: ${{ inputs.ref }} + CONSTRAINTS_BRANCH: ${{ steps.stage.outputs.constraints-branch }} + TARGET_BRANCH: ${{ steps.stage.outputs.target-branch }} + ALLOW_PRE_RELEASES: ${{ steps.stage.outputs.allow-pre-releases }} + run: | + { + echo "## Release constraints" + echo "" + echo "| Parameter | Value |" + echo "|---|---|" + echo "| Version | \`${VERSION}\` |" + echo "| Resolved from ref | \`${REF}\` |" + echo "| Branched off | \`${CONSTRAINTS_BRANCH}\` |" + echo "| Committed to | \`${TARGET_BRANCH}\` |" + echo "| Pre-releases allowed | \`${ALLOW_PRE_RELEASES}\` |" + echo "| Tagged | \`constraints-${VERSION}\` |" + } | tee -a "${GITHUB_STEP_SUMMARY}" + + build-ci-images: + name: "Build CI images" + needs: [build-info] + uses: ./.github/workflows/ci-image-build.yml + permissions: + contents: read + packages: write + with: + runners: '["ubuntu-22.04"]' + platform: "linux/amd64" + push-image: "false" + upload-image-artifact: "true" + upload-mount-cache-artifact: "false" + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + checkout-ref: ${{ inputs.ref }} + use-uv: "true" + upgrade-to-newer-dependencies: "false" + docker-cache: "registry" + disable-airflow-repo-cache: "false" + + generate-constraints: + name: "Generate constraints" + needs: [build-info, build-ci-images] + uses: ./.github/workflows/generate-constraints.yml + with: + runners: '["ubuntu-22.04"]' + platform: "linux/amd64" + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + generate-pypi-constraints: "true" + # Only the PyPI constraints are what a release ships; the other modes serve CI, and + # regenerating them here would move them for reasons unrelated to the release. + generate-no-providers-constraints: "false" + allow-pre-releases: ${{ needs.build-info.outputs.allow-pre-releases }} + debug-resources: "false" + checkout-ref: ${{ inputs.ref }} + use-uv: "true" + secrets: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + + publish-constraints: + runs-on: ["ubuntu-22.04"] + timeout-minutes: 20 + name: "Publish and tag constraints" + needs: [build-info, generate-constraints] + permissions: + contents: write + packages: read + env: + VERSION: ${{ inputs.version }} + CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.constraints-branch }} + TARGET_BRANCH: ${{ needs.build-info.outputs.target-branch }} + steps: + - name: "Cleanup repo" + shell: bash + run: sudo rm -rf ${GITHUB_WORKSPACE}/* + - name: "Checkout ${{ inputs.ref }}" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + # Branched off the shared constraints branch in both cases; what differs is where the commit + # ends up, which the checkout below decides. + - name: "Checkout ${{ needs.build-info.outputs.constraints-branch }}" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: "constraints" + ref: ${{ needs.build-info.outputs.constraints-branch }} + persist-credentials: true + fetch-depth: 0 + - name: "Download constraints from the generate-constraints job" + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: constraints-* + path: ./files + - name: "Switch to ${{ needs.build-info.outputs.target-branch }}" + working-directory: "constraints" + shell: bash + run: git switch -c "${TARGET_BRANCH}" 2>/dev/null || git switch "${TARGET_BRANCH}" + - name: "Diff in constraints for Python: ${{ needs.build-info.outputs.python-versions-list-as-string }}" + run: ./scripts/ci/constraints/ci_diff_constraints.sh + - name: "Commit changed constraint files" + run: ./scripts/ci/constraints/ci_commit_constraints.sh + - name: "Push ${{ needs.build-info.outputs.target-branch }}" + working-directory: "constraints" + shell: bash + run: git push origin "${TARGET_BRANCH}" + - name: "Tag constraints-${{ inputs.version }}" + working-directory: "constraints" + shell: bash + run: | + git tag -a "constraints-${VERSION}" -m "Constraints for Apache Airflow ${VERSION}" + git push origin "constraints-${VERSION}" + - name: "Summary" + shell: bash + run: | + { + echo "Constraints for \`${VERSION}\` are on \`${TARGET_BRANCH}\`," + echo "tagged \`constraints-${VERSION}\`." + } | tee -a "${GITHUB_STEP_SUMMARY}" diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 33881515df933..0841475566cb2 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -588,6 +588,18 @@ still works but is no longer recommended. --sync-branch ${SYNC_BRANCH} ``` + Note: when it reaches the constraints step, `start-rc-process` triggers the `Release + constraints` workflow and waits. The candidate resolves constraints of its own rather than + tagging the `constraints-X-Y` branch tip, and it resolves them **allowing pre-releases for the + providers** — the providers of the wave being voted on exist on PyPI only as `rcN` versions, so + constraints that refused them could not describe what a tester is asked to install. The + allowance is scoped to `apache-airflow-providers-*`; no other package can resolve to a + pre-release, so a beta of some third-party library cannot slip into what the candidate ships. + + The result lands on a branch of its own (`constraints-${VERSION_RC}`) and is tagged + `constraints-${VERSION_RC}`. The shared `constraints-X-Y` branch is left where it was — only + the final release moves it. + **Testing the start-rc-process command:** Before running the actual release command, you can safely test it using: @@ -1372,15 +1384,37 @@ breeze release-management start-release \ Note: The `--task-sdk-version` parameter is optional. If you are releasing Airflow without a corresponding Task SDK release, you can omit this parameter. -Note: When it reaches the constraints step, `start-release` asks whether to base the final -`constraints-${VERSION}` tag on the latest `constraints-X-Y` branch tip instead of the RC -constraints tag. The RC constraints are frozen when the RC is cut, so if any providers were -released (or constraints were otherwise refreshed - see +Note: When it reaches the constraints step, `start-release` resolves the constraints again rather +than promoting the ones the RC was cut with. The RC constraints deliberately pin pre-releases - +that wave's providers exist on PyPI only as `rcN` versions at candidate time - so they can never +become the released constraints by retagging. The regeneration runs without pre-releases against +the same providers now published as finals, commits the result onto the `constraints-X-Y` branch, +pushes it, and tags it `constraints-${VERSION}`. That commit is what makes the released +constraints the new baseline, so refreshing the branch beforehand (see [MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md](MANUALLY_GENERATING_IMAGE_CACHE_AND_CONSTRAINTS.md)) -after the last RC and you want the released constraints to reflect that, answer **yes** to tag the -`constraints-X-Y` branch tip. Otherwise (the default) the final tag matches the RC exactly. If you -do refresh, run the `Update constraints` workflow from `main` with `ref` set to the ref you are -releasing (typically `v3-*-stable`) **before** running `start-release`. +is no longer necessary. + +The resolution runs on CI runners through the `Release constraints` workflow, so the release +manager's machine does not need a CI image for every supported Python. `start-release` triggers it +and waits. You can also run it on its own - to redo a candidate's constraints, or to produce them +for a release cut before this existed: + +```shell script +breeze workflow-run release-constraints --version ${VERSION} --ref v3-1-stable +``` + +The workflow derives the stage from `--version` alone, so there is no separate switch that could +disagree with the version: + +| `--version` | Pre-releases | Lands on | Tagged | +|---|---|---|---| +| `3.1.3rc1` | providers only | `constraints-3.1.3rc1`, branched off `constraints-3-1` | `constraints-3.1.3rc1` | +| `3.1.3` | none | `constraints-3-1` (commit) | `constraints-3.1.3` | + +"Providers only" is literal: the resolution names every `apache-airflow-providers-*` with a +pre-release lower bound and runs under uv's `explicit` strategy, which permits a pre-release solely +for a package marked that way. Nothing else in the dependency graph can resolve to one — which is +why `--pre` is not used, since it would apply to the whole resolution. 4. Make sure to update Airflow version in ``v3-*-test`` branch after cherry-picking to X.Y.1 in diff --git a/dev/breeze/doc/09_release_management_tasks.rst b/dev/breeze/doc/09_release_management_tasks.rst index 70453de37db6e..40694e437c660 100644 --- a/dev/breeze/doc/09_release_management_tasks.rst +++ b/dev/breeze/doc/09_release_management_tasks.rst @@ -1080,6 +1080,27 @@ These are all available flags of ``workflow-run publish-docs`` command: :width: 100% :alt: Breeze workflow-run publish-docs +Resolving the constraints for a release +""""""""""""""""""""""""""""""""""""""" + +To trigger the GitHub Actions workflow that resolves, publishes and tags the constraints belonging to a +release, you can use the ``breeze workflow-run release-constraints`` command. The release commands trigger +it themselves, so this is for redoing a candidate's constraints or producing them for a release cut before +the workflow existed. + +The stage is derived from ``--version`` alone, so it cannot be set inconsistently with it. A candidate +(``3.1.3rc1``) resolves allowing pre-releases for ``apache-airflow-providers-*`` — the providers of the wave +being voted on are on PyPI only as ``rcN`` versions — and lands on a branch of its own, leaving the shared +``constraints-X-Y`` branch where it was. A final (``3.1.3``) resolves without them and commits onto +``constraints-X-Y``, which is what makes the released constraints the baseline everything downstream reads. + +These are all available flags of ``workflow-run release-constraints`` command: + +.. image:: ./images/output_workflow-run_release-constraints.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_workflow-run_release-constraints.svg + :width: 100% + :alt: Breeze workflow-run release-constraints + Publishing the schema files to S3 """"""""""""""""""""""""""""""""" diff --git a/dev/breeze/doc/images/output_release-management_generate-constraints.svg b/dev/breeze/doc/images/output_release-management_generate-constraints.svg index 9a2a1328ea1d0..07e3538c1cef6 100644 --- a/dev/breeze/doc/images/output_release-management_generate-constraints.svg +++ b/dev/breeze/doc/images/output_release-management_generate-constraints.svg @@ -1,4 +1,4 @@ - +