From 8a7aaea34e906e5967a3b279dbc1c43bca9572cf Mon Sep 17 00:00:00 2001 From: galargh Date: Thu, 23 Feb 2023 20:11:04 +0100 Subject: [PATCH 1/8] feat: create scheduled workflow which merges web3-bot PRs --- .github/workflows/merge-prs.yml | 62 +++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/merge-prs.yml diff --git a/.github/workflows/merge-prs.yml b/.github/workflows/merge-prs.yml new file mode 100644 index 00000000..c79ea236 --- /dev/null +++ b/.github/workflows/merge-prs.yml @@ -0,0 +1,62 @@ +name: Merge PRs + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * *" # https://crontab.guru/every-day + +jobs: + dispatch: + name: Merge PRs in targets + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.WEB3BOT_GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.workflow_run.head_commit.id || github.sha }} + - name: Merge PRs in targets that need it + env: + PR_BRANCH: 'web3-bot/sync' + run: | + targets=() + for config in configs/*.json; do + targets+=($(jq -rc ".repositories[] | .target" $config)) + done + failed=() + for target in ${targets[@]}; do + echo "Processing $target" + base="$(gh api "/repos/$target" --jq '.default_branch')" + pr="$(gh api -X GET "/repos/$target/pulls" -f head="$(echo "$target" | cut -d/ -f1):$PR_BRANCH" -f base="$base" -f state=open --jq '.[0]')" + number="$(jq -j '.number' <<< "$pr")" + head_sha="$(jq -j '.head.sha' <<< "$pr")" + user_login="$(jq -j '.user.login' <<< "$pr")" + # checks if a PR exists + if [[ "$pr" == '0' ]] ; then + echo "The PR does not exist. Skipping." + continue + fi + # checks if the PR was created by web3-bot + if [[ "$user_login" != 'web3-bot' ]]; then + echo "The PR wasn't created by web3-bot. Skipping." + continue + fi + mergeable_state="$(gh api -X GET "/repos/$target/pulls/$number" --jq '.mergeable_state')" + # checks if the PR can be merged + if [[ "$mergeable_state" != "clean" ]]; then + echo "The PR cannot be merged because it's in '$mergeable_state'. Skipping." + continue + fi + # tries to merge the PR in target + if result="$(gh api -X PUT "/repos/$target/pulls/$number/merge" -f merge_method=squash)"; then + echo "Successfully merged the PR for '$target'" + else + echo "$result" + echo "Failed to merge the PR for '$target'" + failed+=("$target") + fi + done + if ((${#failed[@]})); then + echo "::error ::Failed to merge PRs in: ${failed[@]}" + exit 1 + fi From aeee1ae82de401ec7c7d709807e83d95b47c3ece Mon Sep 17 00:00:00 2001 From: galargh Date: Thu, 23 Feb 2023 20:12:41 +0100 Subject: [PATCH 2/8] chore: clean up automerge.yml --- .github/workflows/automerge.yml | 58 ----------------------- README.md | 2 - configs/go.json | 1 - configs/js.json | 1 - templates/.github/workflows/automerge.yml | 8 ---- templates/README.md | 8 ++-- 6 files changed, 4 insertions(+), 74 deletions(-) delete mode 100644 .github/workflows/automerge.yml delete mode 100644 templates/.github/workflows/automerge.yml diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml deleted file mode 100644 index ba91bdb5..00000000 --- a/.github/workflows/automerge.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Automatically merge pull requests opened by web3-bot, as soon as (and only if) all tests pass. -# This reduces the friction associated with updating with our workflows. - -on: - workflow_call: - inputs: - job: - required: true - type: string -name: Automerge - -jobs: - automerge-check: - if: github.event.pull_request.user.login == 'web3-bot' - runs-on: ubuntu-latest - outputs: - status: ${{ steps.should-automerge.outputs.status }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Check if we should automerge - id: should-automerge - env: - BASE_REF: ${{ github.event.pull_request.base.ref }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - for commit in $(git rev-list --first-parent origin/$BASE_REF..$HEAD_SHA); do - committer=$(git show --format=$'%ce' -s $commit) - echo "Committer: $committer" - if [[ "$committer" != "web3-bot@users.noreply.github.com" ]]; then - echo "Commit $commit wasn't committed by web3-bot, but by $committer." - echo "status=false" >> $GITHUB_OUTPUT - exit - fi - done - echo "status=true" >> $GITHUB_OUTPUT - automerge: - needs: automerge-check - runs-on: ubuntu-latest - # The check for the user is redundant here, as this job depends on the automerge-check job, - # but it prevents this job from spinning up, just to be skipped shortly after. - if: github.event.pull_request.user.login == 'web3-bot' && needs.automerge-check.outputs.status == 'true' - steps: - - name: Wait on tests - uses: lewagon/wait-on-check-action@3a563271c3f8d1611ed7352809303617ee7e54ac # v1.2.0 - with: - ref: ${{ github.event.pull_request.head.sha }} - repo-token: ${{ github.token }} - wait-interval: 10 - running-workflow-name: '${{ inputs.job }} / ${{ github.job }}' # the name of the check for this job - - name: Merge PR - uses: pascalgn/automerge-action@eb68b061739cb9d81564f8e812d0b3c45f0fb09a # v0.15.5 - env: - GITHUB_TOKEN: "${{ github.token }}" - MERGE_LABELS: "" - MERGE_METHOD: "squash" - MERGE_DELETE_BRANCH: true diff --git a/README.md b/README.md index 7d3e59f2..3b4b64de 100644 --- a/README.md +++ b/README.md @@ -97,8 +97,6 @@ This repository currently defines two workflows for Go repositories: * [go-test](templates/.github/workflows/go-test.yml): Runs all tests, using different compiler versions and operating systems. Whenever one of these workflows is changed, this repository runs the [copy workflow](.github/workflows/copy-workflow.yml). This workflow creates a pull request in every participating repository to update *go-check* and *go-test*. -In order to help with the distribution of these workflows, this repository defines two additional workflows that are distributed across participating repositories: -* [automerge](templates/.github/workflows/automerge.yml): In most cases, an update to the workflows will not cause CI to fail in most participating repositories. To make our life easier, *automerge* automatically merges the pull request if all checks succeed. ## Usage diff --git a/configs/go.json b/configs/go.json index 1f6af0e2..e830df6f 100644 --- a/configs/go.json +++ b/configs/go.json @@ -1,7 +1,6 @@ { "defaults": { "files": [ - ".github/workflows/automerge.yml", ".github/workflows/go-test.yml", ".github/workflows/go-check.yml", ".github/workflows/releaser.yml", diff --git a/configs/js.json b/configs/js.json index f2ddd492..cffac43b 100644 --- a/configs/js.json +++ b/configs/js.json @@ -1,7 +1,6 @@ { "defaults": { "files": [ - ".github/workflows/automerge.yml", ".github/workflows/js-test-and-release.yml" ] }, diff --git a/templates/.github/workflows/automerge.yml b/templates/.github/workflows/automerge.yml deleted file mode 100644 index d57c2a02..00000000 --- a/templates/.github/workflows/automerge.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: Automerge -on: [ pull_request ] - -jobs: - automerge: - uses: protocol/.github/.github/workflows/automerge.yml@master - with: - job: 'automerge' diff --git a/templates/README.md b/templates/README.md index a6ea6a2f..53ca95e6 100644 --- a/templates/README.md +++ b/templates/README.md @@ -55,13 +55,13 @@ The `github` context contains information about the target repository the file i ```json { "defaults": { - "files": [".github/workflows/automerge.yml"], + "files": [".github/workflows/example1.yml"], "is_example": false }, "repositories": [ { "target": "protocol/.github-test-target", - "extra_files": [".github/workflows/example.yml"], + "extra_files": [".github/workflows/example2.yml"], "example": { "greeting": "Hello" }, @@ -79,8 +79,8 @@ The `github` context contains information about the target repository the file i { "config": { "target": "protocol/.github-test-target", - "files": [".github/workflows/automerge.yml"], - "extra_files": [".github/workflows/example.yml"], + "files": [".github/workflows/example1.yml"], + "extra_files": [".github/workflows/example2.yml"], "example": { "greeting": "Hello" }, From 558204d1ce7709eccffa00e93c88b900125c3a74 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 24 Feb 2023 10:30:25 +0100 Subject: [PATCH 3/8] chore: rewrite merge-prs using github-script --- .github/workflows/merge-prs.yml | 96 ++++++++++++++++----------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/merge-prs.yml b/.github/workflows/merge-prs.yml index c79ea236..97802011 100644 --- a/.github/workflows/merge-prs.yml +++ b/.github/workflows/merge-prs.yml @@ -7,56 +7,56 @@ on: jobs: dispatch: - name: Merge PRs in targets + name: Merge PRs runs-on: ubuntu-latest env: GITHUB_TOKEN: ${{ secrets.WEB3BOT_GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.workflow_run.head_commit.id || github.sha }} - - name: Merge PRs in targets that need it + - name: Merge PRs env: - PR_BRANCH: 'web3-bot/sync' - run: | - targets=() - for config in configs/*.json; do - targets+=($(jq -rc ".repositories[] | .target" $config)) - done - failed=() - for target in ${targets[@]}; do - echo "Processing $target" - base="$(gh api "/repos/$target" --jq '.default_branch')" - pr="$(gh api -X GET "/repos/$target/pulls" -f head="$(echo "$target" | cut -d/ -f1):$PR_BRANCH" -f base="$base" -f state=open --jq '.[0]')" - number="$(jq -j '.number' <<< "$pr")" - head_sha="$(jq -j '.head.sha' <<< "$pr")" - user_login="$(jq -j '.user.login' <<< "$pr")" - # checks if a PR exists - if [[ "$pr" == '0' ]] ; then - echo "The PR does not exist. Skipping." - continue - fi - # checks if the PR was created by web3-bot - if [[ "$user_login" != 'web3-bot' ]]; then - echo "The PR wasn't created by web3-bot. Skipping." - continue - fi - mergeable_state="$(gh api -X GET "/repos/$target/pulls/$number" --jq '.mergeable_state')" - # checks if the PR can be merged - if [[ "$mergeable_state" != "clean" ]]; then - echo "The PR cannot be merged because it's in '$mergeable_state'. Skipping." - continue - fi - # tries to merge the PR in target - if result="$(gh api -X PUT "/repos/$target/pulls/$number/merge" -f merge_method=squash)"; then - echo "Successfully merged the PR for '$target'" - else - echo "$result" - echo "Failed to merge the PR for '$target'" - failed+=("$target") - fi - done - if ((${#failed[@]})); then - echo "::error ::Failed to merge PRs in: ${failed[@]}" - exit 1 - fi + QUERY: is:pr author:web3-bot state:open status:success head:web3-bot/sync archived:false + uses: actions/github-script@v6 + with: + retries: 12 + script: | + core.info(`Looking for PRs matching the query: ${process.env.QUERY}`) + const items = await github.paginate(github.rest.search.issuesAndPullRequests, { + q: process.env.QUERY + }) + core.info(`Filtering out the PRs that cannot be merged`) + const prs = [] + for (const item of items) { + core.info(`Retrieving ${item.html_url}`) + const [_, owner, repo] = item.url.match(/repos\/(.+?)\/(.+?)\/issues/) + const pr = await github.rest.pulls.get({ + owner, + repo, + pull_number: item.number + }) + if (pr.mergeable_state == 'clean') { + core.debug(`${pr.html_url} can be merged`) + prs.push(pr) + } else { + core.debug(`${pr.html_url} cannot be merged`) + } + } + core.info(`Attempting to merge the PRs`) + const failed = [] + for (const pr of prs) { + core.info(`Merging ${pr.html_url}`) + try { + await octokit.rest.pulls.merge({ + owner: pr.base.repo.owner.login, + repo: pr.base.repo.name, + pull_number: pr.number, + merge_method: 'squash' + }) + core.debug(`Merged ${pr.html_url}`) + } catch(error) { + core.error(`Couldn't merge ${pr.html_url}, got: ${error}`) + failed.push(pr) + } + } + if (failed.length != 0) { + throw new Error(`Failed to merge ${failed.length} PRs`) + } From 1496e51024314dd0e5fdf2729bd3b4416c4fe529 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 24 Feb 2023 11:10:46 +0100 Subject: [PATCH 4/8] chore: rewrite create-prs using github-script --- .github/workflows/create-prs.yml | 137 +++++++++++++++++-------------- 1 file changed, 77 insertions(+), 60 deletions(-) diff --git a/.github/workflows/create-prs.yml b/.github/workflows/create-prs.yml index 437eeb37..e2ce729e 100644 --- a/.github/workflows/create-prs.yml +++ b/.github/workflows/create-prs.yml @@ -9,67 +9,84 @@ on: jobs: dispatch: - name: Create PRs in targets + name: Create PRs runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.WEB3_BOT_GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.workflow_run.head_commit.id || github.sha }} - - name: Sync PRs in targets that need it + - name: Create PRs env: - PR_TITLE: 'ci: update Unified CI configuration' - PR_BRANCH: 'web3-bot/sync' - run: | - targets=() - for config in configs/*.json; do - targets+=($(jq -rc ".repositories[] | .target" $config)) - done - failed=() - for target in ${targets[@]}; do - echo "Processing $target" - base="$(gh api "/repos/$target" --jq '.default_branch')" - # checks if a PR needs to be created - if [[ "$(gh api -X GET "/repos/$target/compare/$base...$PR_BRANCH" --jq '.status')" == 'ahead' ]]; then - if [[ "$(gh api -X GET "/repos/$target/pulls" -f head="$(echo "$target" | cut -d/ -f1):$PR_BRANCH" -f base="$base" --jq 'length')" != '0' ]] ; then - echo "The PR already exists. Skipping." + BRANCH: web3-bot/sync + GITHUB_TOKEN: ${{ secrets.WEB3_BOT_GITHUB_TOKEN }} + uses: actions/github-script@v6 + with: + retries: 12 + script: | + core.info(`Looking for repositories the user has direct access to`) + const items = await github.paginate(octokit.rest.repos.listForAuthenticatedUser, { + affiliation: 'collaborator' + }) + core.info(`Filtering out the repositories without unmerged branches`) + const repos = [] + for (const item of items) { + core.info(`Checking if a PR can be created for ${item.html_url}`) + let branch + try { + branch = await github.rest.repos.getBranch({ + owner: item.owner.login, + repo: item.name, + branch: process.env.BRANCH + }) + } catch(error) { + if (error.status != 404) { + throw error + } + core.debug(error) + } + if (branch != undefined) { + core.debug(`The branch exists in ${item.html_url}`) + } else { + core.debug(`The branch does not exist in ${item.html_url}`) + continue + } + const compare = await github.rest.repos.compareCommitsWithBasehead({ + owner: item.owner.login, + repo: item.name, + basehead: `${item.default_branch}...${branch.name}` + }) + if (compare.status == 'ahead') { + core.debug(`PR for ${item.html_url} can be created`) + } else { + core.debug(`PR for ${item.html_url} cannot be created`) continue - fi - else - echo "The branch does not exist or has diverged from $base. Skipping." - continue - fi - # tries to create a PR in target - pr_create_attempt=1 - pr_create_max_attempts=12 - pr_create_attempt_interval_in_seconds=1 - pr_create_cooldown_in_seconds=1 - # max cumulative sleep time - 68.25 minutes - while true; do - if result="$(gh api "/repos/$target/pulls" -f title="$PR_TITLE" -f head="$PR_BRANCH" -f base="$base")"; then - echo "Successfully created a PR for '$target' ($pr_create_attempt/$pr_create_max_attempts)" - echo "Sleeping for $pr_create_cooldown_in_seconds seconds before creating a next one" - sleep $pr_create_cooldown_in_seconds - break - fi - if [[ "$(jq -r '.message' <<< "$result")" == 'You have exceeded a secondary rate limit and have been temporarily blocked from content creation. Please retry your request again later.' ]]; then - if (( pr_create_attempt < pr_create_max_attempts )); then - echo "Failed to create a PR for '$target' due to secondary rate limit ($pr_create_attempt/$pr_create_max_attempts)" - echo "Sleeping for $pr_create_attempt_interval_in_seconds seconds before trying again" - sleep $pr_create_attempt_interval_in_seconds - pr_create_attempt_interval_in_seconds=$((pr_create_attempt_interval_in_seconds * 2)) - pr_create_attempt=$((pr_create_attempt + 1)) - continue - fi - fi - echo "$result" - echo "Failed to create a PR for '$target' ($pr_create_attempt/$pr_create_max_attempts)" - failed+=("$target") - break - done - done - if ((${#failed[@]})); then - echo "::error ::Failed to sync PRs in: ${failed[@]}" - exit 1 - fi + } + const pulls = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: item.owner.login, + repo: item.name, + commit_sha: branch.commit.sha + }) + if (pulls.length == 0) { + core.debug(`The PR does not exist yet in ${item.html_url}`) + repos.push(item) + } else { + core.debug('The PR already exists at ${pulls[0].html_url}') + } + } + core.info(`Attempting to create the PRs`) + const failed = [] + for (const repo of repos) { + core.debug(`Creating PR in ${repo.html_url}`) + try { + const pr = await octokit.rest.pulls.create({ + owner: repo.owner.login, + repo: repo.name, + head: process.env.BRANCH, + base: repo.default_branch + }) + core.debug(`Created PR at ${pr.html_url}`) + } catch(error) { + core.error(`Couldn't create a PR for ${repo.html_url}, got: ${error}`) + failed.push(repo) + } + } + if (failed.length != 0) { + throw new Error(`Failed to create PRs in ${failed.length} repos`) + } From b8839935797ae7967ce4edf5e10f0148ec5ac50e Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 24 Feb 2023 11:36:23 +0100 Subject: [PATCH 5/8] feat: create workflow which can delete workflows --- .github/workflows/delete-workflow.yml | 77 +++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 .github/workflows/delete-workflow.yml diff --git a/.github/workflows/delete-workflow.yml b/.github/workflows/delete-workflow.yml new file mode 100644 index 00000000..c853e5a6 --- /dev/null +++ b/.github/workflows/delete-workflow.yml @@ -0,0 +1,77 @@ +name: Delete workflow + +on: + workflow_dispatch: + inputs: + path: + description: The path of the workflow to delete + required: true + dry-run: + description: Whether to run the workflow in dry-run mode + required: false + default: 'true' + +jobs: + dispatch: + name: Create PRs + runs-on: ubuntu-latest + steps: + - name: Create PRs + env: + DRY_RUN: ${{ github.event.inputs.dry-run }} + PATH: ${{ github.event.inputs.path }} + GITHUB_TOKEN: ${{ secrets.WEB3BOT_GITHUB_TOKEN }} + uses: actions/github-script@v6 + with: + script: | + core.info(`Looking for repositories the user has direct access to`) + const items = await github.paginate(octokit.rest.repos.listForAuthenticatedUser, { + affiliation: 'collaborator' + }) + core.info(`Filtering out the repositories without the file`) + const files = [] + for (const item of items) { + core.info(`Checking if the repo at ${item.html_url} contains the file`) + try { + const file = await github.rest.repos.getContent({ + owner: item.owner.login, + repo: item.name, + path: process.env.PATH + }) + core.debug(`The file exists in ${item.html_url}`) + files.push({ + ...file, + repo: item + }) + } catch (err) { + if (err.status != 404) { + throw err + } + core.debug(`The file does not exist in ${item.html_url}`) + } + } + core.info(`Attempting to delete the files`) + const failed = [] + for (const file of files) { + if (process.env.DRY_RUN == 'true') { + core.info(`Would delete the file ${file.html_url}`) + continue + } + core.debug(`Deleting the file ${file.html_url}`) + try { + await github.rest.repos.deleteFile({ + owner: file.repo.owner.login, + repo: file.repo.name, + path: process.env.PATH, + message: `ci: delete ${process.env.PATH}`, + sha: file.sha + }) + core.debug(`Deleted the file ${file.html_url}`) + } catch(error) { + core.error(`Couldn't delete the file ${file.html_url}, got: ${error}`) + failed.push(file) + } + } + if (failed.length != 0) { + throw new Error(`Failed to delete ${failed.length} files`) + } From 6e1e88cedd8e6f7316c111dd40a11b83f0752289 Mon Sep 17 00:00:00 2001 From: galargh Date: Wed, 8 Mar 2023 09:14:27 +0100 Subject: [PATCH 6/8] ci: expand logging --- .github/workflows/create-prs.yml | 18 +++++++++--------- .github/workflows/merge-prs.yml | 6 +++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/create-prs.yml b/.github/workflows/create-prs.yml index e2ce729e..9779cb85 100644 --- a/.github/workflows/create-prs.yml +++ b/.github/workflows/create-prs.yml @@ -39,12 +39,12 @@ jobs: if (error.status != 404) { throw error } - core.debug(error) + core.info(error) } if (branch != undefined) { - core.debug(`The branch exists in ${item.html_url}`) + core.info(`The branch exists in ${item.html_url}`) } else { - core.debug(`The branch does not exist in ${item.html_url}`) + core.info(`The branch does not exist in ${item.html_url}`) continue } const compare = await github.rest.repos.compareCommitsWithBasehead({ @@ -53,9 +53,9 @@ jobs: basehead: `${item.default_branch}...${branch.name}` }) if (compare.status == 'ahead') { - core.debug(`PR for ${item.html_url} can be created`) + core.info(`PR for ${item.html_url} can be created`) } else { - core.debug(`PR for ${item.html_url} cannot be created`) + core.info(`PR for ${item.html_url} cannot be created`) continue } const pulls = await github.rest.repos.listPullRequestsAssociatedWithCommit({ @@ -64,16 +64,16 @@ jobs: commit_sha: branch.commit.sha }) if (pulls.length == 0) { - core.debug(`The PR does not exist yet in ${item.html_url}`) + core.info(`The PR does not exist yet in ${item.html_url}`) repos.push(item) } else { - core.debug('The PR already exists at ${pulls[0].html_url}') + core.info('The PR already exists at ${pulls[0].html_url}') } } core.info(`Attempting to create the PRs`) const failed = [] for (const repo of repos) { - core.debug(`Creating PR in ${repo.html_url}`) + core.info(`Creating PR in ${repo.html_url}`) try { const pr = await octokit.rest.pulls.create({ owner: repo.owner.login, @@ -81,7 +81,7 @@ jobs: head: process.env.BRANCH, base: repo.default_branch }) - core.debug(`Created PR at ${pr.html_url}`) + core.info(`Created PR at ${pr.html_url}`) } catch(error) { core.error(`Couldn't create a PR for ${repo.html_url}, got: ${error}`) failed.push(repo) diff --git a/.github/workflows/merge-prs.yml b/.github/workflows/merge-prs.yml index 97802011..df4fc8e7 100644 --- a/.github/workflows/merge-prs.yml +++ b/.github/workflows/merge-prs.yml @@ -34,10 +34,10 @@ jobs: pull_number: item.number }) if (pr.mergeable_state == 'clean') { - core.debug(`${pr.html_url} can be merged`) + core.info(`${pr.html_url} can be merged`) prs.push(pr) } else { - core.debug(`${pr.html_url} cannot be merged`) + core.info(`${pr.html_url} cannot be merged`) } } core.info(`Attempting to merge the PRs`) @@ -51,7 +51,7 @@ jobs: pull_number: pr.number, merge_method: 'squash' }) - core.debug(`Merged ${pr.html_url}`) + core.info(`Merged ${pr.html_url}`) } catch(error) { core.error(`Couldn't merge ${pr.html_url}, got: ${error}`) failed.push(pr) From 90d2e6432da7d7d585c8a91c8c6d2732e905d206 Mon Sep 17 00:00:00 2001 From: galargh Date: Wed, 8 Mar 2023 09:18:52 +0100 Subject: [PATCH 7/8] fix: pr related github scripts --- .github/workflows/create-prs.yml | 43 ++++++++++++++----- .github/workflows/delete-workflow.yml | 61 ++++++++++++++++++++------- .github/workflows/merge-prs.yml | 35 ++++++++++++--- 3 files changed, 107 insertions(+), 32 deletions(-) diff --git a/.github/workflows/create-prs.yml b/.github/workflows/create-prs.yml index 9779cb85..fbbe707d 100644 --- a/.github/workflows/create-prs.yml +++ b/.github/workflows/create-prs.yml @@ -15,13 +15,37 @@ jobs: - name: Create PRs env: BRANCH: web3-bot/sync - GITHUB_TOKEN: ${{ secrets.WEB3_BOT_GITHUB_TOKEN }} uses: actions/github-script@v6 with: - retries: 12 + github-token: ${{ secrets.WEB3_BOT_GITHUB_TOKEN }} + retries: 0 script: | + const request = async function(req, opts) { + try { + return await req(opts) + } catch(err) { + opts.request.retries = (opts.request.retries || 0) + 1 + if (err.status === 403) { + if (err.response.headers['x-ratelimit-remaining'] === '0') { + const retryAfter = err.response.headers['x-ratelimit-reset'] - Math.floor(Date.now() / 1000) || 1 + core.info(`Rate limit exceeded, retrying in ${retryAfter} seconds`) + await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) + return request(req, opts) + } + if (err.message.toLowerCase().includes('secondary rate limit')) { + const retryAfter = Math.pow(2, opts.request.retries) + core.info(`Secondary rate limit exceeded, retrying in ${retryAfter} seconds`) + await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) + return request(req, opts) + } + } + throw err + } + } + github.hook.wrap('request', request) + core.info(`Looking for repositories the user has direct access to`) - const items = await github.paginate(octokit.rest.repos.listForAuthenticatedUser, { + const items = await github.paginate(github.rest.repos.listForAuthenticatedUser, { affiliation: 'collaborator' }) core.info(`Filtering out the repositories without unmerged branches`) @@ -30,16 +54,15 @@ jobs: core.info(`Checking if a PR can be created for ${item.html_url}`) let branch try { - branch = await github.rest.repos.getBranch({ + branch = (await github.rest.repos.getBranch({ owner: item.owner.login, repo: item.name, branch: process.env.BRANCH - }) + }))?.data } catch(error) { if (error.status != 404) { throw error } - core.info(error) } if (branch != undefined) { core.info(`The branch exists in ${item.html_url}`) @@ -47,7 +70,7 @@ jobs: core.info(`The branch does not exist in ${item.html_url}`) continue } - const compare = await github.rest.repos.compareCommitsWithBasehead({ + const {data: compare} = await github.rest.repos.compareCommitsWithBasehead({ owner: item.owner.login, repo: item.name, basehead: `${item.default_branch}...${branch.name}` @@ -58,7 +81,7 @@ jobs: core.info(`PR for ${item.html_url} cannot be created`) continue } - const pulls = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + const {data: pulls} = await github.rest.repos.listPullRequestsAssociatedWithCommit({ owner: item.owner.login, repo: item.name, commit_sha: branch.commit.sha @@ -67,7 +90,7 @@ jobs: core.info(`The PR does not exist yet in ${item.html_url}`) repos.push(item) } else { - core.info('The PR already exists at ${pulls[0].html_url}') + core.info(`The PR already exists at ${pulls[0].html_url}`) } } core.info(`Attempting to create the PRs`) @@ -81,7 +104,7 @@ jobs: head: process.env.BRANCH, base: repo.default_branch }) - core.info(`Created PR at ${pr.html_url}`) + core.info(`${pr.html_url} created successfully`) } catch(error) { core.error(`Couldn't create a PR for ${repo.html_url}, got: ${error}`) failed.push(repo) diff --git a/.github/workflows/delete-workflow.yml b/.github/workflows/delete-workflow.yml index c853e5a6..9e351d48 100644 --- a/.github/workflows/delete-workflow.yml +++ b/.github/workflows/delete-workflow.yml @@ -18,46 +18,75 @@ jobs: steps: - name: Create PRs env: - DRY_RUN: ${{ github.event.inputs.dry-run }} + DRY_RUN: ${{ github.event.inputs.dry-run || 'true' }} PATH: ${{ github.event.inputs.path }} - GITHUB_TOKEN: ${{ secrets.WEB3BOT_GITHUB_TOKEN }} uses: actions/github-script@v6 with: + github-token: ${{ secrets.WEB3_BOT_GITHUB_TOKEN }} + retries: 0 script: | + const request = async function(req, opts) { + try { + return await req(opts) + } catch(err) { + opts.request.retries = (opts.request.retries || 0) + 1 + if (err.status === 403) { + if (err.response.headers['x-ratelimit-remaining'] === '0') { + const retryAfter = err.response.headers['x-ratelimit-reset'] - Math.floor(Date.now() / 1000) || 1 + core.info(`Rate limit exceeded, retrying in ${retryAfter} seconds`) + await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) + return request(req, opts) + } + if (err.message.toLowerCase().includes('secondary rate limit')) { + const retryAfter = Math.pow(2, opts.request.retries) + core.info(`Secondary rate limit exceeded, retrying in ${retryAfter} seconds`) + await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) + return request(req, opts) + } + } + throw err + } + } + github.hook.wrap('request', request) + core.info(`Looking for repositories the user has direct access to`) - const items = await github.paginate(octokit.rest.repos.listForAuthenticatedUser, { + const items = await github.paginate(github.rest.repos.listForAuthenticatedUser, { affiliation: 'collaborator' }) core.info(`Filtering out the repositories without the file`) const files = [] for (const item of items) { - core.info(`Checking if the repo at ${item.html_url} contains the file`) + core.info(`Checking if there is a file in ${item.html_url}`) + let file try { - const file = await github.rest.repos.getContent({ + file = (await github.rest.repos.getContent({ owner: item.owner.login, repo: item.name, path: process.env.PATH - }) - core.debug(`The file exists in ${item.html_url}`) + }))?.data + } catch(error) { + if (error.status != 404) { + throw error + } + } + if (file) { + core.info(`${file.html_url} exists`) files.push({ ...file, repo: item }) - } catch (err) { - if (err.status != 404) { - throw err - } - core.debug(`The file does not exist in ${item.html_url}`) + } else { + core.info(`The file does not exist in ${item.html_url}`) } } core.info(`Attempting to delete the files`) const failed = [] for (const file of files) { if (process.env.DRY_RUN == 'true') { - core.info(`Would delete the file ${file.html_url}`) + core.info(`Would have deleted ${file.html_url}`) continue } - core.debug(`Deleting the file ${file.html_url}`) + core.debug(`Deleting ${file.html_url}`) try { await github.rest.repos.deleteFile({ owner: file.repo.owner.login, @@ -66,9 +95,9 @@ jobs: message: `ci: delete ${process.env.PATH}`, sha: file.sha }) - core.debug(`Deleted the file ${file.html_url}`) + core.info(`${file.html_url} deleted successfully`) } catch(error) { - core.error(`Couldn't delete the file ${file.html_url}, got: ${error}`) + core.error(`Couldn't delete ${file.html_url}, got: ${error}`) failed.push(file) } } diff --git a/.github/workflows/merge-prs.yml b/.github/workflows/merge-prs.yml index df4fc8e7..5bccc4e2 100644 --- a/.github/workflows/merge-prs.yml +++ b/.github/workflows/merge-prs.yml @@ -9,16 +9,39 @@ jobs: dispatch: name: Merge PRs runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.WEB3BOT_GITHUB_TOKEN }} steps: - name: Merge PRs env: - QUERY: is:pr author:web3-bot state:open status:success head:web3-bot/sync archived:false + QUERY: is:pr author:web3-bot state:open head:web3-bot/sync archived:false uses: actions/github-script@v6 with: - retries: 12 + github-token: ${{ secrets.WEB3_BOT_GITHUB_TOKEN }} + retries: 0 script: | + const request = async function(req, opts) { + try { + return await req(opts) + } catch(err) { + opts.request.retries = (opts.request.retries || 0) + 1 + if (err.status === 403) { + if (err.response.headers['x-ratelimit-remaining'] === '0') { + const retryAfter = err.response.headers['x-ratelimit-reset'] - Math.floor(Date.now() / 1000) || 1 + core.info(`Rate limit exceeded, retrying in ${retryAfter} seconds`) + await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) + return request(req, opts) + } + if (err.message.toLowerCase().includes('secondary rate limit')) { + const retryAfter = Math.pow(2, opts.request.retries) + core.info(`Secondary rate limit exceeded, retrying in ${retryAfter} seconds`) + await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) + return request(req, opts) + } + } + throw err + } + } + github.hook.wrap('request', request) + core.info(`Looking for PRs matching the query: ${process.env.QUERY}`) const items = await github.paginate(github.rest.search.issuesAndPullRequests, { q: process.env.QUERY @@ -28,7 +51,7 @@ jobs: for (const item of items) { core.info(`Retrieving ${item.html_url}`) const [_, owner, repo] = item.url.match(/repos\/(.+?)\/(.+?)\/issues/) - const pr = await github.rest.pulls.get({ + const {data: pr} = await github.rest.pulls.get({ owner, repo, pull_number: item.number @@ -51,7 +74,7 @@ jobs: pull_number: pr.number, merge_method: 'squash' }) - core.info(`Merged ${pr.html_url}`) + core.info(`${pr.html_url} merged successfully`) } catch(error) { core.error(`Couldn't merge ${pr.html_url}, got: ${error}`) failed.push(pr) From 562a52581ef77b78446f2ac73fa0fd009de48def Mon Sep 17 00:00:00 2001 From: galargh Date: Wed, 8 Mar 2023 22:23:27 +0100 Subject: [PATCH 8/8] fix: job names in delete workflow --- .github/workflows/delete-workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/delete-workflow.yml b/.github/workflows/delete-workflow.yml index 9e351d48..12344666 100644 --- a/.github/workflows/delete-workflow.yml +++ b/.github/workflows/delete-workflow.yml @@ -13,13 +13,13 @@ on: jobs: dispatch: - name: Create PRs + name: Delete workflow runs-on: ubuntu-latest steps: - - name: Create PRs + - name: Delete workflow env: DRY_RUN: ${{ github.event.inputs.dry-run || 'true' }} - PATH: ${{ github.event.inputs.path }} + PATH: ${{ github.event.inputs.path || '.github/workflows/automerge.yml' }} uses: actions/github-script@v6 with: github-token: ${{ secrets.WEB3_BOT_GITHUB_TOKEN }}