Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions .github/workflows/release-failure-notification.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
name: Release failure notification

on:
workflow_run:
workflows: [Release]
types: [completed]
branches: [main]

permissions:
actions: read
contents: read

jobs:
notify-failed:
name: Notify Slack - Failed
runs-on: ubuntu-latest
if: >-
github.event.workflow_run.head_repository.full_name == github.repository &&
(github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch')
timeout-minutes: 10
steps:
# This workflow must survive cancellation or runner startup failure in Release.
# Never execute code from the triggering run's head in this privileged workflow.
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.repository.default_branch }}
persist-credentials: false

- name: Download approval thread timestamps
continue-on-error: true
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: release-slack-thread-*
path: ${{ runner.temp }}/release-slack-threads
github-token: ${{ github.token }}
run-id: ${{ github.event.workflow_run.id }}

- name: Check release jobs
id: release-status
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
SLACK_CHANNEL_ID: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }}
THREAD_DIRECTORY: ${{ runner.temp }}/release-slack-threads
with:
retries: 3
script: |
const notification = require('./scripts/release-failure-notification.cjs');
const payload = await notification({ github, context, channel: process.env.SLACK_CHANNEL_ID, threadDirectory: process.env.THREAD_DIRECTORY });
if (payload) core.setOutput('payload', JSON.stringify(payload));

- name: Notify Slack - Failed
if: steps.release-status.outputs.payload != ''
uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
with:
method: chat.postMessage
token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }}
errors: true
retries: 5
payload: ${{ steps.release-status.outputs.payload }}
38 changes: 20 additions & 18 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,26 @@ jobs:
slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }}
posthog_project_api_key: ${{ secrets.POSTHOG_PROJECT_API_KEY }}

preserve-slack-thread:
name: Preserve Slack thread
needs: notify-approval-needed
if: needs.notify-approval-needed.outputs.slack_ts != ''
runs-on: ubuntu-latest
timeout-minutes: 5
continue-on-error: true
steps:
- name: Save approval thread timestamp
env:
THREAD_TS: ${{ needs.notify-approval-needed.outputs.slack_ts }}
run: printf '%s' "$THREAD_TS" > thread-ts.txt

- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: release-slack-thread-${{ github.run_attempt }}
path: thread-ts.txt
retention-days: 30
if-no-files-found: error

release:
name: Publish PHP then Laravel
needs: [check-changesets, prepare-release-candidate, verify-release-candidate, notify-approval-needed]
Expand All @@ -249,10 +269,6 @@ jobs:
permissions:
actions: read
contents: read
outputs:
core-published: ${{ steps.publish-core.outcome == 'success' }}
distribution-published: ${{ steps.publish-distribution.outcome == 'success' }}
laravel-published: ${{ steps.publish-laravel.outcome == 'success' }}
steps:
- name: Notify Slack - Approved
if: needs.notify-approval-needed.outputs.slack_ts != ''
Expand Down Expand Up @@ -528,20 +544,6 @@ jobs:
"alertContext": "The release workflow failed before completion."
}

- name: Notify Slack - Failed
if: steps.check-failure.outputs.was_rejected != 'true'
uses: posthog/.github/.github/actions/slack-thread-reply@cb0979b67dcd585828b61ac45927eef4da8f6287 # main
with:
slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }}
slack_channel_id: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }}
thread_ts: ${{ needs.notify-approval-needed.outputs.slack_ts }}
message: |
❌ Release workflow failed or was cancelled.
${{ needs.prepare-release-candidate.outputs.core-changed == 'true' && format('posthog-php@{0}: {1}', needs.prepare-release-candidate.outputs.core-version, needs.release.outputs.core-published == 'true' && 'published' || 'not completed') || 'posthog-php: unchanged (no release planned)' }}
posthog-laravel@${{ needs.prepare-release-candidate.outputs.version }}: ${{ needs.release.outputs.laravel-published == 'true' && 'published' || needs.release.outputs.distribution-published == 'true' && 'mirror published; source release incomplete' || 'not completed' }}
<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>
emoji_reaction: "x"

- name: Notify Slack - Rejected
if: steps.check-failure.outputs.was_rejected == 'true'
continue-on-error: true
Expand Down
12 changes: 12 additions & 0 deletions RELEASING.md
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,18 @@ Ensure `.changeset/*.md` contains at least one `patch`, `minor`, or `major` inte

`scripts/bump-version.sh` is hash-pinned in `release.yml`. If it changes, recompute its SHA256 and update `EXPECTED_BUMP_SCRIPT_SHA256` in the same reviewed PR. Preparation rejects a mismatch before executing the script.

### Failed or cancelled release

The independent [release failure notification workflow](.github/workflows/release-failure-notification.yml) checks every completed main-branch release attempt and alerts if a job failed, timed out, or was cancelled. It also alerts when the run failed but the publishing job never ran, which happens when GitHub Actions stops mid-run and the in-run **Release Failed** job cannot run. It only acts on runs that a push or a manual dispatch started in this repository.

The in-run **Release Failed** job sends the PostHog failure event and replies in Slack only when the release was explicitly rejected. The independent workflow does not repeat that reply for a rejection. Every other failed or cancelled release is reported only by the independent workflow. If the packages are published, alerts describe follow-up problems, such as a skipped released notification, rather than claiming the release failed.

Cancelled runs with no jobs are ignored, including superseded queued runs and manual cancellations before any job is created.

Job inspection retries transient GitHub API failures up to three times. If inspection still fails, the workflow logs the HTTP status or error type and posts a fallback alert with the overall run status and logs link. Error diagnostics stay in the run log, not in the Slack message.

Alerts reply to the approval thread in the release approvals Slack channel. The release workflow preserves the thread timestamp in a run artifact for 30 days, independently of release approval. Failed-job reruns reuse the latest available thread from the same or an earlier attempt. If the timestamp is missing, invalid, or unavailable, the alert falls back to a new channel message. The workflow must be present on the default branch to run.

### Partial publication

One source version commit and publishing to two repositories cannot be transactional. A failure can leave committed versions without tags, core published but Laravel unpublished, or a distribution tag without its source release.
Expand Down
103 changes: 103 additions & 0 deletions scripts/release-failure-notification.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
const { readdirSync, readFileSync } = require('node:fs');
const { join } = require('node:path');

const unsuccessful = new Set(['failure', 'cancelled', 'timed_out', 'action_required', 'startup_failure', 'stale']);
const publishJobs = ['Publish PHP then Laravel'];
const failureNotificationJob = 'Notify Slack/PostHog - Release Failed';

const isJob = (job, name) => job.name === name || job.name.startsWith(`${name} (`);
const stepSucceeded = (job, name) => job?.steps?.some(step => step.name === name && step.conclusion === 'success');

function readThreadTimestamp(directory, attempt) {
if (!directory) return undefined;
try {
const entries = readdirSync(directory);
// A single matching artifact is extracted straight into the directory.
// Failed-job reruns reuse the approval thread from the latest earlier attempt.
const files = entries.includes('thread-ts.txt')
? [join(directory, 'thread-ts.txt')]
: entries
.map(name => /^release-slack-thread-(\d+)$/.exec(name)?.[1])
.filter(value => value && Number(value) <= attempt)
.sort((a, b) => Number(b) - Number(a))
.map(value => join(directory, `release-slack-thread-${value}`, 'thread-ts.txt'));
if (files.length === 0) return undefined;
const timestamp = readFileSync(files[0], 'utf8').trim();
return /^\d+\.\d+$/.test(timestamp) ? timestamp : undefined;
} catch (error) {
if (error.code !== 'ENOENT') console.error('Could not read Slack thread:', error.message);
return undefined;
}
}

module.exports = async function releaseFailureNotification({ github, context, channel, threadDirectory }) {
const run = context.payload.workflow_run;
const threadTs = readThreadTimestamp(threadDirectory, run.run_attempt);
const destination = { channel, ...(threadTs ? { thread_ts: threadTs } : {}) };
let jobs;
try {
jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, {
...context.repo,
run_id: run.id,
attempt_number: run.run_attempt,
per_page: 100,
});
} catch (error) {
console.error('Failed to inspect release jobs:', error.status ?? error.name, error.message);
return {
...destination,
text: [
`⚠️ Could not inspect release jobs for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`,
'Please check this release manually; individual job results are unavailable.',
`<${run.html_url}|View release logs> (run: ${run.conclusion})`,
].join('\n'),
unfurl_links: false,
unfurl_media: false,
};
}

if (jobs.length === 0 && run.conclusion === 'cancelled') {
return null;
}

const failedJobs = jobs.filter(job => unsuccessful.has(job.conclusion));
const failureNotification = jobs.find(job => job.name === failureNotificationJob);
const rejectionDelivered = stepSucceeded(failureNotification, 'Notify Slack - Rejected');
const alreadyReported = job => isJob(job, publishJobs[0]) && rejectionDelivered && job.steps?.length === 0;

const published = publishJobs.every(name => {
const cells = jobs.filter(job => isJob(job, name));
return cells.length > 0 && cells.every(job => job.conclusion === 'success');
});
const releasedNotification = jobs.find(job => job.name === 'Notify Slack - Released');
const problems = failedJobs
.filter(job => !alreadyReported(job))
.map(job => `• <${job.html_url}|${job.name}>: ${job.conclusion}`);

if (run.conclusion !== 'success' && failedJobs.length === 0) {
for (const name of publishJobs) {
if (!jobs.some(job => isJob(job, name))) problems.push(`• ${name}: never ran`);
}
}

if (published && (!releasedNotification || releasedNotification.conclusion === 'skipped')) {
problems.push('• The released Slack notification did not run.');
}

if (problems.length === 0 && (run.conclusion === 'success' || failedJobs.length > 0)) {
return null;
}

return {
...destination,
text: [
published
? `⚠️ PHP and Laravel packages published; follow-up problems for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`
: `❌ Release process failed or was cancelled for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`,
...problems,
`<${run.html_url}|View release logs> (run: ${run.conclusion})`,
].join('\n'),
unfurl_links: false,
unfurl_media: false,
};
};
Loading