chore(ci): fix flakey integration tests (#7958) #11076
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Integration Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - '**' | |
| - '!release-please--**' | |
| jobs: | |
| setup: | |
| name: Create test site | |
| runs-on: ubuntu-latest | |
| # Skip for fork PRs since they don't have access to secrets | |
| if: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == true) }} | |
| timeout-minutes: 5 | |
| outputs: | |
| site-id: ${{ steps.create-site.outputs.site-id }} | |
| site-name: ${{ steps.create-site.outputs.site-name }} | |
| steps: | |
| - name: Create site | |
| id: create-site | |
| run: | | |
| SITE_NAME="netlify-test-deploy-$(openssl rand -hex 4)" | |
| echo "Creating test site: $SITE_NAME" | |
| HTTP_CODE=$(curl -s -o response.json -w '%{http_code}' -X POST \ | |
| -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \ | |
| -H "Content-Type: application/json" \ | |
| -d "{\"name\": \"$SITE_NAME\"}" \ | |
| "https://api.netlify.com/api/v1/netlify-integration-testing/sites") | |
| if [ "$HTTP_CODE" -ne 201 ]; then | |
| echo "::warning::Failed to create site. HTTP $HTTP_CODE. Live tests will be skipped." | |
| exit 0 | |
| fi | |
| SITE_ID=$(jq -r '.id' response.json) | |
| echo "site-id=$SITE_ID" >> "$GITHUB_OUTPUT" | |
| echo "site-name=$SITE_NAME" >> "$GITHUB_OUTPUT" | |
| echo "Created site $SITE_NAME with ID $SITE_ID" | |
| env: | |
| NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} | |
| integration: | |
| name: Integration | |
| needs: [setup] | |
| # Run even if setup was skipped (fork PRs) or failed to create a site. | |
| # Tests that need a live site will be skipped if NETLIFY_LIVE_TEST_SITE_ID is not set. | |
| if: ${{ !cancelled() }} | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 40 | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macOS-latest, windows-2025] | |
| # Pinning 20.x version as a temporary workaround due to this https://github.com/nodejs/node/issues/52884 | |
| node-version: ['20.12.2', '22', '24'] | |
| shard: ['1/4', '2/4', '3/4', '4/4'] | |
| fail-fast: false | |
| steps: | |
| # This improves Windows network performance, we need this since we open many ports in our tests | |
| - name: Increase Windows port limit and reduce time wait delay | |
| run: | | |
| netsh int ipv4 set dynamicport tcp start=1025 num=64511 | |
| REG ADD HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\TCPIP\Parameters /v TcpTimedWaitDelay /t REG_DWORD /d 30 /f | |
| if: "${{ matrix.os == 'windows-2025' }}" | |
| - name: Git checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| cache: npm | |
| check-latest: true | |
| - name: Install PNPM | |
| run: | | |
| corepack enable | |
| corepack prepare pnpm@9.14.2 --activate | |
| - name: Setup Deno | |
| uses: denoland/setup-deno@v1 | |
| with: | |
| deno-version: 2.6.6 | |
| - name: Install core dependencies | |
| run: npm ci --no-audit | |
| - name: Build project | |
| run: npm run build | |
| if: '${{!steps.release-check.outputs.IS_RELEASE}}' | |
| - name: Prepare tests | |
| run: npm run test:init | |
| - name: Tests | |
| run: npm run test:integration -- --coverage --shard=${{ matrix.shard }} | |
| env: | |
| # GitHub secrets are not available when running on PR from forks | |
| # We set a flag so we can skip tests that access Netlify API | |
| NETLIFY_TEST_DISABLE_LIVE: | |
| ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == true }} | |
| NETLIFY_TEST_ACCOUNT_SLUG: 'netlify-integration-testing' | |
| NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} | |
| # Site created by the setup job, shared across all matrix jobs | |
| NETLIFY_LIVE_TEST_SITE_ID: ${{ needs.setup.outputs.site-id }} | |
| NETLIFY_LIVE_TEST_SITE_NAME: ${{ needs.setup.outputs.site-name }} | |
| # NETLIFY_TEST_GITHUB_TOKEN is used to avoid reaching GitHub API limits in exec-fetcher.js | |
| NETLIFY_TEST_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # Changes the polling interval used by the file watcher | |
| CHOKIDAR_INTERVAL: 20 | |
| CHOKIDAR_USEPOLLING: 1 | |
| - name: Get test coverage flags | |
| id: test-coverage-flags | |
| # For windows we have to use $env: | |
| run: |- | |
| os=${{ matrix.os }} | |
| node=$(node --version) | |
| echo "os=${os/-latest/}" >> $GITHUB_OUTPUT | |
| echo "os=${os/-latest/}" >> $env:GITHUB_OUTPUT | |
| echo "node=node_${node/.*.*/}" >> $GITHUB_OUTPUT | |
| echo "node=node_${node/.*.*/}" >> $env:GITHUB_OUTPUT | |
| shell: bash | |
| - name: Sanitize shard for artefact name | |
| id: sanitize-shard-name | |
| run: echo "shard=$(echo '${{ matrix.shard }}' | tr '/' '-')" >> $GITHUB_OUTPUT | |
| - name: Store npm error artefacts | |
| uses: actions/upload-artifact@v6 | |
| if: always() | |
| with: | |
| name: npm-logs--${{ matrix.os }}--${{ matrix.node-version }}--${{ steps.sanitize-shard-name.outputs.shard }} | |
| path: | | |
| ~/.npm/_logs/**/* | |
| - uses: codecov/codecov-action@v5 | |
| continue-on-error: true | |
| with: | |
| flags: ${{ steps.test-coverage-flags.outputs.os }},${{ steps.test-coverage-flags.outputs.node }} | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| cleanup: | |
| name: Delete test site | |
| needs: [setup, integration] | |
| if: ${{ always() && needs.setup.outputs.site-id }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Delete site | |
| run: | | |
| echo "Deleting test site ${{ needs.setup.outputs.site-id }}" | |
| curl -sf -X DELETE \ | |
| -H "Authorization: Bearer $NETLIFY_AUTH_TOKEN" \ | |
| "https://api.netlify.com/api/v1/sites/${{ needs.setup.outputs.site-id }}" | |
| echo "Deleted successfully" | |
| env: | |
| NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} |