diff --git a/.github/workflows/flaky-test-detector.agent.lock.yml b/.github/workflows/flaky-test-detector.agent.lock.yml index 6be62fb3afd..1d8408f9898 100644 --- a/.github/workflows/flaky-test-detector.agent.lock.yml +++ b/.github/workflows/flaky-test-detector.agent.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"1da19f5978f14a592b1ad44c966e4110a859a1d597a5cc0ba4a669a514221bf6","body_hash":"1a34cf702b3f0d038eb2e39017f4a8f9590de6bc5717b70fdc3995ba522bbd46","compiler_version":"v0.77.5","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"75fd9fc8b4a80e79c33a746add269bb3bc10593e649423a4e3d4dd600998d529","body_hash":"4d22b0aa2607c8fa3ca253059e11fb06b30c5c3e6a4163a90f4ecd7e3cb5d47d","compiler_version":"v0.77.5","strict":true,"agent_id":"copilot"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"3ea13c02d765410340d533515cb31a7eef2baaf0","version":"v0.77.5"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.58"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.22"},{"image":"ghcr.io/github/github-mcp-server:v1.1.0"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} # ___ _ _ # / _ \ | | (_) @@ -22,7 +22,7 @@ # # For more information: https://github.github.com/gh-aw/introduction/overview/ # -# Scheduled daily workflow that scans recent msbuild CI builds (approved PRs + rolling main builds) for tests that fail across multiple independent sources, files/updates flaky-test tracking issues, then quarantines the new candidates with [ActiveIssue]. It also scans the quarantine pipeline (definition 344) to un-quarantine tests that have gone consistently green, opening ONE combined draft PR per run. +# Scheduled daily workflow that scans recent msbuild CI builds (approved PRs + rolling main builds) for tests that fail across multiple independent sources, files/updates flaky-test tracking issues, then quarantines the new candidates with [ActiveIssue]. It also scans the quarantine pipeline (definition 344) to un-quarantine tests that have gone consistently green, opening ONE combined ready-for-review PR per run. # # Secrets used: # - COPILOT_GITHUB_TOKEN @@ -190,23 +190,23 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_8955ab952b88b36a_EOF' + cat << 'GH_AW_PROMPT_2c0a03a17d953291_EOF' - GH_AW_PROMPT_8955ab952b88b36a_EOF + GH_AW_PROMPT_2c0a03a17d953291_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_8955ab952b88b36a_EOF' + cat << 'GH_AW_PROMPT_2c0a03a17d953291_EOF' Tools: add_comment(max:12), create_issue(max:5), create_pull_request, missing_tool, missing_data, noop - GH_AW_PROMPT_8955ab952b88b36a_EOF + GH_AW_PROMPT_2c0a03a17d953291_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_create_pull_request.md" - cat << 'GH_AW_PROMPT_8955ab952b88b36a_EOF' + cat << 'GH_AW_PROMPT_2c0a03a17d953291_EOF' - GH_AW_PROMPT_8955ab952b88b36a_EOF + GH_AW_PROMPT_2c0a03a17d953291_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" - cat << 'GH_AW_PROMPT_8955ab952b88b36a_EOF' + cat << 'GH_AW_PROMPT_2c0a03a17d953291_EOF' The following GitHub context information is available for this workflow: {{#if github.actor}} @@ -235,12 +235,12 @@ jobs: {{/if}} - GH_AW_PROMPT_8955ab952b88b36a_EOF + GH_AW_PROMPT_2c0a03a17d953291_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/cli_proxy_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_8955ab952b88b36a_EOF' + cat << 'GH_AW_PROMPT_2c0a03a17d953291_EOF' {{#runtime-import .github/workflows/flaky-test-detector.agent.md}} - GH_AW_PROMPT_8955ab952b88b36a_EOF + GH_AW_PROMPT_2c0a03a17d953291_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -452,9 +452,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_365cea5788eb5f1d_EOF' - {"add_comment":{"max":12,"target":"*"},"create_issue":{"labels":["flaky-test"],"max":5,"title_prefix":"[Flaky Test] "},"create_pull_request":{"allowed_files":["src/**/*.cs"],"base_branch":"main","draft":true,"excluded_files":[".github/**"],"labels":["flaky-test"],"max":1,"max_patch_files":100,"max_patch_size":1024,"protect_top_level_dot_folders":true,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS","DESIGN.md","README.md","CONTRIBUTING.md","CHANGELOG.md","SECURITY.md","CODE_OF_CONDUCT.md","AGENTS.md","CLAUDE.md","GEMINI.md"],"protected_files_policy":"request_review","title_prefix":"[Flaky Test] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_365cea5788eb5f1d_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_26ea2747868b2d46_EOF' + {"add_comment":{"max":12,"target":"*"},"create_issue":{"labels":["flaky-test"],"max":5,"title_prefix":"[Flaky Test] "},"create_pull_request":{"allowed_files":["src/**/*.cs"],"base_branch":"main","draft":false,"excluded_files":[".github/**"],"labels":["flaky-test"],"max":1,"max_patch_files":100,"max_patch_size":1024,"protect_top_level_dot_folders":true,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS","DESIGN.md","README.md","CONTRIBUTING.md","CHANGELOG.md","SECURITY.md","CODE_OF_CONDUCT.md","AGENTS.md","CLAUDE.md","GEMINI.md"],"protected_files_policy":"request_review","title_prefix":"[Flaky Test] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"},"report_incomplete":{}} + GH_AW_SAFE_OUTPUTS_CONFIG_26ea2747868b2d46_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -462,7 +462,7 @@ jobs: "description_suffixes": { "add_comment": " CONSTRAINTS: Maximum 12 comment(s) can be added. Target: *. Supports reply_to_id for discussion threading.", "create_issue": " CONSTRAINTS: Maximum 5 issue(s) can be created. Title will be prefixed with \"[Flaky Test] \". Labels [\"flaky-test\"] will be automatically added.", - "create_pull_request": " CONSTRAINTS: Maximum 1 pull request(s) can be created. Title will be prefixed with \"[Flaky Test] \". Labels [\"flaky-test\"] will be automatically added. PRs will be created as drafts." + "create_pull_request": " CONSTRAINTS: Maximum 1 pull request(s) can be created. Title will be prefixed with \"[Flaky Test] \". Labels [\"flaky-test\"] will be automatically added." }, "repo_params": {}, "dynamic_tools": [] @@ -724,7 +724,7 @@ jobs: mkdir -p /home/runner/.copilot GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_d27f8877643775d3_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_a074c41de8fb9e5e_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "safeoutputs": { @@ -749,7 +749,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_d27f8877643775d3_EOF + GH_AW_MCP_CONFIG_a074c41de8fb9e5e_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true @@ -1254,7 +1254,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: WORKFLOW_NAME: "Flaky Test Triage" - WORKFLOW_DESCRIPTION: "Scheduled daily workflow that scans recent msbuild CI builds (approved PRs + rolling main builds) for tests that fail across multiple independent sources, files/updates flaky-test tracking issues, then quarantines the new candidates with [ActiveIssue]. It also scans the quarantine pipeline (definition 344) to un-quarantine tests that have gone consistently green, opening ONE combined draft PR per run." + WORKFLOW_DESCRIPTION: "Scheduled daily workflow that scans recent msbuild CI builds (approved PRs + rolling main builds) for tests that fail across multiple independent sources, files/updates flaky-test tracking issues, then quarantines the new candidates with [ActiveIssue]. It also scans the quarantine pipeline (definition 344) to un-quarantine tests that have gone consistently green, opening ONE combined ready-for-review PR per run." HAS_PATCH: ${{ needs.agent.outputs.has_patch }} with: script: | @@ -1504,7 +1504,7 @@ jobs: GH_AW_ALLOWED_DOMAINS: "*.vsblob.vsassets.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.nuget.org,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,azuresearch-usnc.nuget.org,azuresearch-ussc.nuget.org,builds.dotnet.microsoft.com,ci.dot.net,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dc.services.visualstudio.com,dev.azure.com,dist.nuget.org,dnceng.pkgs.visualstudio.com,dot.net,dotnet.microsoft.com,dotnetcli.blob.core.windows.net,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,nuget.org,nuget.pkg.github.com,nugetregistryv2prod.blob.core.windows.net,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,oneocsp.microsoft.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.dev.azure.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com,www.microsoft.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":12,\"target\":\"*\"},\"create_issue\":{\"labels\":[\"flaky-test\"],\"max\":5,\"title_prefix\":\"[Flaky Test] \"},\"create_pull_request\":{\"allowed_files\":[\"src/**/*.cs\"],\"base_branch\":\"main\",\"draft\":true,\"excluded_files\":[\".github/**\"],\"labels\":[\"flaky-test\"],\"max\":1,\"max_patch_files\":100,\"max_patch_size\":1024,\"protect_top_level_dot_folders\":true,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"DESIGN.md\",\"README.md\",\"CONTRIBUTING.md\",\"CHANGELOG.md\",\"SECURITY.md\",\"CODE_OF_CONDUCT.md\",\"AGENTS.md\",\"CLAUDE.md\",\"GEMINI.md\"],\"protected_files_policy\":\"request_review\",\"title_prefix\":\"[Flaky Test] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}" + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":12,\"target\":\"*\"},\"create_issue\":{\"labels\":[\"flaky-test\"],\"max\":5,\"title_prefix\":\"[Flaky Test] \"},\"create_pull_request\":{\"allowed_files\":[\"src/**/*.cs\"],\"base_branch\":\"main\",\"draft\":false,\"excluded_files\":[\".github/**\"],\"labels\":[\"flaky-test\"],\"max\":1,\"max_patch_files\":100,\"max_patch_size\":1024,\"protect_top_level_dot_folders\":true,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"DESIGN.md\",\"README.md\",\"CONTRIBUTING.md\",\"CHANGELOG.md\",\"SECURITY.md\",\"CODE_OF_CONDUCT.md\",\"AGENTS.md\",\"CLAUDE.md\",\"GEMINI.md\"],\"protected_files_policy\":\"request_review\",\"title_prefix\":\"[Flaky Test] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}" GH_AW_CI_TRIGGER_TOKEN: ${{ secrets.GH_AW_CI_TRIGGER_TOKEN }} with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/flaky-test-detector.agent.md b/.github/workflows/flaky-test-detector.agent.md index 1fd63f71018..c963acf8386 100644 --- a/.github/workflows/flaky-test-detector.agent.md +++ b/.github/workflows/flaky-test-detector.agent.md @@ -1,6 +1,6 @@ --- name: "Flaky Test Triage" -description: "Scheduled daily workflow that scans recent msbuild CI builds (approved PRs + rolling main builds) for tests that fail across multiple independent sources, files/updates flaky-test tracking issues, then quarantines the new candidates with [ActiveIssue]. It also scans the quarantine pipeline (definition 344) to un-quarantine tests that have gone consistently green, opening ONE combined draft PR per run." +description: "Scheduled daily workflow that scans recent msbuild CI builds (approved PRs + rolling main builds) for tests that fail across multiple independent sources, files/updates flaky-test tracking issues, then quarantines the new candidates with [ActiveIssue]. It also scans the quarantine pipeline (definition 344) to un-quarantine tests that have gone consistently green, opening ONE combined ready-for-review PR per run." on: schedule: daily workflow_dispatch: # Allow manual triggering @@ -56,7 +56,7 @@ safe-outputs: create-pull-request: title-prefix: "[Flaky Test] " labels: [flaky-test] - draft: true + draft: false base-branch: main max: 1 # Exclusive allowlist: this workflow only ever edits test sources under src/ (adding or removing an @@ -80,7 +80,7 @@ timeout-minutes: 60 You are an automated maintenance agent for the **dotnet/msbuild** repository. Your job is to find **flaky tests** — tests that fail intermittently rather than because of a real product regression — -track them as GitHub issues, and, in a **single combined draft pull request per run**, **quarantine** +track them as GitHub issues, and, in a **single combined ready-for-review pull request per run**, **quarantine** them so CI stops being disrupted — and **un-quarantine** tests the quarantine pipeline has proven green again. This workflow does **not** reproduce flakes or author any code fixes: proposing a determinism fix is the job of the **separate** auto-fixer workflow (`flaky-test-fixer.agent.md`), which @@ -122,7 +122,7 @@ repo-wide text search, then locate the class/method within it. 5. For each selected test apply a **quarantine** or an **un-quarantine**, accumulating **all** `.cs` edits in the working tree (Step 6). 6. Build the whole repo **once** to validate the edits compile (Step 7), then open **exactly one** - combined draft PR (Step 8). + combined PR (Step 8). ## Step 1 — Run the detector @@ -247,7 +247,8 @@ benign `Malformed version:` warning to stderr; it is harmless — judge success and the exit code, not by that line.) - **If a related issue is OPEN:** post an `add_comment` to that issue number with the **new** evidence - (latest sources, build URLs, dates, legs/TFMs). Do not open a duplicate. + (latest sources, build URLs, dates, legs/TFMs), rendering rolling build ids as markdown links to their + AzDO build results pages as described below. Do not open a duplicate. - **If a related issue exists but is CLOSED recently** (e.g. within ~30 days): do **not** open a duplicate. **First confirm the flake actually recurred *after* the issue was resolved** — the detector's look-back window (`-DaysBack`) routinely includes builds from *before* a fix landed, so @@ -281,7 +282,11 @@ and the exit code, not by that line.) using the normalized `testName` exactly. - Then include an evidence summary: distinct sources (PRs + rolling builds), PR numbers, rolling build ids, affected legs/TFMs, assemblies, first/last seen, a representative error message, and - links to `sampleBuildUrl`. Provide the fully-qualified test name and the assembly so the separate + links to `sampleBuildUrl`. **Render every rolling build id as a markdown link** to its AzDO build + results page rather than as plain text — e.g. `[1430301](https://dev.azure.com/dnceng-public/public/_build/results?buildId=1430301)`. + Build the URL by taking the `sampleBuildUrl` form and substituting each build id into its + `buildId=` query parameter, so the org/project path always matches the detector's own data. + Provide the fully-qualified test name and the assembly so the separate auto-fixer workflow (or a human) can locate it. - Add brief guidance: the test will be **quarantined** via `[ActiveIssue("")]` (from `Microsoft.DotNet.XUnitV3Extensions`, namespace `Xunit`) — not `[Fact(Skip=...)]` — until the @@ -433,15 +438,15 @@ The whole-repo build takes ~2-3 minutes — **never cancel it**. Interpret the r - **Build fails for environmental/network reasons** — NuGet restore cannot reach a feed, a blocked domain, or an SDK-download failure, *not* a compile error: **do not retry the build and do not loop.** Re-running against a blocked feed burns the entire token budget on NuGet retries and huge logs. The - edits are mechanical attribute add/removes and are low-risk, so **open the PR anyway** — it is a draft, - so its own CI is the first real compile of these edits — and note in the PR body that the local + edits are mechanical attribute add/removes and are low-risk, so **open the PR anyway** — it opens + ready-for-review, so its own CI is the first real compile of these edits — and note in the PR body that the local validation build was blocked by the environment. One failed build attempt is enough to decide this. Only treat a failure as environmental when it is clearly about restore/feed/SDK access. If the log contains C# compiler errors (`error CS...`), malformed-attribute syntax, or test-project compile errors, it is **not** environmental — fix or drop the offending edit per the second bullet. -## Step 8 — Validate the diff and open ONE combined draft PR +## Step 8 — Validate the diff and open ONE combined PR First confirm the diff is **limited and correct**: @@ -453,8 +458,8 @@ First confirm the diff is **limited and correct**: command) and exact-string-match each remaining test's marker against them again — a concurrent run may have opened a combined PR since Step 5. Revert and drop any test now covered by an open PR. -If, after this, no edits remain, open **no** PR and emit a `noop`. Otherwise open **exactly one** draft -PR (`create_pull_request`, base `main`, label `flaky-test`) containing all accumulated edits: +If, after this, no edits remain, open **no** PR and emit a `noop`. Otherwise open **exactly one** +ready-for-review PR (`create_pull_request`, base `main`, label `flaky-test`) containing all accumulated edits: - Title: e.g. `Quarantine/un-quarantine flaky tests` (the `[Flaky Test] ` prefix is added automatically); summarize the mix of actions. @@ -476,6 +481,11 @@ PR (`create_pull_request`, base `main`, label `flaky-test`) containing all accum instead and do **not** write a closing keyword. For a **narrowing**, always `Tracked by #`. - If the local validation build (Step 7) was blocked by the environment, say so in the PR body so a reviewer knows CI is the first real compile of these edits. +- **Do NOT add a "new flaky test issues filed this run" (or similar) section listing the tracking issues + you filed via `create_issue`.** Those newly-filed issues are **not acted on by this PR** (they become + quarantine-eligible only on a future run), so referencing their `#` here creates a misleading + issue↔PR cross-link. The PR body must reference **only** the issues for tests it actually quarantines or + un-quarantines this run. Newly-filed issues stand on their own. - Post one `add_comment` on each included test's tracking issue summarizing the action and linking the PR. ## Important @@ -487,7 +497,7 @@ PR (`create_pull_request`, base `main`, label `flaky-test`) containing all accum - **Un-quarantines have their own cap** (Step 5b: **at most 5** per run) and still fold into the **same single** combined PR. If the `add-comment` budget (12) is tight, prioritize comments for new quarantines over un-quarantine confirmations. -- **Open at most ONE pull request**, and it must be a **draft** based on `main`. +- **Open at most ONE pull request**, and it must be a **non-draft (ready-for-review)** PR based on `main`. - **Never modify anything under `.github/**`** (no workflow, skill, or action edits) and never touch root manifests (`NuGet.config`, `global.json`, `Directory.Packages.props`, etc.). This workflow only ever edits **test sources** under `src/` — adding or removing an `[ActiveIssue]` attribute — and **never** diff --git a/.github/workflows/flaky-test-fixer.agent.lock.yml b/.github/workflows/flaky-test-fixer.agent.lock.yml index 421465d305c..ff02a1f4cc9 100644 --- a/.github/workflows/flaky-test-fixer.agent.lock.yml +++ b/.github/workflows/flaky-test-fixer.agent.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"fffb06f2bdec706b21e4f744b09f2edddc4bb3a1c768e77090bd7c4fffbcdf2c","body_hash":"88261b4a827b1a70e8ec343fb5033f05e604845b35165b62dd3cec046f9631eb","compiler_version":"v0.77.5","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"e913c0c24ac7a5554d719ea026aed34d0eda18fa4becc8b8ec81a8f796a4fb40","body_hash":"f8e247e10fa0e65d0ee3398bd8e04b2f10d621a4f37c9b09799d6525d2108146","compiler_version":"v0.77.5","strict":true,"agent_id":"copilot"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"3ea13c02d765410340d533515cb31a7eef2baaf0","version":"v0.77.5"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.58"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.22"},{"image":"ghcr.io/github/github-mcp-server:v1.1.0"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} # ___ _ _ # / _ \ | | (_) @@ -22,7 +22,7 @@ # # For more information: https://github.github.com/gh-aw/introduction/overview/ # -# Scheduled daily workflow that proposes evidence-based fixes for tests that are ALREADY quarantined ([ActiveIssue]) but STILL FLAKING in the quarantine pipeline (definition 344). It mines the accumulated over-time failure evidence (consistent error signatures + stack traces), diagnoses a minimal TEST-ONLY root cause without any local reproduction, and opens one individual draft PR per confidently-fixable test. By default it KEEPS the [ActiveIssue] in place so the quarantine pipeline validates the fix over the following days and the separate detector workflow un-quarantines once green. When confidence is VERY high (a fully-explained deterministic root cause with a complete fix), it ALSO removes the [ActiveIssue] in the same PR so normal PR CI runs the test as additional pre-merge validation, and says so in the PR body. +# Scheduled daily workflow that proposes evidence-based fixes for tests that are ALREADY quarantined ([ActiveIssue]) but STILL FLAKING in the quarantine pipeline (definition 344). It mines the accumulated over-time failure evidence (consistent error signatures + stack traces), diagnoses a minimal TEST-ONLY root cause without any local reproduction, and opens one individual ready-for-review PR per confidently-fixable test. By default it KEEPS the [ActiveIssue] in place so the quarantine pipeline validates the fix over the following days and the separate detector workflow un-quarantines once green. When confidence is VERY high (a fully-explained deterministic root cause with a complete fix), it ALSO removes the [ActiveIssue] in the same PR so normal PR CI runs the test as additional pre-merge validation, and says so in the PR body. # # Secrets used: # - COPILOT_GITHUB_TOKEN @@ -189,23 +189,23 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_f638f0faaf0b761c_EOF' + cat << 'GH_AW_PROMPT_6dd0608569adbcce_EOF' - GH_AW_PROMPT_f638f0faaf0b761c_EOF + GH_AW_PROMPT_6dd0608569adbcce_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_f638f0faaf0b761c_EOF' + cat << 'GH_AW_PROMPT_6dd0608569adbcce_EOF' Tools: add_comment(max:3), create_pull_request(max:3), missing_tool, missing_data, noop - GH_AW_PROMPT_f638f0faaf0b761c_EOF + GH_AW_PROMPT_6dd0608569adbcce_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_create_pull_request.md" - cat << 'GH_AW_PROMPT_f638f0faaf0b761c_EOF' + cat << 'GH_AW_PROMPT_6dd0608569adbcce_EOF' - GH_AW_PROMPT_f638f0faaf0b761c_EOF + GH_AW_PROMPT_6dd0608569adbcce_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" - cat << 'GH_AW_PROMPT_f638f0faaf0b761c_EOF' + cat << 'GH_AW_PROMPT_6dd0608569adbcce_EOF' The following GitHub context information is available for this workflow: {{#if github.actor}} @@ -234,12 +234,12 @@ jobs: {{/if}} - GH_AW_PROMPT_f638f0faaf0b761c_EOF + GH_AW_PROMPT_6dd0608569adbcce_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/cli_proxy_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_f638f0faaf0b761c_EOF' + cat << 'GH_AW_PROMPT_6dd0608569adbcce_EOF' {{#runtime-import .github/workflows/flaky-test-fixer.agent.md}} - GH_AW_PROMPT_f638f0faaf0b761c_EOF + GH_AW_PROMPT_6dd0608569adbcce_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -451,16 +451,16 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_a2a5413cddc2f293_EOF' - {"add_comment":{"max":3,"target":"*"},"create_pull_request":{"allowed_files":["src/**/*UnitTests*/**/*.cs","src/**/*.Tests/**/*.cs"],"auto_close_issue":false,"base_branch":"main","draft":true,"excluded_files":[".github/**"],"labels":["flaky-test"],"max":3,"max_patch_files":100,"max_patch_size":1024,"protect_top_level_dot_folders":true,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS","DESIGN.md","README.md","CONTRIBUTING.md","CHANGELOG.md","SECURITY.md","CODE_OF_CONDUCT.md","AGENTS.md","CLAUDE.md","GEMINI.md"],"protected_files_policy":"request_review","title_prefix":"[Flaky Test Fix] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_a2a5413cddc2f293_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_c4eea778e10b312d_EOF' + {"add_comment":{"max":3,"target":"*"},"create_pull_request":{"auto_close_issue":false,"base_branch":"main","draft":false,"excluded_files":[".github/**"],"labels":["flaky-test"],"max":3,"max_patch_files":100,"max_patch_size":1024,"protect_top_level_dot_folders":true,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS","DESIGN.md","README.md","CONTRIBUTING.md","CHANGELOG.md","SECURITY.md","CODE_OF_CONDUCT.md","AGENTS.md","CLAUDE.md","GEMINI.md"],"protected_files_policy":"request_review","title_prefix":"[Flaky Test Fix] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"},"report_incomplete":{}} + GH_AW_SAFE_OUTPUTS_CONFIG_c4eea778e10b312d_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | { "description_suffixes": { "add_comment": " CONSTRAINTS: Maximum 3 comment(s) can be added. Target: *. Supports reply_to_id for discussion threading.", - "create_pull_request": " CONSTRAINTS: Maximum 3 pull request(s) can be created. Title will be prefixed with \"[Flaky Test Fix] \". Labels [\"flaky-test\"] will be automatically added. PRs will be created as drafts." + "create_pull_request": " CONSTRAINTS: Maximum 3 pull request(s) can be created. Title will be prefixed with \"[Flaky Test Fix] \". Labels [\"flaky-test\"] will be automatically added." }, "repo_params": {}, "dynamic_tools": [] @@ -686,7 +686,7 @@ jobs: mkdir -p /home/runner/.copilot GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_66841c7dd770b1a4_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_3cb2a8c298efec1c_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "safeoutputs": { @@ -711,7 +711,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_66841c7dd770b1a4_EOF + GH_AW_MCP_CONFIG_3cb2a8c298efec1c_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true @@ -1216,7 +1216,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: WORKFLOW_NAME: "Flaky Test Auto-Fixer" - WORKFLOW_DESCRIPTION: "Scheduled daily workflow that proposes evidence-based fixes for tests that are ALREADY quarantined ([ActiveIssue]) but STILL FLAKING in the quarantine pipeline (definition 344). It mines the accumulated over-time failure evidence (consistent error signatures + stack traces), diagnoses a minimal TEST-ONLY root cause without any local reproduction, and opens one individual draft PR per confidently-fixable test. By default it KEEPS the [ActiveIssue] in place so the quarantine pipeline validates the fix over the following days and the separate detector workflow un-quarantines once green. When confidence is VERY high (a fully-explained deterministic root cause with a complete fix), it ALSO removes the [ActiveIssue] in the same PR so normal PR CI runs the test as additional pre-merge validation, and says so in the PR body." + WORKFLOW_DESCRIPTION: "Scheduled daily workflow that proposes evidence-based fixes for tests that are ALREADY quarantined ([ActiveIssue]) but STILL FLAKING in the quarantine pipeline (definition 344). It mines the accumulated over-time failure evidence (consistent error signatures + stack traces), diagnoses a minimal TEST-ONLY root cause without any local reproduction, and opens one individual ready-for-review PR per confidently-fixable test. By default it KEEPS the [ActiveIssue] in place so the quarantine pipeline validates the fix over the following days and the separate detector workflow un-quarantines once green. When confidence is VERY high (a fully-explained deterministic root cause with a complete fix), it ALSO removes the [ActiveIssue] in the same PR so normal PR CI runs the test as additional pre-merge validation, and says so in the PR body." HAS_PATCH: ${{ needs.agent.outputs.has_patch }} with: script: | @@ -1464,7 +1464,7 @@ jobs: GH_AW_ALLOWED_DOMAINS: "*.vsblob.vsassets.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.nuget.org,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,azuresearch-usnc.nuget.org,azuresearch-ussc.nuget.org,builds.dotnet.microsoft.com,ci.dot.net,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dc.services.visualstudio.com,dev.azure.com,dist.nuget.org,dnceng.pkgs.visualstudio.com,dot.net,dotnet.microsoft.com,dotnetcli.blob.core.windows.net,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,nuget.org,nuget.pkg.github.com,nugetregistryv2prod.blob.core.windows.net,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,oneocsp.microsoft.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.dev.azure.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com,www.microsoft.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":3,\"target\":\"*\"},\"create_pull_request\":{\"allowed_files\":[\"src/**/*UnitTests*/**/*.cs\",\"src/**/*.Tests/**/*.cs\"],\"auto_close_issue\":false,\"base_branch\":\"main\",\"draft\":true,\"excluded_files\":[\".github/**\"],\"labels\":[\"flaky-test\"],\"max\":3,\"max_patch_files\":100,\"max_patch_size\":1024,\"protect_top_level_dot_folders\":true,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"DESIGN.md\",\"README.md\",\"CONTRIBUTING.md\",\"CHANGELOG.md\",\"SECURITY.md\",\"CODE_OF_CONDUCT.md\",\"AGENTS.md\",\"CLAUDE.md\",\"GEMINI.md\"],\"protected_files_policy\":\"request_review\",\"title_prefix\":\"[Flaky Test Fix] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}" + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":3,\"target\":\"*\"},\"create_pull_request\":{\"auto_close_issue\":false,\"base_branch\":\"main\",\"draft\":false,\"excluded_files\":[\".github/**\"],\"labels\":[\"flaky-test\"],\"max\":3,\"max_patch_files\":100,\"max_patch_size\":1024,\"protect_top_level_dot_folders\":true,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"DESIGN.md\",\"README.md\",\"CONTRIBUTING.md\",\"CHANGELOG.md\",\"SECURITY.md\",\"CODE_OF_CONDUCT.md\",\"AGENTS.md\",\"CLAUDE.md\",\"GEMINI.md\"],\"protected_files_policy\":\"request_review\",\"title_prefix\":\"[Flaky Test Fix] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}" GH_AW_CI_TRIGGER_TOKEN: ${{ secrets.GH_AW_CI_TRIGGER_TOKEN }} with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/flaky-test-fixer.agent.md b/.github/workflows/flaky-test-fixer.agent.md index 4371e892da2..3272cafb7f5 100644 --- a/.github/workflows/flaky-test-fixer.agent.md +++ b/.github/workflows/flaky-test-fixer.agent.md @@ -1,6 +1,6 @@ --- name: "Flaky Test Auto-Fixer" -description: "Scheduled daily workflow that proposes evidence-based fixes for tests that are ALREADY quarantined ([ActiveIssue]) but STILL FLAKING in the quarantine pipeline (definition 344). It mines the accumulated over-time failure evidence (consistent error signatures + stack traces), diagnoses a minimal TEST-ONLY root cause without any local reproduction, and opens one individual draft PR per confidently-fixable test. By default it KEEPS the [ActiveIssue] in place so the quarantine pipeline validates the fix over the following days and the separate detector workflow un-quarantines once green. When confidence is VERY high (a fully-explained deterministic root cause with a complete fix), it ALSO removes the [ActiveIssue] in the same PR so normal PR CI runs the test as additional pre-merge validation, and says so in the PR body." +description: "Scheduled daily workflow that proposes evidence-based fixes for tests that are ALREADY quarantined ([ActiveIssue]) but STILL FLAKING in the quarantine pipeline (definition 344). It mines the accumulated over-time failure evidence (consistent error signatures + stack traces), diagnoses a minimal TEST-ONLY root cause without any local reproduction, and opens one individual ready-for-review PR per confidently-fixable test. By default it KEEPS the [ActiveIssue] in place so the quarantine pipeline validates the fix over the following days and the separate detector workflow un-quarantines once green. When confidence is VERY high (a fully-explained deterministic root cause with a complete fix), it ALSO removes the [ActiveIssue] in the same PR so normal PR CI runs the test as additional pre-merge validation, and says so in the PR body." on: # Pinned ~1 hour after the detector (which runs at 11:38 UTC) so the fixer sees the detector's # latest quarantine/un-quarantine state before it proposes fixes. Explicit cron (not `daily`) so @@ -55,21 +55,20 @@ safe-outputs: create-pull-request: title-prefix: "[Flaky Test Fix] " labels: [flaky-test] - draft: true + draft: false base-branch: main - # One INDEPENDENT draft PR per fixed test (not a combined PR). `max` above 1 lets the agent emit + # One INDEPENDENT ready-for-review PR per fixed test (not a combined PR). `max` above 1 lets the agent emit # several create_pull_request outputs in one run, each captured as its own branch/bundle. max: 3 # Do not append a closing `Fixes #N` to the PR — the quarantine is intentionally retained and the # tracking issue must stay open until the detector un-quarantines after def 344 proves it green. auto-close-issue: false - # Exclusive allowlist: this workflow only ever edits TEST sources (all msbuild xUnit test projects - # live under `*UnitTests*` / `*.Tests` directories). Product code, root manifests, and `.github/**` - # are all refused outright — a far stronger guard than `src/**/*.cs`, which would also permit - # product code. Combined with the per-fix "edit only the test's own file" rule in the body. - allowed-files: - - "src/**/*UnitTests*/**/*.cs" - - "src/**/*.Tests/**/*.cs" + # This workflow only ever edits TEST sources. There is intentionally **no** `allowed-files` + # allowlist: a directory-glob allowlist produced false negatives on valid test files (e.g. + # `src/Build.UnitTests/.../Preprocessor_Tests.cs`), silently blocking legitimate fixes and failing + # the run. The product-code guard is instead provided by `excluded-files` + `protected-files`, the + # body's "edit only the test's own file" rule (+ a git diff check), and mandatory human review + # before any PR can merge. # Belt-and-braces enforcement of "never touch .github/**" (also enforced in the prompt + a git diff check). excluded-files: - ".github/**" @@ -86,7 +85,7 @@ tests that are **already quarantined** with `[ActiveIssue]` but are **still flak quarantine pipeline, diagnose the root cause **from the accumulated failure evidence** (error messages + stack traces gathered over many builds and days — **not** from local reproduction), and, when you are **highly confident** of a **minimal, test-only** fix, open **one -individual draft pull request per fixed test**. +individual ready-for-review pull request per fixed test**. **By default you keep the `[ActiveIssue]` quarantine in place.** In that case this PR is a *candidate fix*, not a validated one: normal PR CI still excludes the quarantined test, so the only thing that @@ -139,7 +138,7 @@ repo-wide text search, then locate the class/method within it. 6. Apply each fix; by default **keep** the `[ActiveIssue]`, but at **very high confidence** also remove it in the same file (Step 6 / Step 5b). 7. Build the whole repo **once** to validate the fixes compile (Step 7). -8. Open **one individual draft PR per fixed test** (Step 8). +8. Open **one individual ready-for-review PR per fixed test** (Step 8). ## Step 1 — Scan the quarantine pipeline (definition 344) @@ -350,16 +349,16 @@ The whole-repo build takes ~2-3 minutes — **never cancel it**. Interpret the r the other, compiling fixes. - **Build fails for environmental/network reasons** (NuGet restore cannot reach a feed, a blocked domain, an SDK-download failure — *not* a compile error): **do not retry the build and do not - loop.** The PRs are drafts, so their own CI is the first real compile; **open them anyway** and + loop.** The PR opens ready-for-review, so its own CI is the first real compile; **open them anyway** and note in each PR body that the local validation build was blocked by the environment. One failed attempt is enough to decide this. Because each fix is confined to its **own** test file (Step 5b) and the fixes are independent, a clean union build means each individual fix compiles too. -## Step 8 — Open one individual draft PR per fixed test +## Step 8 — Open one individual ready-for-review PR per fixed test -Each fix becomes its **own** draft PR (not a combined PR). First re-check dedup: re-run the Step 3 +Each fix becomes its **own** ready-for-review PR (not a combined PR). First re-check dedup: re-run the Step 3 `gh pr list ... --label flaky-test` query — a concurrent detector or fixer run may have opened a PR for one of your tests since Step 3; **drop** any test now covered (by marker, issue number, or file). @@ -429,7 +428,7 @@ Each PR: line on the fixed test. This workflow **never** files or closes issues, and **never** edits product code, shared test helpers, `.github/**`, or root manifests. Each PR edits exactly **one** test source file. -- Every PR is a **draft** based on `main`, and is a **candidate** fix pending human review — validated +- Every PR is a **ready-for-review** PR based on `main`, and is a **candidate** fix pending human review — validated by def 344 after merge (default), or additionally by the PR's own CI when you un-quarantined (Step 5b). Never write a closing keyword before the tracking-issue reference. - The JSON report is the **only** source of truth — never invent failures, evidence, or test names.