Adopt Clever Test Selection (CTS) as parallel, non-blocking PR pipeline

.gitignore

@@ -17,6 +17,7 @@ artifacts/
 .dotnet/
 .tools/
 .packages/
+.cts/
 BenchmarkDotNet.Artifacts/
 
 # Visual Studio 2015 cache/options directory

MSBuild.VSTest.slnx

@@ -0,0 +1,87 @@
+<Solution>
+  <!--
+    CI-only solution used by Clever Test Selection (CTS).
+
+    Contains the production projects (mirroring MSBuild.slnx, minus
+    Samples/Package which aren't part of the test surface) plus the
+    sibling *.UnitTests.VSTest.csproj wrappers. The wrappers import their
+    original test csproj but force the VSTest runner stack (CTS does not
+    yet work with Microsoft.Testing.Platform) and pin the TFM to net10.0
+    so a single `dotnet build MSBuild.VSTest.slnx` succeeds.
+
+    MSBuild.slnx is intentionally left untouched so developer / IDE /
+    existing PR pipeline behavior is unchanged — this slnx is only built
+    by azure-pipelines/cts-collect.yml and cts-apply.yml.
+
+    KNOWN COVERAGE GAP: the wrappers run net10.0 only. The regular PR
+    pipeline runs both net10.0 and net472 (Windows). The CTS pipeline is
+    parallel and non-blocking, so net472 signal is still provided by the
+    existing PR pipeline — but CTS does not add incrementality for the
+    net472 leg today. Tracked for follow-up; see scripts/cts/README.md.
+  -->
+  <Configurations>
+    <Platform Name="Any CPU" />
+    <Platform Name="x64" />
+  </Configurations>
+  <Folder Name="/Production/">
+    <Project Path="src/Build/Microsoft.Build.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/Framework/Microsoft.Build.Framework.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/MSBuild/MSBuild.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/MSBuild.Bootstrap/MSBuild.Bootstrap.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/MSBuildTaskHost/MSBuildTaskHost.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/StringTools/StringTools.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/Tasks/Microsoft.Build.Tasks.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/Utilities/Microsoft.Build.Utilities.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+  </Folder>
+  <Folder Name="/TestSupport/">
+    <Project Path="src/UnitTests.Shared/Microsoft.Build.UnitTests.Shared.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/Xunit.NetCore.Extensions/Xunit.NetCore.Extensions.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+  </Folder>
+  <Folder Name="/Tests/">
+    <Project Path="src/Build.OM.UnitTests/Microsoft.Build.Engine.OM.UnitTests.VSTest.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/Build.UnitTests/Microsoft.Build.Engine.UnitTests.VSTest.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/BuildCheck.UnitTests/Microsoft.Build.BuildCheck.UnitTests.VSTest.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/Framework.UnitTests/Microsoft.Build.Framework.UnitTests.VSTest.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/MSBuild.UnitTests/Microsoft.Build.CommandLine.UnitTests.VSTest.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/StringTools.UnitTests/StringTools.UnitTests.VSTest.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/Tasks.UnitTests/Microsoft.Build.Tasks.UnitTests.VSTest.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+    <Project Path="src/Utilities.UnitTests/Microsoft.Build.Utilities.UnitTests.VSTest.csproj">
+      <Platform Solution="*|x64" Project="x64" />
+    </Project>
+  </Folder>
+</Solution>
+

azure-pipelines/cts-apply.yml

@@ -0,0 +1,149 @@
+# azure-pipelines/cts-apply.yml
+#
+# Parallel, non-blocking PR pipeline that exercises Clever Test Selection.
+#
+# Behavior:
+#   1. Download the most recent `cts-baseline-<os>` artifact from the collect
+#      pipeline on `main` (latestFromBranch).
+#   2. Build MSBuild.VSTest.slnx and run `cts apply vstest` against the
+#      wrapped projects.
+#   3. Emit cts-metrics.json per OS (schema: scripts/cts/METRICS.md).
+#
+# Fallback: if `collectPipelineId` is 0 (pipeline not yet registered) or the
+# baseline download fails, the apply step is skipped and only metrics are
+# emitted. We do NOT run a full `dotnet test` as a fallback — the regular PR
+# pipeline keeps providing test signal and duplicating it here just adds CI
+# minutes and confusing duplicate failures.
+#
+# This pipeline runs in parallel with the existing PR pipeline. It is
+# non-blocking (continueOnError on jobs) while we collect data on
+# incrementality.
+#
+# ============================================================
+# TODO(1ES): before registering this pipeline in DevDiv ADO, wrap it in
+# `extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates`
+# and move PublishPipelineArtifact@1 into the templated `outputs:` block.
+# See vs-insertion.yml for the pattern in this repo.
+# ============================================================
+
+trigger: none
+
+# Shadow pipeline — runs on PR but CANNOT block merge.
+#
+# How "non-blocking" is enforced (defense in depth):
+#   1. continueOnError: true on every job below: any step failure inside
+#      the job marks the job as succeededWithIssues, not failed.
+#   2. continueOnError: true on the slow/risky steps inside apply-steps.yml
+#      (cts apply itself, baseline downloads, metrics emission), so even
+#      individual step failures don't propagate.
+#   3. The pipeline run can still end as 'partiallySucceeded' / 'failed'
+#      from ADO's perspective when something inside a `continueOnError`
+#      job goes wrong — that is fine for the *pipeline* result, but it
+#      means whoever registers this pipeline definition in ADO MUST NOT
+#      mark it as a required check in the branch policy of `main` /
+#      `exp/*` / `vs*`. See "Project Settings → Repos → Branch policies →
+#      Build Validation"; this pipeline must be either absent from that
+#      list, or present with the "Required" toggle OFF (ADO will then
+#      attach the build status to the PR for visibility but not gate the
+#      merge button on it).
+#
+# Trigger set: PRs targeting main only. Although the regular PR pipeline
+# (.vsts-dotnet-ci.yml) also covers exp/* and vs* branches, those have no
+# matching CTS-Collect runs (collect runs daily on main), so applying the
+# main baseline against a vs17.x release branch would either be stale or
+# fall back. Keep the shadow pipeline scoped to where it can produce
+# meaningful incrementality data.
+pr:
+  branches:
+    include:
+      - main
+  paths:
+    exclude:
+      - documentation/**
+      - .github/**
+
+# Manual queue is also supported. Useful for running against a specific
+# baseline branch before merge:
+#
+#   az pipelines run --id <CTS-Apply id> `
+#     --branch <source-branch> `
+#     --parameters collectSourceBranch=refs/heads/<baseline-branch>
+#
+# In normal PR flow the auto-trigger above takes care of it and you don't
+# need to touch the parameters — `collectSourceBranch` defaults to
+# `refs/heads/main`, which is where the daily CTS-Collect schedule runs.
+
+parameters:
+# NuGet feed to *install* the `cts` global tool from (read-only). We never
+# publish/push anything to this feed; it's strictly the source of the tool.
+- name: ctsFeed
+  type: string
+  default: https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json
+- name: collectPipelineId
+  type: number
+  # Numeric id of the CTS-Collect pipeline definition in DevDiv ADO.
+  # Pass 0 to disable the apply step (it will short-circuit to a
+  # metrics-only run with fallbackReason=collect-pipeline-not-configured).
+  default: 28380
+- name: collectProject
+  type: string
+  default: DevDiv
+- name: collectSourceBranch
+  type: string
+  # Which branch's CTS-Collect runs we pull the baseline from. Defaults to
+  # `refs/heads/main` for the steady-state (collect runs daily on main; PRs
+  # apply against the latest such baseline). While iterating on a feature
+  # branch -- e.g. before the first successful collect-on-main run, or to
+  # exercise the pipeline pre-merge -- override this to a different branch
+  # (e.g. `refs/heads/adopt-cts`) via the Pipeline UI → Variables, or by
+  # passing `--parameters collectSourceBranch=refs/heads/<branch>` to
+  # `az pipelines run`.
+  default: refs/heads/main
+# Explicit `cts` tool version to install. Empty => latest prerelease. Keep this
+# in lockstep with cts-collect.yml's `ctsToolVersion` so apply and collect run
+# the same tool build (baseline format compatibility).
+- name: ctsToolVersion
+  type: string
+  default: ''
+
+variables:
+  BuildConfiguration: Debug
+  CtsConfigPath: $(Build.SourcesDirectory)/scripts/cts/cts.config.json
+  BaselineDir:   $(Build.SourcesDirectory)/.cts/baseline
+
+jobs:
+- job: Apply_Windows
+  displayName: CTS apply (Windows, non-blocking)
+  pool:
+    name: VSEng-MicroBuildVSStable
+    demands:
+    - agent.os -equals Windows_NT
+  timeoutInMinutes: 120
+  continueOnError: true
+  steps:
+  - template: cts-steps/apply-steps.yml
+    parameters:
+      osTag: windows
+      ctsFeed: ${{ parameters.ctsFeed }}
+      collectPipelineId: ${{ parameters.collectPipelineId }}
+      collectProject: ${{ parameters.collectProject }}
+      collectSourceBranch: ${{ parameters.collectSourceBranch }}
+      ctsToolVersion: ${{ parameters.ctsToolVersion }}
+
+- job: Apply_Linux
+  displayName: CTS apply (Linux, non-blocking)
+  pool:
+    name: AzurePipelines-EO
+    image: 1ESPT-Ubuntu22.04
+    os: linux
+  timeoutInMinutes: 120
+  continueOnError: true
+  steps:
+  - template: cts-steps/apply-steps.yml
+    parameters:
+      osTag: linux
+      ctsFeed: ${{ parameters.ctsFeed }}
+      collectPipelineId: ${{ parameters.collectPipelineId }}
+      collectProject: ${{ parameters.collectProject }}
+      collectSourceBranch: ${{ parameters.collectSourceBranch }}
+      ctsToolVersion: ${{ parameters.ctsToolVersion }}

azure-pipelines/cts-collect.yml

@@ -0,0 +1,82 @@
+# azure-pipelines/cts-collect.yml
+#
+# Daily Clever Test Selection (CTS) baseline collector.
+#
+# Trigger: scheduled, 03:00 UTC every day, against `main` only. Does not run
+# on PR or push. The published artifacts are consumed by cts-apply.yml.
+#
+# Storage: cts writes a snapshot directory locally; we publish it as a
+# pipeline artifact named `cts-baseline-<osTag>` (one slot per OS,
+# overwritten daily). The collect metadata (SHA, timestamp) lives in a
+# sibling `cts-collect-metrics-<osTag>` artifact whose contents follow
+# scripts/cts/METRICS.md.
+#
+# Retention: 14 days, set on the pipeline definition in the ADO UI
+# (not configurable from yaml).
+#
+# ============================================================
+# TODO(1ES): before registering this pipeline in DevDiv ADO, wrap it in
+# `extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates`
+# and move PublishPipelineArtifact@1 into the templated `outputs:` block.
+# See vs-insertion.yml for the pattern in this repo.
+# ============================================================
+
+trigger: none
+pr: none
+
+# Daily baseline collection at 03:00 UTC against main. The artifacts this
+# produces (cts-baseline-<os> + cts-collect-metrics-<os>) are consumed by
+# cts-apply.yml on every PR.
+schedules:
+- cron: '0 3 * * *'
+  displayName: Daily CTS baseline (03:00 UTC)
+  branches:
+    include:
+      - main
+  always: true
+
+parameters:
+# NuGet feed to *install* the `cts` global tool from (read-only). We never
+# publish/push anything to this feed; it's strictly the source of the tool.
+- name: ctsFeed
+  type: string
+  default: https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json
+# Explicit `cts` tool version to install. Empty => latest prerelease. Pin this
+# to the same value in cts-apply.yml so the daily baseline and the per-PR apply
+# run stay on the same tool build (baseline format compatibility).
+- name: ctsToolVersion
+  type: string
+  default: ''
+
+variables:
+  BuildConfiguration: Debug
+  CtsConfigPath: $(Build.SourcesDirectory)/scripts/cts/cts.config.json
+
+jobs:
+- job: Collect_Windows
+  displayName: Collect (Windows)
+  pool:
+    name: VSEng-MicroBuildVSStable
+    demands:
+    - agent.os -equals Windows_NT
+  timeoutInMinutes: 180
+  steps:
+  - template: cts-steps/collect-steps.yml
+    parameters:
+      osTag: windows
+      ctsFeed: ${{ parameters.ctsFeed }}
+      ctsToolVersion: ${{ parameters.ctsToolVersion }}
+
+- job: Collect_Linux
+  displayName: Collect (Linux)
+  pool:
+    name: AzurePipelines-EO
+    image: 1ESPT-Ubuntu22.04
+    os: linux
+  timeoutInMinutes: 180
+  steps:
+  - template: cts-steps/collect-steps.yml
+    parameters:
+      osTag: linux
+      ctsFeed: ${{ parameters.ctsFeed }}
+      ctsToolVersion: ${{ parameters.ctsToolVersion }}