diff --git a/.gitignore b/.gitignore index 43f47c9e72a..42066304213 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ artifacts/ .dotnet/ .tools/ .packages/ +.cts/ BenchmarkDotNet.Artifacts/ # Visual Studio 2015 cache/options directory diff --git a/MSBuild.VSTest.slnx b/MSBuild.VSTest.slnx new file mode 100644 index 00000000000..23421c82a66 --- /dev/null +++ b/MSBuild.VSTest.slnx @@ -0,0 +1,87 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/azure-pipelines/cts-apply.yml b/azure-pipelines/cts-apply.yml new file mode 100644 index 00000000000..4124089eb63 --- /dev/null +++ b/azure-pipelines/cts-apply.yml @@ -0,0 +1,149 @@ +# azure-pipelines/cts-apply.yml +# +# Parallel, non-blocking PR pipeline that exercises Clever Test Selection. +# +# Behavior: +# 1. Download the most recent `cts-baseline-` artifact from the collect +# pipeline on `main` (latestFromBranch). +# 2. Build MSBuild.VSTest.slnx and run `cts apply vstest` against the +# wrapped projects. +# 3. Emit cts-metrics.json per OS (schema: scripts/cts/METRICS.md). +# +# Fallback: if `collectPipelineId` is 0 (pipeline not yet registered) or the +# baseline download fails, the apply step is skipped and only metrics are +# emitted. We do NOT run a full `dotnet test` as a fallback — the regular PR +# pipeline keeps providing test signal and duplicating it here just adds CI +# minutes and confusing duplicate failures. +# +# This pipeline runs in parallel with the existing PR pipeline. It is +# non-blocking (continueOnError on jobs) while we collect data on +# incrementality. +# +# ============================================================ +# TODO(1ES): before registering this pipeline in DevDiv ADO, wrap it in +# `extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates` +# and move PublishPipelineArtifact@1 into the templated `outputs:` block. +# See vs-insertion.yml for the pattern in this repo. +# ============================================================ + +trigger: none + +# Shadow pipeline — runs on PR but CANNOT block merge. +# +# How "non-blocking" is enforced (defense in depth): +# 1. continueOnError: true on every job below: any step failure inside +# the job marks the job as succeededWithIssues, not failed. +# 2. continueOnError: true on the slow/risky steps inside apply-steps.yml +# (cts apply itself, baseline downloads, metrics emission), so even +# individual step failures don't propagate. +# 3. The pipeline run can still end as 'partiallySucceeded' / 'failed' +# from ADO's perspective when something inside a `continueOnError` +# job goes wrong — that is fine for the *pipeline* result, but it +# means whoever registers this pipeline definition in ADO MUST NOT +# mark it as a required check in the branch policy of `main` / +# `exp/*` / `vs*`. See "Project Settings → Repos → Branch policies → +# Build Validation"; this pipeline must be either absent from that +# list, or present with the "Required" toggle OFF (ADO will then +# attach the build status to the PR for visibility but not gate the +# merge button on it). +# +# Trigger set: PRs targeting main only. Although the regular PR pipeline +# (.vsts-dotnet-ci.yml) also covers exp/* and vs* branches, those have no +# matching CTS-Collect runs (collect runs daily on main), so applying the +# main baseline against a vs17.x release branch would either be stale or +# fall back. Keep the shadow pipeline scoped to where it can produce +# meaningful incrementality data. +pr: + branches: + include: + - main + paths: + exclude: + - documentation/** + - .github/** + +# Manual queue is also supported. Useful for running against a specific +# baseline branch before merge: +# +# az pipelines run --id ` +# --branch ` +# --parameters collectSourceBranch=refs/heads/ +# +# In normal PR flow the auto-trigger above takes care of it and you don't +# need to touch the parameters — `collectSourceBranch` defaults to +# `refs/heads/main`, which is where the daily CTS-Collect schedule runs. + +parameters: +# NuGet feed to *install* the `cts` global tool from (read-only). We never +# publish/push anything to this feed; it's strictly the source of the tool. +- name: ctsFeed + type: string + default: https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json +- name: collectPipelineId + type: number + # Numeric id of the CTS-Collect pipeline definition in DevDiv ADO. + # Pass 0 to disable the apply step (it will short-circuit to a + # metrics-only run with fallbackReason=collect-pipeline-not-configured). + default: 28380 +- name: collectProject + type: string + default: DevDiv +- name: collectSourceBranch + type: string + # Which branch's CTS-Collect runs we pull the baseline from. Defaults to + # `refs/heads/main` for the steady-state (collect runs daily on main; PRs + # apply against the latest such baseline). While iterating on a feature + # branch -- e.g. before the first successful collect-on-main run, or to + # exercise the pipeline pre-merge -- override this to a different branch + # (e.g. `refs/heads/adopt-cts`) via the Pipeline UI → Variables, or by + # passing `--parameters collectSourceBranch=refs/heads/` to + # `az pipelines run`. + default: refs/heads/main +# Explicit `cts` tool version to install. Empty => latest prerelease. Keep this +# in lockstep with cts-collect.yml's `ctsToolVersion` so apply and collect run +# the same tool build (baseline format compatibility). +- name: ctsToolVersion + type: string + default: '' + +variables: + BuildConfiguration: Debug + CtsConfigPath: $(Build.SourcesDirectory)/scripts/cts/cts.config.json + BaselineDir: $(Build.SourcesDirectory)/.cts/baseline + +jobs: +- job: Apply_Windows + displayName: CTS apply (Windows, non-blocking) + pool: + name: VSEng-MicroBuildVSStable + demands: + - agent.os -equals Windows_NT + timeoutInMinutes: 120 + continueOnError: true + steps: + - template: cts-steps/apply-steps.yml + parameters: + osTag: windows + ctsFeed: ${{ parameters.ctsFeed }} + collectPipelineId: ${{ parameters.collectPipelineId }} + collectProject: ${{ parameters.collectProject }} + collectSourceBranch: ${{ parameters.collectSourceBranch }} + ctsToolVersion: ${{ parameters.ctsToolVersion }} + +- job: Apply_Linux + displayName: CTS apply (Linux, non-blocking) + pool: + name: AzurePipelines-EO + image: 1ESPT-Ubuntu22.04 + os: linux + timeoutInMinutes: 120 + continueOnError: true + steps: + - template: cts-steps/apply-steps.yml + parameters: + osTag: linux + ctsFeed: ${{ parameters.ctsFeed }} + collectPipelineId: ${{ parameters.collectPipelineId }} + collectProject: ${{ parameters.collectProject }} + collectSourceBranch: ${{ parameters.collectSourceBranch }} + ctsToolVersion: ${{ parameters.ctsToolVersion }} diff --git a/azure-pipelines/cts-collect.yml b/azure-pipelines/cts-collect.yml new file mode 100644 index 00000000000..dd584be13fc --- /dev/null +++ b/azure-pipelines/cts-collect.yml @@ -0,0 +1,82 @@ +# azure-pipelines/cts-collect.yml +# +# Daily Clever Test Selection (CTS) baseline collector. +# +# Trigger: scheduled, 03:00 UTC every day, against `main` only. Does not run +# on PR or push. The published artifacts are consumed by cts-apply.yml. +# +# Storage: cts writes a snapshot directory locally; we publish it as a +# pipeline artifact named `cts-baseline-` (one slot per OS, +# overwritten daily). The collect metadata (SHA, timestamp) lives in a +# sibling `cts-collect-metrics-` artifact whose contents follow +# scripts/cts/METRICS.md. +# +# Retention: 14 days, set on the pipeline definition in the ADO UI +# (not configurable from yaml). +# +# ============================================================ +# TODO(1ES): before registering this pipeline in DevDiv ADO, wrap it in +# `extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates` +# and move PublishPipelineArtifact@1 into the templated `outputs:` block. +# See vs-insertion.yml for the pattern in this repo. +# ============================================================ + +trigger: none +pr: none + +# Daily baseline collection at 03:00 UTC against main. The artifacts this +# produces (cts-baseline- + cts-collect-metrics-) are consumed by +# cts-apply.yml on every PR. +schedules: +- cron: '0 3 * * *' + displayName: Daily CTS baseline (03:00 UTC) + branches: + include: + - main + always: true + +parameters: +# NuGet feed to *install* the `cts` global tool from (read-only). We never +# publish/push anything to this feed; it's strictly the source of the tool. +- name: ctsFeed + type: string + default: https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json +# Explicit `cts` tool version to install. Empty => latest prerelease. Pin this +# to the same value in cts-apply.yml so the daily baseline and the per-PR apply +# run stay on the same tool build (baseline format compatibility). +- name: ctsToolVersion + type: string + default: '' + +variables: + BuildConfiguration: Debug + CtsConfigPath: $(Build.SourcesDirectory)/scripts/cts/cts.config.json + +jobs: +- job: Collect_Windows + displayName: Collect (Windows) + pool: + name: VSEng-MicroBuildVSStable + demands: + - agent.os -equals Windows_NT + timeoutInMinutes: 180 + steps: + - template: cts-steps/collect-steps.yml + parameters: + osTag: windows + ctsFeed: ${{ parameters.ctsFeed }} + ctsToolVersion: ${{ parameters.ctsToolVersion }} + +- job: Collect_Linux + displayName: Collect (Linux) + pool: + name: AzurePipelines-EO + image: 1ESPT-Ubuntu22.04 + os: linux + timeoutInMinutes: 180 + steps: + - template: cts-steps/collect-steps.yml + parameters: + osTag: linux + ctsFeed: ${{ parameters.ctsFeed }} + ctsToolVersion: ${{ parameters.ctsToolVersion }} diff --git a/azure-pipelines/cts-steps/apply-steps.yml b/azure-pipelines/cts-steps/apply-steps.yml new file mode 100644 index 00000000000..42efb4266f3 --- /dev/null +++ b/azure-pipelines/cts-steps/apply-steps.yml @@ -0,0 +1,398 @@ +# azure-pipelines/cts-steps/apply-steps.yml +# +# Per-OS step list reused by cts-apply.yml. +# +# Strategy: download the most recent baseline artifact published by cts-collect.yml +# from main (latestFromBranch) — one per OS, overwritten daily. We do NOT walk +# git ancestry; instead we record the baseline SHA we got and `baselineAgeCommits` +# = git rev-list --count ... If the baseline isn't an +# ancestor of the PR HEAD (e.g. rebased PR or main moved sideways), cts will widen +# selection to compensate — still correct. +# +# Inputs: +# osTag : "windows" | "linux" +# ctsFeed : NuGet feed URL where the `cts` global tool lives +# collectPipelineId : pipeline definition id of cts-collect.yml; 0 disables +# the apply step entirely (see "Fallback" below). +# collectProject : ADO project the collect pipeline lives in (e.g. "DevDiv"). +# +# Fallback: if collectPipelineId is 0 or the artifact download fails, the step +# emits cts-metrics.json with fallbackReason and exits *without* running a full +# `dotnet test`. The regular PR pipeline keeps providing test signal; this +# pipeline is non-blocking and there's no value in duplicating its work. + +parameters: +- name: osTag + type: string +- name: ctsFeed + type: string +- name: collectPipelineId + type: number +- name: collectProject + type: string +- name: collectSourceBranch + type: string + default: refs/heads/main +- name: ctsToolVersion + # Optional explicit version of the `cts` global tool to install (e.g. + # "1.2.3-preview"). Leave empty to float to the latest prerelease. Pin it + # to the SAME value used by cts-collect.yml so apply and collect always run + # the same tool build — otherwise a prerelease published between the daily + # collect and a PR apply can change the baseline serialization format and + # make `cts apply` silently fall back. Bump the pin intentionally. + type: string + default: '' + +steps: +- checkout: self + fetchDepth: 0 # Need history for baselineAgeCommits. + clean: true + +# Drift guard: MSBuild.VSTest.slnx must mirror MSBuild.slnx's production graph +# (see scripts/cts/Check-SlnxParity.ps1). Non-blocking so it never gates a PR — +# a failure surfaces as succeededWithIssues and the "Present/Missing" project +# lists are printed to the step log for triage. +- pwsh: ./scripts/cts/Check-SlnxParity.ps1 + displayName: Check MSBuild.VSTest.slnx parity (non-blocking) + continueOnError: true + +- task: UseDotNet@2 + # cts itself is built against net8.0; see collect-steps.yml. + displayName: Install .NET 8 runtime (cts apphost) + inputs: + packageType: runtime + version: 8.0.x + +- task: UseDotNet@2 + # net10.0 runtime for vstest's testhost.exe; see collect-steps.yml. + displayName: Install .NET 10 runtime (testhost) + inputs: + packageType: runtime + version: 10.0.x + includePreviewVersions: true + +- task: NuGetAuthenticate@1 + displayName: Authenticate to internal NuGet feeds + +- pwsh: | + $ErrorActionPreference = 'Stop' + # The repo's NuGet.config uses package source mapping, so `dotnet tool + # update --add-source ` is rejected with "The --add-source option + # cannot be combined with package source mapping." Work around this by + # writing a temp standalone NuGet.config that contains *only* the cts + # feed (no mapping) and pointing the install at it via --configfile. + $cfgDir = Join-Path $env:AGENT_TEMPDIRECTORY 'cts-install' + New-Item -ItemType Directory -Force -Path $cfgDir | Out-Null + $cfgPath = Join-Path $cfgDir 'NuGet.config' + $feed = '${{ parameters.ctsFeed }}' + $lines = @( + '', + '', + ' ', + ' ', + (' '), + ' ', + '' + ) + Set-Content -Path $cfgPath -Value $lines -Encoding utf8 + $installArgs = @('tool', 'update', 'cts', '--global', '--prerelease', '--configfile', "$cfgPath") + $toolVersion = '${{ parameters.ctsToolVersion }}' + if ($toolVersion) { $installArgs += @('--version', $toolVersion) } + dotnet @installArgs + if ($LASTEXITCODE -ne 0) { throw "cts install/update failed" } + $toolsDir = if ($IsWindows) { Join-Path $env:USERPROFILE '.dotnet\tools' } else { Join-Path $env:HOME '.dotnet/tools' } + Write-Host "##vso[task.prependpath]$toolsDir" + displayName: Install cts tool + +- pwsh: | + $headSha = git rev-parse HEAD + Write-Host "##vso[task.setvariable variable=PrHeadSha]$headSha" + Write-Host "PR HEAD SHA: $headSha" + displayName: Resolve PR HEAD SHA + +- ${{ if ne(parameters.collectPipelineId, 0) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download latest cts-baseline-${{ parameters.osTag }} + continueOnError: true + inputs: + buildType: specific + project: ${{ parameters.collectProject }} + definition: ${{ parameters.collectPipelineId }} + buildVersionToDownload: latestFromBranch + branchName: ${{ parameters.collectSourceBranch }} + artifactName: cts-baseline-${{ parameters.osTag }} + targetPath: $(BaselineDir) + # Required: the collect pipeline frequently ends with overall result + # = "failed" or "partiallySucceeded" because one OS leg may fail while + # the other publishes a usable baseline (and even on the green side, + # `cts collect` itself returns 1 on any test failure, which we + # tolerate but ADO still records). Without these flags + # latestFromBranch finds no candidate runs and the task errors with + # "No builds currently exist in the pipeline definition supplied." + allowPartiallySucceededBuilds: true + allowFailedBuilds: true + - task: DownloadPipelineArtifact@2 + displayName: Download latest cts-collect-metrics-${{ parameters.osTag }} + continueOnError: true + inputs: + buildType: specific + project: ${{ parameters.collectProject }} + definition: ${{ parameters.collectPipelineId }} + buildVersionToDownload: latestFromBranch + branchName: ${{ parameters.collectSourceBranch }} + artifactName: cts-collect-metrics-${{ parameters.osTag }} + targetPath: $(Build.SourcesDirectory)/.cts/baseline-meta + allowPartiallySucceededBuilds: true + allowFailedBuilds: true + +- pwsh: | + # Determine whether the baseline download succeeded and, if so, extract the SHA + # the baseline was taken at from its sibling cts-metrics.json. + $fallback = $null + $baselineSha = $null + $baselineAgeMinutes = $null + if (${{ parameters.collectPipelineId }} -eq 0) { + $fallback = 'collect-pipeline-not-configured' + } elseif (-not (Test-Path "$(BaselineDir)") -or + -not (Get-ChildItem "$(BaselineDir)" -Recurse -ErrorAction SilentlyContinue)) { + $fallback = 'baseline-download-failed' + } else { + $metaFile = Join-Path "$(Build.SourcesDirectory)/.cts/baseline-meta" 'cts-metrics.json' + if (Test-Path $metaFile) { + try { + $meta = Get-Content $metaFile -Raw | ConvertFrom-Json + $baselineSha = $meta.sha + $finish = [DateTimeOffset]::Parse($meta.timestampUtc) + $baselineAgeMinutes = [int]([DateTimeOffset]::UtcNow - $finish).TotalMinutes + } catch { + Write-Host "##vso[task.logissue type=warning]Failed to parse baseline metadata: $_" + } + } + if (-not $baselineSha) { $fallback = 'baseline-metadata-missing' } + } + foreach ($pair in @( + @{ k = 'fallbackReason'; v = $fallback }, + @{ k = 'baselineSha'; v = $baselineSha }, + @{ k = 'baselineAgeMinutes'; v = $baselineAgeMinutes })) { + $val = if ($null -eq $pair.v) { '' } else { "$($pair.v)" } + Write-Host "##vso[task.setvariable variable=$($pair.k)]$val" + } + Write-Host "Resolved baseline: sha=$baselineSha, ageMin=$baselineAgeMinutes, fallback=$fallback" + displayName: Resolve baseline metadata + +- pwsh: | + # Use Arcade's build entry point so init-tools / bootstrap runs before + # the slnx build (MSBuild.Bootstrap needs DotNetInstallScriptRootPath + # + the bundled SDK props). See collect-steps.yml. + # UsingToolVisualStudioIbcTraining=false: see collect-steps.yml note. + if ($IsWindows) { + & "$(Build.SourcesDirectory)/eng/common/build.ps1" ` + -restore -build ` + -configuration $(BuildConfiguration) ` + -projects "$(Build.SourcesDirectory)/MSBuild.VSTest.slnx" ` + -verbosity minimal ` + /p:UsingToolVisualStudioIbcTraining=false + } else { + bash "$(Build.SourcesDirectory)/eng/common/build.sh" ` + --restore --build ` + --configuration $(BuildConfiguration) ` + --projects "$(Build.SourcesDirectory)/MSBuild.VSTest.slnx" ` + --verbosity minimal ` + /p:UsingToolVisualStudioIbcTraining=false + } + if ($LASTEXITCODE -ne 0) { throw "Build MSBuild.VSTest.slnx failed" } + displayName: Build MSBuild.VSTest.slnx + +- pwsh: | + $sw = [Diagnostics.Stopwatch]::StartNew() + $logsDir = "$(Build.SourcesDirectory)/.cts/logs" + New-Item -ItemType Directory -Force -Path $logsDir | Out-Null + $applyLog = Join-Path $logsDir 'cts-apply.console.log' + $fallback = '$(fallbackReason)' + $exit = 0 + if ($fallback) { + Write-Host "##vso[task.logissue type=warning]Skipping cts apply (fallback: $fallback). This run produces metrics only; the regular PR pipeline still provides test signal." + } else { + # Stream cts output to both the console (so you can watch it live in the + # ADO log) and a file (so the next step can parse the summary block). + cts apply vstest ` + --rootPath "$(Build.SourcesDirectory)" ` + --config "$(CtsConfigPath)" ` + --storage-type filesystem ` + --storage-type-filesystem-dir "$(BaselineDir)" ` + --tag "$(baselineSha)" ` + --logs-directory "$logsDir" ` + --coverage ` + --print-console-output 2>&1 | Tee-Object -FilePath $applyLog + $exit = $LASTEXITCODE + if ($exit -ne 0) { + Write-Host "##vso[task.logissue type=warning]cts apply exit $exit (non-blocking)" + # Record as a fallback reason so it shows up in metrics aggregation, + # but only if the baseline data wasn't enough to produce meaningful + # selection. Some test failures don't break the value of the run. + if (-not (Test-Path $applyLog) -or + -not (Select-String -Path $applyLog -Pattern '^\s*discovered:' -Quiet)) { + Write-Host "##vso[task.setvariable variable=fallbackReason]cts-apply-error" + } + } + } + $sw.Stop() + Write-Host "##vso[task.setvariable variable=CtsApplyWallTimeMs]$([int]$sw.ElapsedMilliseconds)" + Write-Host "##vso[task.setvariable variable=CtsApplyExit]$exit" + Write-Host "##vso[task.setvariable variable=CtsApplyLog]$applyLog" + displayName: cts apply vstest + continueOnError: true + +- pwsh: | + # Parse the `=== Summary ===` block emitted by cts apply (same shape as + # cts collect: lines like " discovered: N", " executed: N", " failed: + # N", " succeeded: N", " skipped: N"). Best-effort: if we can't find + # the file or any field, leave that field $null in the metrics + the + # summary box still renders with "n/a". + function Get-CtsField($log, $name) { + if (-not (Test-Path $log)) { return $null } + $line = Select-String -Path $log -Pattern "^\s*${name}:\s*(\S+)" | Select-Object -Last 1 + if (-not $line) { return $null } + $val = $line.Matches[0].Groups[1].Value + if ($val -match '^\d+$') { return [int]$val } + return $val + } + function Get-CtsRatio($log, $patterns) { + if (-not (Test-Path $log)) { return $null } + foreach ($p in $patterns) { + $m = Select-String -Path $log -Pattern $p | Select-Object -Last 1 + if ($m) { + $sel = [int]$m.Matches[0].Groups[1].Value + $tot = [int]$m.Matches[0].Groups[2].Value + if ($tot -gt 0) { + return [pscustomobject]@{ Selected = $sel; Total = $tot; Ratio = [math]::Round($sel / $tot, 4) } + } + } + } + return $null + } + + $log = '$(CtsApplyLog)' + $disc = Get-CtsField $log 'discovered' + $exec = Get-CtsField $log 'executed' + $passed = Get-CtsField $log 'succeeded' + $failed = Get-CtsField $log 'failed' + $skip = Get-CtsField $log 'skipped' + # cts apply emits something like "Selected N out of M tests" or + # "selection: N/M" — try a few patterns. If none match we still print + # what we know (executed / discovered) which is also a useful proxy. + $sel = Get-CtsRatio $log @( + 'Selected\s+(\d+)\s+out\s+of\s+(\d+)\s+tests', + 'selection:\s*(\d+)\s*/\s*(\d+)', + 'impacted:\s*(\d+)\s*/\s*(\d+)' + ) + + # Headline: incrementality % = 1 - executed/discovered if we don't have + # an explicit selection line. Discovered is the total candidate set; the + # number cts actually ran is the "selected" set. + $headlineSelected = if ($sel) { $sel.Selected } else { $exec } + $headlineTotal = if ($sel) { $sel.Total } else { $disc } + $incrementality = $null + if ($headlineTotal -and $headlineTotal -gt 0 -and $null -ne $headlineSelected) { + $incrementality = [math]::Round(100.0 * (1.0 - ($headlineSelected / $headlineTotal)), 2) + } + + function F($v) { if ($null -eq $v) { 'n/a' } else { "$v" } } + + $bar = '=' * 78 + Write-Host '' + Write-Host $bar -ForegroundColor Cyan + Write-Host ' CTS INCREMENTALITY SUMMARY (${{ parameters.osTag }})' -ForegroundColor Cyan + Write-Host $bar -ForegroundColor Cyan + Write-Host (" baseline SHA : {0}" -f (F '$(baselineSha)')) + Write-Host (" baseline age (min) : {0}" -f (F '$(baselineAgeMinutes)')) + Write-Host (" PR HEAD SHA : {0}" -f (F '$(PrHeadSha)')) + Write-Host (" fallback reason : {0}" -f (F '$(fallbackReason)')) + Write-Host ('-' * 78) -ForegroundColor DarkCyan + Write-Host (" candidate tests : {0}" -f (F $headlineTotal)) + Write-Host (" selected tests : {0}" -f (F $headlineSelected)) + Write-Host (" INCREMENTALITY : {0}%" -f (F $incrementality)) -ForegroundColor Yellow + Write-Host ('-' * 78) -ForegroundColor DarkCyan + Write-Host (" discovered : {0}" -f (F $disc)) + Write-Host (" executed : {0}" -f (F $exec)) + Write-Host (" succeeded : {0}" -f (F $passed)) -ForegroundColor Green + Write-Host (" failed : {0}" -f (F $failed)) -ForegroundColor $(if ($failed -gt 0) { 'Red' } else { 'Green' }) + Write-Host (" skipped : {0}" -f (F $skip)) + Write-Host (" cts apply exit : {0}" -f (F '$(CtsApplyExit)')) + Write-Host (" wall time (ms) : {0}" -f (F '$(CtsApplyWallTimeMs)')) + Write-Host $bar -ForegroundColor Cyan + Write-Host ' Compare these counts against the regular PR pipeline (.vsts-dotnet-ci.yml)' -ForegroundColor DarkGray + Write-Host ' to validate the incrementality reading.' -ForegroundColor DarkGray + Write-Host $bar -ForegroundColor Cyan + Write-Host '' + + # Expose the parsed numbers as variables so the metrics step picks them up. + foreach ($pair in @( + @{ k = 'CtsDiscovered'; v = $disc }, + @{ k = 'CtsExecuted'; v = $exec }, + @{ k = 'CtsPassed'; v = $passed }, + @{ k = 'CtsFailed'; v = $failed }, + @{ k = 'CtsSkipped'; v = $skip }, + @{ k = 'CtsSelectedTests'; v = $headlineSelected }, + @{ k = 'CtsTotalTests'; v = $headlineTotal }, + @{ k = 'CtsIncrementalityPct';v = $incrementality })) { + $val = if ($null -eq $pair.v) { '' } else { "$($pair.v)" } + Write-Host "##vso[task.setvariable variable=$($pair.k)]$val" + } + displayName: Summarize CTS apply results + condition: always() + +- pwsh: | + $version = (& cts --version 2>$null) -join ' ' + function Box($v) { if ($null -eq $v -or $v -eq '') { return $null } else { return $v } } + function BoxInt($v) { $b = Box $v; if ($null -eq $b) { return $null } else { return [int]$b } } + function BoxNum($v) { $b = Box $v; if ($null -eq $b) { return $null } else { return [double]$b } } + $baselineSha = Box '$(baselineSha)' + $baselineAgeMin = Box '$(baselineAgeMinutes)' + $wallTime = Box '$(CtsApplyWallTimeMs)' + $baselineAgeCommits = $null + if ($baselineSha) { + try { + $count = git rev-list --count "$baselineSha..$(PrHeadSha)" 2>$null + if ($LASTEXITCODE -eq 0) { $baselineAgeCommits = [int]$count } + } catch { } + } + $metrics = [ordered]@{ + schemaVersion = 1 + phase = 'apply' + os = '${{ parameters.osTag }}' + pipelineRunId = '$(Build.BuildId)' + pipelineRunUrl = "$(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" + repoBranch = '$(Build.SourceBranch)' + wallTimeMs = if ($wallTime) { [int]$wallTime } else { $null } + ctsToolVersion = $version + timestampUtc = (Get-Date).ToUniversalTime().ToString('o') + prHeadSha = '$(PrHeadSha)' + baselineSha = $baselineSha + baselineAgeCommits = $baselineAgeCommits + baselineAgeMinutes = if ($baselineAgeMin) { [int]$baselineAgeMin } else { $null } + fallbackReason = Box '$(fallbackReason)' + selectedTestCount = BoxInt '$(CtsSelectedTests)' + totalCandidateTestCount = BoxInt '$(CtsTotalTests)' + incrementalityPercent = BoxNum '$(CtsIncrementalityPct)' + executedTestCount = BoxInt '$(CtsExecuted)' + passedTestCount = BoxInt '$(CtsPassed)' + failedTestCount = BoxInt '$(CtsFailed)' + skippedTestCount = BoxInt '$(CtsSkipped)' + } + $out = "$(Build.ArtifactStagingDirectory)/cts-apply-metrics-${{ parameters.osTag }}/cts-metrics.json" + New-Item -ItemType Directory -Force -Path (Split-Path $out) | Out-Null + $metrics | ConvertTo-Json -Depth 5 | Set-Content -Path $out -Encoding utf8 + Write-Host "##[group]CTS apply metrics" + Get-Content $out | Write-Host + Write-Host "##[endgroup]" + displayName: Emit cts-metrics.json + condition: always() + +- task: PublishPipelineArtifact@1 + # TODO(1ES): replace with templated `outputs:` once wrapped in 1ES PT. + displayName: Publish cts-apply-metrics artifact + condition: always() + inputs: + targetPath: $(Build.ArtifactStagingDirectory)/cts-apply-metrics-${{ parameters.osTag }} + artifact: cts-apply-metrics-${{ parameters.osTag }} diff --git a/azure-pipelines/cts-steps/collect-steps.yml b/azure-pipelines/cts-steps/collect-steps.yml new file mode 100644 index 00000000000..4e8625769f5 --- /dev/null +++ b/azure-pipelines/cts-steps/collect-steps.yml @@ -0,0 +1,269 @@ +# azure-pipelines/cts-steps/collect-steps.yml +# +# Per-OS step list reused by cts-collect.yml. +# +# Inputs: +# osTag : "windows" | "linux" +# ctsFeed : NuGet feed URL where the `cts` global tool lives +# +# Published artifacts: +# cts-baseline- : the snapshot directory (one slot per OS, +# overwritten by the latest successful daily run). +# cts-collect-metrics- : cts-metrics.json (schema in scripts/cts/METRICS.md); +# contains the SHA that the baseline was taken at. + +parameters: +- name: osTag + type: string +- name: ctsFeed + type: string +- name: ctsToolVersion + # Optional explicit version of the `cts` global tool to install (e.g. + # "1.2.3-preview"). Leave empty to float to the latest prerelease. Pinning + # keeps the daily collect run and the per-PR apply run on the SAME tool + # build, so a mid-day prerelease can't change the baseline serialization + # format out from under `cts apply` (which would surface only as an opaque + # fallbackReason=baseline-download-failed). Bump the pin intentionally. + type: string + default: '' + +steps: +- checkout: self + fetchDepth: 1 + clean: true + +- task: UseDotNet@2 + # cts itself is built against net8.0 (its apphost looks for the .NET 8 + # runtime). Install it into the agent hostedtoolcache so apphosts pick + # it up via DOTNET_ROOT. + displayName: Install .NET 8 runtime (cts apphost) + inputs: + packageType: runtime + version: 8.0.x + +- task: UseDotNet@2 + # vstest spawns testhost.exe (an apphost) per test DLL; the test DLLs + # target net10.0, so testhost needs Microsoft.NETCore.App 10.0.x in the + # agent hostedtoolcache. We can't use `useGlobalJson: true` here because + # this repo's global.json uses Arcade's `tools.dotnet` field rather than + # the standard `sdk.version`, which UseDotNet@2 can't parse. The full + # SDK install used to compile the slnx is handled by eng/common/build.ps1 + # which puts it in the repo-local .dotnet/ -- but apphosts don't look + # there, only at DOTNET_ROOT. + displayName: Install .NET 10 runtime (testhost) + inputs: + packageType: runtime + version: 10.0.x + includePreviewVersions: true + +- task: NuGetAuthenticate@1 + displayName: Authenticate to internal NuGet feeds + +- pwsh: | + $ErrorActionPreference = 'Stop' + # The repo's NuGet.config uses package source mapping, so `dotnet tool + # update --add-source ` is rejected with "The --add-source option + # cannot be combined with package source mapping." Work around this by + # writing a temp standalone NuGet.config that contains *only* the cts + # feed (no mapping) and pointing the install at it via --configfile. + $cfgDir = Join-Path $env:AGENT_TEMPDIRECTORY 'cts-install' + New-Item -ItemType Directory -Force -Path $cfgDir | Out-Null + $cfgPath = Join-Path $cfgDir 'NuGet.config' + $feed = '${{ parameters.ctsFeed }}' + $lines = @( + '', + '', + ' ', + ' ', + (' '), + ' ', + '' + ) + Set-Content -Path $cfgPath -Value $lines -Encoding utf8 + # `tool update` is idempotent: installs if missing, updates if outdated. + # Avoids the "tool already installed" exit on warm self-hosted agents. + $installArgs = @('tool', 'update', 'cts', '--global', '--prerelease', '--configfile', "$cfgPath") + $toolVersion = '${{ parameters.ctsToolVersion }}' + if ($toolVersion) { $installArgs += @('--version', $toolVersion) } + dotnet @installArgs + if ($LASTEXITCODE -ne 0) { throw "cts install/update failed" } + $toolsDir = if ($IsWindows) { Join-Path $env:USERPROFILE '.dotnet\tools' } else { Join-Path $env:HOME '.dotnet/tools' } + Write-Host "##vso[task.prependpath]$toolsDir" + displayName: Install cts tool + +- pwsh: cts --version + displayName: Print cts version + +- pwsh: | + $sha = git rev-parse HEAD + Write-Host "##vso[task.setvariable variable=CtsSha]$sha" + Write-Host "Collect baseline SHA: $sha" + displayName: Resolve HEAD SHA + +- pwsh: | + # Use Arcade's build entry point so init-tools / bootstrap (which the + # MSBuild.Bootstrap project requires for DotNetInstallScriptRootPath + + # bundled SDK props) runs before the slnx build. Direct `dotnet build + # MSBuild.VSTest.slnx` skips init and fails with MSB4044/MSB3030. + # UsingToolVisualStudioIbcTraining=false: we're not producing VS + # Insertion outputs, so skip Arcade's AfterSigning IBC-inputs step + # which otherwise errors on missing artifacts/VSSetup/Debug/Insertion. + if ($IsWindows) { + & "$(Build.SourcesDirectory)/eng/common/build.ps1" ` + -restore -build ` + -configuration $(BuildConfiguration) ` + -projects "$(Build.SourcesDirectory)/MSBuild.VSTest.slnx" ` + -verbosity minimal ` + /p:UsingToolVisualStudioIbcTraining=false + } else { + bash "$(Build.SourcesDirectory)/eng/common/build.sh" ` + --restore --build ` + --configuration $(BuildConfiguration) ` + --projects "$(Build.SourcesDirectory)/MSBuild.VSTest.slnx" ` + --verbosity minimal ` + /p:UsingToolVisualStudioIbcTraining=false + } + if ($LASTEXITCODE -ne 0) { throw "Build MSBuild.VSTest.slnx failed" } + displayName: Build MSBuild.VSTest.slnx + +- pwsh: | + $sw = [Diagnostics.Stopwatch]::StartNew() + $baselineDir = "$(Build.SourcesDirectory)/.cts/baseline" + $logsDir = "$(Build.SourcesDirectory)/.cts/logs" + New-Item -ItemType Directory -Force -Path $baselineDir, $logsDir | Out-Null + + # Echo baseline dir as a pipeline-scope variable so the publish step + # can always find it regardless of which step set it. + Write-Host "##vso[task.setvariable variable=CtsBaselineDir]$baselineDir" + + # The whole script is wrapped so we can guarantee CtsBaselineProduced + # is published (true/false) no matter what failure mode happens. + $exit = -1 + try { + cts collect vstest ` + --rootPath "$(Build.SourcesDirectory)" ` + --config "$(CtsConfigPath)" ` + --storage-type filesystem ` + --storage-type-filesystem-dir "$baselineDir" ` + --tag "$(CtsSha)" ` + --logs-directory "$logsDir" ` + --coverage ` + --print-console-output + $exit = $LASTEXITCODE + } catch { + Write-Host "##vso[task.logissue type=error]cts collect threw: $_" + $exit = -2 + } finally { + $sw.Stop() + Write-Host "##vso[task.setvariable variable=CtsWallTimeMs]$([int]$sw.ElapsedMilliseconds)" + Write-Host "##vso[task.setvariable variable=CtsExitCode]$exit" + + # Independently determine whether the baseline directory has content + # we'd want to publish, regardless of what cts's exit code said. + $baselineHasContent = $false + if (Test-Path $baselineDir -PathType Container) { + $fileCount = @(Get-ChildItem -Path $baselineDir -Recurse -File -ErrorAction SilentlyContinue).Count + $totalBytes = (Get-ChildItem -Path $baselineDir -Recurse -File -ErrorAction SilentlyContinue | + Measure-Object -Property Length -Sum).Sum + if ($null -eq $totalBytes) { $totalBytes = 0 } + Write-Host "Baseline diagnostics: $fileCount file(s), $totalBytes byte(s) at $baselineDir" + if ($fileCount -gt 0 -and $totalBytes -gt 0) { + $baselineHasContent = $true + } + } else { + Write-Host "Baseline diagnostics: $baselineDir does not exist." + } + + # The publish step keys off this variable: 'true' means "we have + # data worth publishing as cts-baseline-". Anything else means + # skip the publish to avoid an empty/garbage artifact masquerading + # as a usable baseline (which would silently break cts apply for + # everyone consuming this artifact). + $produced = if ($baselineHasContent) { 'true' } else { 'false' } + Write-Host "##vso[task.setvariable variable=CtsBaselineProduced]$produced" + Write-Host "Decision: CtsBaselineProduced=$produced (exit=$exit)" + } + + # cts collect exits non-zero if any individual test failed, even if the + # baseline was produced. For our purposes the baseline is still useful + # (coverage data exists for the tests that did execute). Treat the run + # as successful when: + # * exit == 0 → all tests passed + # * exit == 1 AND a non-empty baseline was produced + # Any other combination is a real cts failure (tool crash, IO error, + # config error). The publish step is independently gated on + # CtsBaselineProduced=true so we still publish triage data when + # available. + if ($exit -eq 0) { + Write-Host "cts collect succeeded with all tests passing." + } elseif ($exit -eq 1 -and $baselineHasContent) { + Write-Host "##vso[task.logissue type=warning]cts collect reported test failures (exit 1) but produced a baseline; continuing." + } else { + # Reset LASTEXITCODE so the throw is what kills the step, not the + # implicit non-zero from cts. + $global:LASTEXITCODE = 0 + throw "cts collect failed (exit=$exit, baselineProduced=$baselineHasContent)" + } + + # Reset $LASTEXITCODE so the pwsh task doesn't inherit cts's non-zero + # exit even though we decided to continue. Without this, the agent + # marks the whole task as Failed (which skips the artifact publish). + $global:LASTEXITCODE = 0 + exit 0 + displayName: cts collect vstest + +- task: PublishPipelineArtifact@1 + # TODO(1ES): when this pipeline is wrapped in 1ES.Official.PipelineTemplate.yml, + # replace this task with a templated `outputs:` entry. See PR description. + # + # Belt-and-suspenders: condition is evaluated independent of the cts step's + # success — we publish whenever the cts step said "the baseline directory + # has real content". This protects against future regressions where the + # cts step somehow throws (e.g. an unhandled exception in the parsing + # code) but the baseline was already on disk: we still get the artifact + # for triage / apply consumption. + displayName: Publish cts-baseline-${{ parameters.osTag }} + condition: and(always(), eq(variables['CtsBaselineProduced'], 'true')) + inputs: + targetPath: $(Build.SourcesDirectory)/.cts/baseline + artifact: cts-baseline-${{ parameters.osTag }} + +- pwsh: | + if ('$(CtsBaselineProduced)' -ne 'true') { + Write-Host "##vso[task.logissue type=warning]cts-baseline-${{ parameters.osTag }} was NOT published (CtsBaselineProduced=$(CtsBaselineProduced)). Downstream cts apply runs will hit fallbackReason=baseline-download-failed. See the 'cts collect vstest' step log for the underlying reason." + } else { + Write-Host "cts-baseline-${{ parameters.osTag }} published successfully (CtsBaselineProduced=true)." + } + displayName: Confirm baseline publish status + condition: always() + +- pwsh: | + $version = (& cts --version 2>$null) -join ' ' + $metrics = [ordered]@{ + schemaVersion = 1 + phase = 'collect' + os = '${{ parameters.osTag }}' + pipelineRunId = '$(Build.BuildId)' + pipelineRunUrl = "$(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" + repoBranch = '$(Build.SourceBranch)' + wallTimeMs = if ('$(CtsWallTimeMs)' -ne '') { [int]'$(CtsWallTimeMs)' } else { $null } + ctsToolVersion = $version + timestampUtc = (Get-Date).ToUniversalTime().ToString('o') + sha = '$(CtsSha)' + artifactName = "cts-baseline-${{ parameters.osTag }}" + } + $out = "$(Build.ArtifactStagingDirectory)/cts-collect-metrics-${{ parameters.osTag }}/cts-metrics.json" + New-Item -ItemType Directory -Force -Path (Split-Path $out) | Out-Null + $metrics | ConvertTo-Json -Depth 5 | Set-Content -Path $out -Encoding utf8 + Write-Host "##[group]CTS collect metrics" + Get-Content $out | Write-Host + Write-Host "##[endgroup]" + displayName: Emit cts-metrics.json + condition: always() + +- task: PublishPipelineArtifact@1 + displayName: Publish cts-collect-metrics artifact + condition: always() + inputs: + targetPath: $(Build.ArtifactStagingDirectory)/cts-collect-metrics-${{ parameters.osTag }} + artifact: cts-collect-metrics-${{ parameters.osTag }} diff --git a/scripts/cts/Check-SlnxParity.ps1 b/scripts/cts/Check-SlnxParity.ps1 new file mode 100644 index 00000000000..b062f61bf5e --- /dev/null +++ b/scripts/cts/Check-SlnxParity.ps1 @@ -0,0 +1,75 @@ +<# +.SYNOPSIS + Verifies that MSBuild.VSTest.slnx and MSBuild.slnx reference the same + production projects. Fails (exit 1) on drift. + +.DESCRIPTION + MSBuild.VSTest.slnx mirrors MSBuild.slnx's production graph and adds + sibling *.UnitTests.VSTest.csproj wrappers in /Tests/. When someone + adds a new production project to MSBuild.slnx they need to add it to + MSBuild.VSTest.slnx too, otherwise CTS will silently build a stale + project graph. + + Heuristic: consider every src/* project under /Production/ in + MSBuild.VSTest.slnx and every src/* project in MSBuild.slnx that is + not a *.UnitTests* / *.Tests project; the two sets must match. + + Wired into azure-pipelines/cts-apply.yml as a non-blocking PR-time step + (see the "Check MSBuild.VSTest.slnx parity" step). Runs standalone with no + dependency on the `cts` tool, so it can also be used as a local pre-commit + check: `pwsh ./scripts/cts/Check-SlnxParity.ps1`. +#> +[CmdletBinding()] +param() + +$ErrorActionPreference = 'Stop' +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path + +function Get-ProductionProjects([string]$slnx) { + $xml = [xml](Get-Content (Join-Path $repoRoot $slnx) -Raw) + $all = @() + $all += @($xml.Solution.Project) + foreach ($f in @($xml.Solution.Folder)) { $all += @($f.Project) } + + # Production set = src/* projects that are not tests, not samples, + # not packaging metadata, and not the Roslyn analyzer (which lives + # outside the CTS test surface). + $excludeLike = @( + '*UnitTests*', '*.Tests/*', '*Benchmark*', + 'src/Samples/*', 'src/Package/*', + 'src/ThreadSafeTaskAnalyzer/*', + # TestSupport projects mirrored only into MSBuild.VSTest.slnx + # because the wrappers reference them transitively. They are not + # "production" in MSBuild.slnx terms. + 'src/UnitTests.Shared/*', 'src/Xunit.NetCore.Extensions/*' + ) + + $all | + Where-Object { $_ -and $_.Path -and $_.Path -like 'src/*' } | + ForEach-Object { ($_.Path -replace '\\','/') } | + Where-Object { + $p = $_ + -not ($excludeLike | Where-Object { $p -like $_ }) + } | + Sort-Object -Unique +} + +$mainProd = Get-ProductionProjects 'MSBuild.slnx' +$vstestProd = Get-ProductionProjects 'MSBuild.VSTest.slnx' + +$missing = $mainProd | Where-Object { $_ -notin $vstestProd } +$extra = $vstestProd | Where-Object { $_ -notin $mainProd } + +if ($missing -or $extra) { + Write-Host "MSBuild.VSTest.slnx is out of sync with MSBuild.slnx." -ForegroundColor Red + if ($missing) { + Write-Host " Missing in MSBuild.VSTest.slnx:" -ForegroundColor Red + $missing | ForEach-Object { Write-Host " $_" } + } + if ($extra) { + Write-Host " Present in MSBuild.VSTest.slnx but not MSBuild.slnx:" -ForegroundColor Red + $extra | ForEach-Object { Write-Host " $_" } + } + exit 1 +} +Write-Host ("MSBuild.VSTest.slnx production set matches MSBuild.slnx ({0} projects)." -f $mainProd.Count) -ForegroundColor Green diff --git a/scripts/cts/Collect-Local.ps1 b/scripts/cts/Collect-Local.ps1 new file mode 100644 index 00000000000..7baef33252a --- /dev/null +++ b/scripts/cts/Collect-Local.ps1 @@ -0,0 +1,97 @@ +<# +.SYNOPSIS + Collects a CTS baseline for the given test project(s). + +.DESCRIPTION + Runs `cts collect vstest --coverage` against the *.UnitTests.VSTest + wrapper for each project, populating the local filesystem cache under + /.cts/baseline. The baseline is keyed by HEAD SHA, so this script + requires a clean working tree. + +.PARAMETER Project + Short key from projects.json (e.g. StringTools). Default: all projects. + +.PARAMETER SkipBuild + Reuse already-built VSTest DLLs. + +.PARAMETER TimeoutMinutes + Per-project timeout. Default 15. + +.PARAMETER Dop + Degree of parallelism for CTS. Default 4. +#> +[CmdletBinding()] +param( + [string]$Project, + [switch]$SkipBuild, + [int]$TimeoutMinutes = 15, + [int]$Dop = 4 +) + +. (Join-Path $PSScriptRoot '_Common.ps1') + +Ensure-Cli +Assert-CleanRepo + +$projects = Get-Projects -Filter $Project +$null = New-Item -ItemType Directory -Force -Path $script:BaselineDir, $script:LogsDir + +$totalSw = [Diagnostics.Stopwatch]::StartNew() +$summary = @() + +foreach ($p in $projects) { + Write-Host "==== Collect $($p.Key) ====" -ForegroundColor Cyan + + if (-not $SkipBuild) { Build-Project $p } + + $dll = Get-ProjectDllPath $p + if (-not (Test-Path $dll)) { + throw "DLL not found at '$dll'. Did the build run? Drop -SkipBuild." + } + + $logFile = Join-Path $script:LogsDir "collect-$($p.Key).log" + $errFile = Join-Path $script:LogsDir "collect-$($p.Key).err.log" + + $ctsArgs = @( + 'collect','vstest', + '--rootPath', $script:RepoRoot, + '--config', $script:ConfigPath, + '--storage-type', 'filesystem', + '--storage-type-filesystem-dir', $script:BaselineDir, + '--tag', (Get-ProjectTag $p), + '--logs-directory', $script:LogsDir, + '--filter', (Get-ProjectDllFilter $p), + '--dop', $Dop, + '--coverage', + '--print-console-output' + ) + + $sw = [Diagnostics.Stopwatch]::StartNew() + $proc = Start-Process -FilePath cts -ArgumentList (ConvertTo-ProcessArgumentList $ctsArgs) ` + -RedirectStandardOutput $logFile -RedirectStandardError $errFile -PassThru -NoNewWindow + if (-not $proc.WaitForExit($TimeoutMinutes * 60 * 1000)) { + Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue + $sw.Stop() + Write-Host " TIMEOUT after $TimeoutMinutes min" -ForegroundColor Red + Get-Content $logFile -Tail 30 | ForEach-Object { Write-Host " $_" } + $summary += [pscustomobject]@{ Project = $p.Key; Status = 'TIMEOUT'; Duration = $sw.Elapsed } + continue + } + $sw.Stop() + + $status = if ($proc.ExitCode -eq 0) { 'OK' } else { "EXIT $($proc.ExitCode)" } + $color = if ($proc.ExitCode -eq 0) { 'Green' } else { 'Red' } + Write-Host " $status ($([int]$sw.Elapsed.TotalSeconds)s) -> $logFile" -ForegroundColor $color + if ($proc.ExitCode -ne 0) { + Get-Content $logFile -Tail 15 | ForEach-Object { Write-Host " $_" } + } + $summary += [pscustomobject]@{ Project = $p.Key; Status = $status; Duration = $sw.Elapsed } +} + +$totalSw.Stop() +Write-Host "" +Write-Host "==== Summary ($([int]$totalSw.Elapsed.TotalSeconds)s total) ====" -ForegroundColor Cyan +$summary | Format-Table -AutoSize | Out-Host +Write-Host "Baseline cache: $script:BaselineDir" -ForegroundColor DarkGray + +if ($summary | Where-Object { $_.Status -ne 'OK' }) { exit 1 } else { exit 0 } diff --git a/scripts/cts/METRICS.md b/scripts/cts/METRICS.md new file mode 100644 index 00000000000..ac105e319d5 --- /dev/null +++ b/scripts/cts/METRICS.md @@ -0,0 +1,69 @@ +# CTS pipeline telemetry — `cts-metrics.json` + +Both `azure-pipelines/cts-collect.yml` and `azure-pipelines/cts-apply.yml` +emit a single `cts-metrics.json` file per OS per run and publish it as a +pipeline artifact. They also echo it inside a `##[group]CTS apply metrics` +or `##[group]CTS collect metrics` log block (depending on phase) so the +data is searchable in pipeline logs. + +The schema is intentionally flat so it can be ingested later into Kusto / +App Insights without remodelling. Unknown fields should be ignored by +consumers; new fields will be added at the end of this list. + +## Common fields (always present) + +| Field | Type | Notes | +| --- | --- | --- | +| `schemaVersion` | int | Bump on any breaking change. Current: `1`. | +| `phase` | string | `"collect"` or `"apply"`. | +| `os` | string | `"windows"` or `"linux"` (matches pool image family). | +| `pipelineRunId` | string | `$(Build.BuildId)`. | +| `pipelineRunUrl` | string | Link to the run for human triage. | +| `repoBranch` | string | `$(Build.SourceBranch)`. | +| `wallTimeMs` | int \| null | Wall-clock time spent inside the `cts` invocation. | +| `ctsToolVersion` | string | Output of `cts --version`. | +| `timestampUtc` | string | ISO-8601, run start. | + +## `phase=collect` + +| Field | Type | Notes | +| --- | --- | --- | +| `sha` | string | HEAD SHA of `main` at job start. The baseline tag passed to `cts collect --tag`. | +| `artifactName` | string | Pipeline artifact name (`cts-baseline-`). | + +## `phase=apply` + +| Field | Type | Notes | +| --- | --- | --- | +| `prHeadSha` | string | HEAD SHA of the PR being validated. | +| `baselineSha` | string \| null | SHA of the snapshot we resolved (null on fallback). | +| `baselineAgeCommits` | int \| null | `git rev-list --count ..`. | +| `baselineAgeMinutes` | int \| null | Wall-clock age of the baseline. | +| `fallbackReason` | string \| null | `null` on happy path. Known values: `"collect-pipeline-not-configured"` (apply yaml's `collectPipelineId == 0`), `"baseline-download-failed"`, `"baseline-metadata-missing"`, `"cts-apply-error"`. | +| `selectedTestCount` | int \| null | Tests CTS selected as impacted (headline number). `null` on fallback. | +| `totalCandidateTestCount` | int \| null | Total candidate tests considered (headline number). `null` on fallback. | +| `incrementalityPercent` | number \| null | Percentage reduction achieved by the selection. `null` on fallback. | +| `executedTestCount` | int \| null | Tests actually executed by the apply run. `null` on fallback. | +| `passedTestCount` | int \| null | Executed tests that passed. `null` on fallback. | +| `failedTestCount` | int \| null | Executed tests that failed. `null` on fallback. | +| `skippedTestCount` | int \| null | Executed tests that were skipped. `null` on fallback. | + +## Coverage note + +Tests are net10.0 only — the wrappers cannot host the net472 leg (xunit.v3 +requires `OutputType=Exe`, CTS requires `OutputType=Library`). The regular +PR pipeline continues to provide net472 signal. See `scripts/cts/README.md` +→ "Coverage gap" for the details. + +## Not yet populated + +The following fields are intentionally absent from v1 of the schema; they +require parsing the `cts` log/JSON output and will be added in a follow-up: + +* `testCount`, `moduleCount`, `coverageBytes` (collect) + +## Conventions + +* When a value cannot be computed it is emitted as JSON `null` (never omitted). +* Times are in UTC, milliseconds. +* Per-OS files keep the field set identical so an aggregator can append rows. diff --git a/scripts/cts/README.md b/scripts/cts/README.md new file mode 100644 index 00000000000..5bda0ef0b80 --- /dev/null +++ b/scripts/cts/README.md @@ -0,0 +1,115 @@ +# scripts/cts — Local CTS (Clever Test Selection) harness + +These scripts run CTS against this repository in **VSTest mode**, which +avoids the MTP↔CTS JsonRpc hang we hit during the initial adoption attempt. + +They drive the **sibling `*.UnitTests.VSTest.csproj`** wrappers next to each +test project. Each wrapper imports the original `.csproj` so the test +surface stays in sync; only the runner stack differs. + +## Layout + +| File | Role | +| -------------------------- | ---------------------------------------------------------- | +| `projects.json` | Registry of `*.UnitTests.VSTest` projects + demo files | +| `cts.config.json` | CTS configuration (Modules / SourceCodeFiles / Filter) | +| `_Common.ps1` | Tiny PS helpers (paths, build, registry lookup) | +| `Collect-Local.ps1` | `cts collect vstest --coverage` → baseline in `.cts/` | +| `Run-Local.ps1` | `cts apply vstest --local-development` → impacted-only run | +| `demos/Demo-NoChange.ps1` | Apply with no edits (expect 0 impacted) | +| `demos/Demo-NarrowEdit.ps1` | Touch a narrow source file (expect partial selection) | +| `demos/Demo-BroadEdit.ps1` | Touch a broad source file (expect ~all selected) | +| `demos/Demo-UnrelatedEdit.ps1` | Touch a file outside the project (expect 0 impacted) | + +Anything you would tweak as configuration belongs in `projects.json` or +`cts.config.json`, not in the PowerShell. + +## Prerequisites + +```powershell +dotnet tool install cts --global --prerelease ` + --add-source https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json +``` + +The `cts` command must resolve on PATH. A `.cts/` directory at repo root is +used as the local filesystem cache (gitignored). + +## Workflow + +```powershell +# 1. Collect baseline (clean working tree required) +.\scripts\cts\Collect-Local.ps1 -Project StringTools + +# 2. Make changes, then run only impacted tests +.\scripts\cts\Run-Local.ps1 -Project StringTools + +# 3. See it in action +.\scripts\cts\demos\Demo-NoChange.ps1 -Project StringTools +.\scripts\cts\demos\Demo-NarrowEdit.ps1 -Project StringTools +.\scripts\cts\demos\Demo-BroadEdit.ps1 -Project StringTools +.\scripts\cts\demos\Demo-UnrelatedEdit.ps1 -Project StringTools +``` + +Omit `-Project` to operate on every project registered in `projects.json`. + +## Coverage gap vs the regular PR pipeline + +The wrappers are pinned to `TargetFrameworks=net10.0`. xunit.v3 rejects +`net472` when `OutputType=Library` (which CTS needs so it can host the +test DLL), so we cannot run the .NET Framework leg through CTS today. + +What this means in practice: + +| TFM | Regular PR pipeline | CTS pipeline | +| -------------- | ------------------- | ------------ | +| `net10.0` | ✅ | ✅ | +| `net472` (Win) | ✅ | ❌ | + +The CTS pipeline is parallel and **non-blocking**; the regular PR +pipeline continues to provide net472 signal and remains the merge gate. +CTS adds incrementality for the net10.0 subset only. Closing the gap +requires either an `OutputType=Exe` net472 wrapper variant (needs +validation that `cts vstest` works against a .NET Framework Exe host) or +a legacy-xunit wrapper for net472. + +## Local vs CI + +These scripts are **local-only** — they use the filesystem cache under +`/.cts/` so iterating between collect and apply is instant. They +target Windows; the CI pipelines invoke `cts` directly and do not source +`_Common.ps1`, so non-Windows local users should call `cts` themselves. + +CI runs the same `cts` tool but against ADO pipeline artifacts as the +snapshot store. Two pipelines drive it: + +* `azure-pipelines/cts-collect.yml` — scheduled daily at 03:00 UTC against + `main`; uses `--storage-type filesystem` locally on the agent and + publishes the snapshot directory as `cts-baseline-` (one slot per + OS, overwritten daily) plus a sibling `cts-collect-metrics-` + artifact containing the SHA the baseline was taken at. +* `azure-pipelines/cts-apply.yml` — runs in parallel on PRs (non-blocking); + downloads the latest `cts-baseline-` from main via + `DownloadPipelineArtifact@2`, runs `cts apply vstest` against + `MSBuild.VSTest.slnx`. If `collectPipelineId` isn't configured yet (or + download fails), the apply step is skipped and only metrics are emitted + — we do **not** duplicate the regular PR pipeline's full test run. + +Both pipelines emit `cts-metrics.json` per OS (schema documented in +[`METRICS.md`](METRICS.md)) so we can later quantify the incrementality +CTS achieves. + +`Check-SlnxParity.ps1` verifies that `MSBuild.VSTest.slnx`'s production +project list matches `MSBuild.slnx`'s so additions to one solution don't +silently skip the other. It is wired into `azure-pipelines/cts-apply.yml` as +a **non-blocking** PR-time step, and can also be run locally: +`pwsh ./scripts/cts/Check-SlnxParity.ps1`. + +## Notes + +* Baseline + logs live at `/.cts/` (gitignored). +* The `.VSTest.csproj` wrappers output to + `artifacts/bin//Debug/net10.0/` — distinct from the + default MTP variant. +* Demos hardcode per-project demo files via `projects.json` → + `DemoFiles.{Broad,Narrow,Unrelated}`. Projects without `DemoFiles` skip + the narrow/broad/unrelated demos. diff --git a/scripts/cts/Run-Local.ps1 b/scripts/cts/Run-Local.ps1 new file mode 100644 index 00000000000..d55dfedbf3d --- /dev/null +++ b/scripts/cts/Run-Local.ps1 @@ -0,0 +1,104 @@ +<# +.SYNOPSIS + Runs only the tests CTS considers impacted by the current working tree + against the *.UnitTests.VSTest projects. + +.DESCRIPTION + Runs `cts apply vstest --local-development` per project. Requires a + baseline from .\Collect-Local.ps1. + +.PARAMETER Project + Short key from projects.json (e.g. StringTools). Default: all projects. + +.PARAMETER SkipBuild + Reuse already-built VSTest DLLs. + +.PARAMETER TimeoutMinutes + Per-project timeout. Default 15. + +.PARAMETER Dop + Degree of parallelism for CTS. Default 4. +#> +[CmdletBinding()] +param( + [string]$Project, + [switch]$SkipBuild, + [int]$TimeoutMinutes = 15, + [int]$Dop = 4 +) + +. (Join-Path $PSScriptRoot '_Common.ps1') + +Ensure-Cli + +if (-not (Test-Path $script:BaselineDir) -or + -not (Get-ChildItem -Path $script:BaselineDir -Recurse -ErrorAction SilentlyContinue)) { + throw "No CTS baseline at '$script:BaselineDir'. Run .\Collect-Local.ps1 first." +} + +$projects = Get-Projects -Filter $Project +$null = New-Item -ItemType Directory -Force -Path $script:LogsDir + +$totalSw = [Diagnostics.Stopwatch]::StartNew() +$summary = @() + +foreach ($p in $projects) { + Write-Host "==== Apply $($p.Key) ====" -ForegroundColor Cyan + + if (-not $SkipBuild) { Build-Project $p } + + $dll = Get-ProjectDllPath $p + if (-not (Test-Path $dll)) { + throw "DLL not found at '$dll'. Did the build run? Drop -SkipBuild." + } + + $logFile = Join-Path $script:LogsDir "apply-$($p.Key).log" + $errFile = Join-Path $script:LogsDir "apply-$($p.Key).err.log" + + $ctsArgs = @( + 'apply','vstest', + '--rootPath', $script:RepoRoot, + '--config', $script:ConfigPath, + '--storage-type', 'filesystem', + '--storage-type-filesystem-dir', $script:BaselineDir, + '--tag', (Get-ProjectTag $p), + '--local-development', + '--logs-directory', $script:LogsDir, + '--filter', (Get-ProjectDllFilter $p), + '--dop', $Dop, + '--print-console-output' + ) + + $sw = [Diagnostics.Stopwatch]::StartNew() + $proc = Start-Process -FilePath cts -ArgumentList (ConvertTo-ProcessArgumentList $ctsArgs) ` + -RedirectStandardOutput $logFile -RedirectStandardError $errFile -PassThru -NoNewWindow + if (-not $proc.WaitForExit($TimeoutMinutes * 60 * 1000)) { + Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue + $sw.Stop() + Write-Host " TIMEOUT after $TimeoutMinutes min" -ForegroundColor Red + Get-Content $logFile -Tail 30 | ForEach-Object { Write-Host " $_" } + $summary += [pscustomobject]@{ Project = $p.Key; Status = 'TIMEOUT'; Duration = $sw.Elapsed } + continue + } + $sw.Stop() + + $status = if ($proc.ExitCode -eq 0) { 'OK' } else { "EXIT $($proc.ExitCode)" } + $color = if ($proc.ExitCode -eq 0) { 'Green' } else { 'Red' } + Write-Host " $status ($([int]$sw.Elapsed.TotalSeconds)s) -> $logFile" -ForegroundColor $color + + Get-Content $logFile -Tail 30 | + Where-Object { $_ -match 'impacted|executed|succeeded|failed|reason ' } | + ForEach-Object { Write-Host " $_" -ForegroundColor DarkGray } + + if ($proc.ExitCode -ne 0) { + Get-Content $logFile -Tail 10 | ForEach-Object { Write-Host " $_" } + } + $summary += [pscustomobject]@{ Project = $p.Key; Status = $status; Duration = $sw.Elapsed } +} + +$totalSw.Stop() +Write-Host "" +Write-Host "==== Summary ($([int]$totalSw.Elapsed.TotalSeconds)s total) ====" -ForegroundColor Cyan +$summary | Format-Table -AutoSize | Out-Host + +if ($summary | Where-Object { $_.Status -ne 'OK' }) { exit 1 } else { exit 0 } diff --git a/scripts/cts/_Common.ps1 b/scripts/cts/_Common.ps1 new file mode 100644 index 00000000000..bf5f100d7d2 --- /dev/null +++ b/scripts/cts/_Common.ps1 @@ -0,0 +1,114 @@ +# scripts/cts/_Common.ps1 +# +# Shared helpers for Collect-Local.ps1, Run-Local.ps1, and the demo scripts. +# Configuration lives in two JSON files alongside this script: +# +# projects.json - the registry of *.UnitTests.VSTest projects +# cts.config.json - the CTS configuration (Modules/SourceCodeFiles/Filter/...) +# +# This file intentionally contains only paths and functions; nothing that you +# would tweak as configuration belongs here. +# +# These helpers target the local Windows developer workflow. On non-Windows +# we early-out with a clear message — the CI pipelines invoke `cts` directly +# and do not source this file. + +$ErrorActionPreference = 'Stop' + +if (-not $IsWindows -and $PSVersionTable.PSEdition -ne 'Desktop') { + throw "scripts/cts/*.ps1 currently target Windows. The CI pipelines call cts directly; for local use on Linux/macOS, invoke cts manually." +} + +# Paths. Use forward slashes everywhere — Join-Path normalises to the host +# separator on Windows and they are valid as-is on Linux/macOS, so the same +# helpers will work if the Windows-only guard above is ever relaxed. +$script:RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path +$_dotnetFile = if ($IsWindows -or $PSVersionTable.PSEdition -eq 'Desktop') { 'dotnet.exe' } else { 'dotnet' } +$script:DotnetExe = Join-Path (Join-Path $script:RepoRoot '.dotnet') $_dotnetFile +$script:CtsRoot = Join-Path $script:RepoRoot '.cts' +$script:BaselineDir = Join-Path $script:CtsRoot 'baseline' +$script:LogsDir = Join-Path $script:CtsRoot 'logs' +$script:ConfigPath = Join-Path $PSScriptRoot 'cts.config.json' +$script:ProjectsPath= Join-Path $PSScriptRoot 'projects.json' + +function Get-Projects { + param([string]$Filter) + $all = Get-Content $script:ProjectsPath -Raw | ConvertFrom-Json + if (-not $Filter) { return $all } + $match = $all | Where-Object { $_.Key -ieq $Filter -or $_.CsProj -ieq $Filter } + if (-not $match) { + throw "Unknown -Project '$Filter'. Known keys: $(($all.Key) -join ', ')" + } + return @($match) +} + +function Ensure-Cli { + if (-not (Get-Command cts -ErrorAction SilentlyContinue)) { + throw "'cts' is not on PATH. Install with: dotnet tool install cts --global --prerelease --add-source https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json" + } + if (-not (Test-Path $script:DotnetExe)) { + throw "Local dotnet at '$script:DotnetExe' not found. Run .\build.cmd once to bootstrap." + } +} + +function Build-Project { + param([Parameter(Mandatory)] $Project) + $csproj = Join-Path $script:RepoRoot $Project.CsProj + Write-Host " build $($Project.Key) ..." -ForegroundColor DarkGray -NoNewline + $sw = [Diagnostics.Stopwatch]::StartNew() + $out = & $script:DotnetExe build -c Debug $csproj -f net10.0 -v:q -nologo 2>&1 + $sw.Stop() + if ($LASTEXITCODE -ne 0) { + Write-Host " FAIL ($([int]$sw.Elapsed.TotalSeconds)s)" -ForegroundColor Red + $out | Select-Object -Last 30 | ForEach-Object { Write-Host " $_" } + throw "Build of $($Project.CsProj) failed with exit $LASTEXITCODE" + } + Write-Host " ok ($([int]$sw.Elapsed.TotalSeconds)s)" -ForegroundColor DarkGreen +} + +function Get-ProjectDllPath { + param([Parameter(Mandatory)] $Project) + return (Join-Path $script:RepoRoot ("artifacts/bin/{0}/Debug/net10.0/{1}" -f $Project.BinDir, $Project.Dll)) +} + +function Get-ProjectDllFilter { + param([Parameter(Mandatory)] $Project) + # CTS --filter is a glob relative to --rootPath (repo root). + return "artifacts/bin/$($Project.BinDir)/Debug/net10.0/$($Project.Dll)" +} + +function Get-ProjectTag { + param([Parameter(Mandatory)] $Project) + return "local-$($Project.Key.ToLower())" +} + +function ConvertTo-ProcessArgumentList { + # Start-Process -ArgumentList joins array items with a single space and does + # NOT quote them, so any value containing whitespace (e.g. a repo path with + # spaces) is split into multiple arguments by the target process. Quote each + # token that needs it and escape embedded quotes so `cts` receives exactly + # the intended arguments. + param([Parameter(Mandatory)] [object[]]$Arguments) + $quoted = foreach ($a in $Arguments) { + $s = [string]$a + if ($s -eq '' -or $s -match '[\s"]') { + '"' + ($s -replace '"', '\"') + '"' + } + else { + $s + } + } + return ($quoted -join ' ') +} + +function Assert-CleanRepo { + Push-Location $script:RepoRoot + try { $dirty = git status --porcelain; $gitExit = $LASTEXITCODE } + finally { Pop-Location } + if ($gitExit -ne 0) { throw "git status failed (exit $gitExit). Is this a git repo with git on PATH?" } + if ($dirty) { + Write-Host "Working tree is dirty:" -ForegroundColor Red + $dirty | ForEach-Object { Write-Host " $_" } + throw "CTS collect requires a clean working tree. Commit/stash first." + } +} diff --git a/scripts/cts/cts.config.json b/scripts/cts/cts.config.json new file mode 100644 index 00000000000..2e78ac2c393 --- /dev/null +++ b/scripts/cts/cts.config.json @@ -0,0 +1,51 @@ +{ + "SourceCodeFiles": { + "Include": [ + "src/**/*.cs" + ], + "Exclude": [ + "**/obj/**", + "**/bin/**", + "**/TestAssets/**" + ] + }, + "Files": { + "Exclude": [ + "**/*.md", + "**/*.yml", + "**/*.yaml", + "**/*.png", + "documentation/**" + ] + }, + "Modules": { + "Include": [], + "Exclude": [ + "**/Microsoft.Testing.*.dll", + "**/Microsoft.TestPlatform*.dll", + "**/Microsoft.VisualStudio.TestPlatform*.dll", + "**/Microsoft.VisualStudio.CodeCoverage*.dll", + "**/Microsoft.DotNet.*.dll", + "**/xunit.*.dll", + "**/Xunit.*.dll", + "**/testhost*", + "**/Microsoft.Bcl.*.dll", + "**/Microsoft.ApplicationInsights.dll", + "**/Newtonsoft.Json.dll", + "**/Shouldly.dll", + "**/AwesomeAssertions.dll", + "**/FakeItEasy.dll", + "**/Verify.*.dll", + "**/System*.dll", + "**/runtimes/**" + ] + }, + "Filter": { + "Include": [ + "**/artifacts/bin/*.VSTest/Debug/net10.0/*.UnitTests.dll" + ], + "Exclude": [ + "**/*.resources.dll" + ] + } +} \ No newline at end of file diff --git a/scripts/cts/demos/Demo-BroadEdit.ps1 b/scripts/cts/demos/Demo-BroadEdit.ps1 new file mode 100644 index 00000000000..72237a4091d --- /dev/null +++ b/scripts/cts/demos/Demo-BroadEdit.ps1 @@ -0,0 +1,20 @@ +<# +.SYNOPSIS + Demo: touch a *broad* source file (used by most tests) and run CTS apply. + Expectation: most or all tests are selected. The edit is reverted on exit. + +.PARAMETER Project + Project key from projects.json. Default StringTools. +#> +[CmdletBinding()] +param([string]$Project = 'StringTools') + +. (Join-Path $PSScriptRoot '_DemoCommon.ps1') + +$proj = Resolve-DemoProject -Key $Project +$file = Get-DemoFile -Project $proj -Kind Broad + +Invoke-DemoApply -Project $proj ` + -Title "[$($proj.Key)] Apply after touching broad file: $($file.Relative)" ` + -TouchFile $file.Full ` + -ExpectMessage 'impacted ~= total (most/all selected)' diff --git a/scripts/cts/demos/Demo-NarrowEdit.ps1 b/scripts/cts/demos/Demo-NarrowEdit.ps1 new file mode 100644 index 00000000000..1d14827be2e --- /dev/null +++ b/scripts/cts/demos/Demo-NarrowEdit.ps1 @@ -0,0 +1,22 @@ +<# +.SYNOPSIS + Demo: touch a *narrow* source file (only some tests cover it) and run CTS + apply. + Expectation: partial selection (impacted < total). The edit is reverted + on exit. + +.PARAMETER Project + Project key from projects.json. Default StringTools. +#> +[CmdletBinding()] +param([string]$Project = 'StringTools') + +. (Join-Path $PSScriptRoot '_DemoCommon.ps1') + +$proj = Resolve-DemoProject -Key $Project +$file = Get-DemoFile -Project $proj -Kind Narrow + +Invoke-DemoApply -Project $proj ` + -Title "[$($proj.Key)] Apply after touching narrow file: $($file.Relative)" ` + -TouchFile $file.Full ` + -ExpectMessage 'impacted > 0 but < total (partial selection)' diff --git a/scripts/cts/demos/Demo-NoChange.ps1 b/scripts/cts/demos/Demo-NoChange.ps1 new file mode 100644 index 00000000000..dd2e7c52cf5 --- /dev/null +++ b/scripts/cts/demos/Demo-NoChange.ps1 @@ -0,0 +1,18 @@ +<# +.SYNOPSIS + Demo: run CTS apply with NO working-tree changes. + Expectation: 0 tests selected, 0 executed. + +.PARAMETER Project + Project key from projects.json. Default StringTools. +#> +[CmdletBinding()] +param([string]$Project = 'StringTools') + +. (Join-Path $PSScriptRoot '_DemoCommon.ps1') + +$proj = Resolve-DemoProject -Key $Project + +Invoke-DemoApply -Project $proj ` + -Title "[$($proj.Key)] Apply with NO working-tree changes" ` + -ExpectMessage 'impacted = 0, executed = 0' diff --git a/scripts/cts/demos/Demo-UnrelatedEdit.ps1 b/scripts/cts/demos/Demo-UnrelatedEdit.ps1 new file mode 100644 index 00000000000..21dc165fdc2 --- /dev/null +++ b/scripts/cts/demos/Demo-UnrelatedEdit.ps1 @@ -0,0 +1,21 @@ +<# +.SYNOPSIS + Demo: touch a source file that this project does NOT depend on and run + CTS apply. + Expectation: 0 tests selected. The edit is reverted on exit. + +.PARAMETER Project + Project key from projects.json. Default StringTools. +#> +[CmdletBinding()] +param([string]$Project = 'StringTools') + +. (Join-Path $PSScriptRoot '_DemoCommon.ps1') + +$proj = Resolve-DemoProject -Key $Project +$file = Get-DemoFile -Project $proj -Kind Unrelated + +Invoke-DemoApply -Project $proj ` + -Title "[$($proj.Key)] Apply after touching unrelated file: $($file.Relative)" ` + -TouchFile $file.Full ` + -ExpectMessage 'impacted = 0 (unrelated file does not affect this project)' diff --git a/scripts/cts/demos/_DemoCommon.ps1 b/scripts/cts/demos/_DemoCommon.ps1 new file mode 100644 index 00000000000..e43d4107790 --- /dev/null +++ b/scripts/cts/demos/_DemoCommon.ps1 @@ -0,0 +1,68 @@ +# scripts/cts/demos/_DemoCommon.ps1 +# Shared helpers for the demo scripts. Keep tiny. + +. (Join-Path $PSScriptRoot '../_Common.ps1') + +function Resolve-DemoProject { + param([Parameter(Mandatory)] [string]$Key) + $proj = (Get-Projects -Filter $Key)[0] + if (-not (Test-Path $script:BaselineDir) -or + -not (Get-ChildItem -Path $script:BaselineDir -Recurse -ErrorAction SilentlyContinue)) { + throw "No baseline at $script:BaselineDir. Run .\Collect-Local.ps1 -Project $Key first." + } + return $proj +} + +function Get-DemoFile { + param( + [Parameter(Mandatory)] $Project, + [Parameter(Mandatory)] [ValidateSet('Broad','Narrow','Unrelated')] [string]$Kind + ) + if (-not $Project.DemoFiles) { + throw "Project '$($Project.Key)' has no DemoFiles defined in projects.json." + } + $rel = $Project.DemoFiles.$Kind + if (-not $rel) { + throw "Project '$($Project.Key)' has no DemoFiles.$Kind defined in projects.json." + } + $full = Join-Path $script:RepoRoot $rel + if (-not (Test-Path $full)) { + throw "DemoFile '$rel' does not exist in the repo." + } + return [pscustomobject]@{ Relative = $rel; Full = $full } +} + +function Invoke-DemoApply { + param( + [Parameter(Mandatory)] $Project, + [Parameter(Mandatory)] [string]$Title, + [string]$TouchFile, + [string]$ExpectMessage + ) + + Write-Host '' + Write-Host ('-' * 72) -ForegroundColor Cyan + Write-Host " $Title" -ForegroundColor Cyan + Write-Host ('-' * 72) -ForegroundColor Cyan + if ($ExpectMessage) { Write-Host " expected: $ExpectMessage" -ForegroundColor DarkGray } + + if ($TouchFile) { + Add-Content -Path $TouchFile -Value '' + Write-Host " + appended one blank line to $([IO.Path]::GetRelativePath($script:RepoRoot, $TouchFile))" -ForegroundColor Yellow + } + + try { + & (Join-Path $PSScriptRoot '../Run-Local.ps1') -Project $Project.Key -SkipBuild -TimeoutMinutes 5 | Out-Null + $log = Join-Path $script:LogsDir "apply-$($Project.Key).log" + Get-Content $log -Tail 30 | + Where-Object { $_ -match 'impacted test\(s\):|executed:|succeeded:|failed:|reason ' } | + ForEach-Object { Write-Host " $_" -ForegroundColor Green } + } + finally { + if ($TouchFile) { + Push-Location $script:RepoRoot + try { git checkout -- $TouchFile | Out-Null } + finally { Pop-Location } + } + } +} diff --git a/scripts/cts/projects.json b/scripts/cts/projects.json new file mode 100644 index 00000000000..fbb6f2c068a --- /dev/null +++ b/scripts/cts/projects.json @@ -0,0 +1,65 @@ +[ + { + "Key": "StringTools", + "CsProj": "src/StringTools.UnitTests/StringTools.UnitTests.VSTest.csproj", + "Dll": "Microsoft.NET.StringTools.UnitTests.dll", + "BinDir": "StringTools.UnitTests.VSTest", + "DemoFiles": { + "Broad": "src/StringTools/StringTools.cs", + "Narrow": "src/StringTools/WeakStringCache.cs", + "Unrelated": "src/Tasks/AssemblyDependency/Resolver.cs" + } + }, + { + "Key": "Framework", + "CsProj": "src/Framework.UnitTests/Microsoft.Build.Framework.UnitTests.VSTest.csproj", + "Dll": "Microsoft.Build.Framework.UnitTests.dll", + "BinDir": "Microsoft.Build.Framework.UnitTests.VSTest", + "DemoFiles": { + "Broad": "src/Framework/BuildEventArgs.cs", + "Narrow": "src/Framework/ProjectStartedEventArgs.cs", + "Unrelated": "src/Tasks/AssemblyDependency/Resolver.cs" + } + }, + { + "Key": "Utilities", + "CsProj": "src/Utilities.UnitTests/Microsoft.Build.Utilities.UnitTests.VSTest.csproj", + "Dll": "Microsoft.Build.Utilities.UnitTests.dll", + "BinDir": "Microsoft.Build.Utilities.UnitTests.VSTest", + "DemoFiles": { + "Broad": "src/Utilities/Logger.cs", + "Narrow": "src/Utilities/CommandLineBuilder.cs", + "Unrelated": "src/Tasks/AssemblyDependency/Resolver.cs" + } + }, + { + "Key": "EngineOM", + "CsProj": "src/Build.OM.UnitTests/Microsoft.Build.Engine.OM.UnitTests.VSTest.csproj", + "Dll": "Microsoft.Build.Engine.OM.UnitTests.dll", + "BinDir": "Microsoft.Build.Engine.OM.UnitTests.VSTest" + }, + { + "Key": "CommandLine", + "CsProj": "src/MSBuild.UnitTests/Microsoft.Build.CommandLine.UnitTests.VSTest.csproj", + "Dll": "Microsoft.Build.CommandLine.UnitTests.dll", + "BinDir": "Microsoft.Build.CommandLine.UnitTests.VSTest" + }, + { + "Key": "Engine", + "CsProj": "src/Build.UnitTests/Microsoft.Build.Engine.UnitTests.VSTest.csproj", + "Dll": "Microsoft.Build.Engine.UnitTests.dll", + "BinDir": "Microsoft.Build.Engine.UnitTests.VSTest" + }, + { + "Key": "Tasks", + "CsProj": "src/Tasks.UnitTests/Microsoft.Build.Tasks.UnitTests.VSTest.csproj", + "Dll": "Microsoft.Build.Tasks.UnitTests.dll", + "BinDir": "Microsoft.Build.Tasks.UnitTests.VSTest" + }, + { + "Key": "BuildCheck", + "CsProj": "src/BuildCheck.UnitTests/Microsoft.Build.BuildCheck.UnitTests.VSTest.csproj", + "Dll": "Microsoft.Build.BuildCheck.UnitTests.dll", + "BinDir": "Microsoft.Build.BuildCheck.UnitTests.VSTest" + } +] diff --git a/src/Build.OM.UnitTests/Microsoft.Build.Engine.OM.UnitTests.VSTest.csproj b/src/Build.OM.UnitTests/Microsoft.Build.Engine.OM.UnitTests.VSTest.csproj new file mode 100644 index 00000000000..81a60d6f76f --- /dev/null +++ b/src/Build.OM.UnitTests/Microsoft.Build.Engine.OM.UnitTests.VSTest.csproj @@ -0,0 +1,32 @@ + + + + + + $(MSBuildProjectName.Replace('.VSTest','')) + + + true + true + Library + false + + + true + + + + + + net10.0 + + + \ No newline at end of file diff --git a/src/Build.UnitTests/Microsoft.Build.Engine.UnitTests.VSTest.csproj b/src/Build.UnitTests/Microsoft.Build.Engine.UnitTests.VSTest.csproj new file mode 100644 index 00000000000..261082feabd --- /dev/null +++ b/src/Build.UnitTests/Microsoft.Build.Engine.UnitTests.VSTest.csproj @@ -0,0 +1,32 @@ + + + + + + $(MSBuildProjectName.Replace('.VSTest','')) + + + true + true + Library + false + + + true + + + + + + net10.0 + + + \ No newline at end of file diff --git a/src/BuildCheck.UnitTests/Microsoft.Build.BuildCheck.UnitTests.VSTest.csproj b/src/BuildCheck.UnitTests/Microsoft.Build.BuildCheck.UnitTests.VSTest.csproj new file mode 100644 index 00000000000..d8d55f4b44d --- /dev/null +++ b/src/BuildCheck.UnitTests/Microsoft.Build.BuildCheck.UnitTests.VSTest.csproj @@ -0,0 +1,32 @@ + + + + + + $(MSBuildProjectName.Replace('.VSTest','')) + + + true + true + Library + false + + + true + + + + + + net10.0 + + + \ No newline at end of file diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets index e18cb35f1bd..ee9a6364ca9 100644 --- a/src/Directory.Build.targets +++ b/src/Directory.Build.targets @@ -64,13 +64,27 @@ $(XUnitDesktopSettingsFile) + + + + + - + diff --git a/src/Framework.UnitTests/Microsoft.Build.Framework.UnitTests.VSTest.csproj b/src/Framework.UnitTests/Microsoft.Build.Framework.UnitTests.VSTest.csproj new file mode 100644 index 00000000000..4ac0f40943b --- /dev/null +++ b/src/Framework.UnitTests/Microsoft.Build.Framework.UnitTests.VSTest.csproj @@ -0,0 +1,32 @@ + + + + + + $(MSBuildProjectName.Replace('.VSTest','')) + + + true + true + Library + false + + + true + + + + + + net10.0 + + + \ No newline at end of file diff --git a/src/MSBuild.UnitTests/Microsoft.Build.CommandLine.UnitTests.VSTest.csproj b/src/MSBuild.UnitTests/Microsoft.Build.CommandLine.UnitTests.VSTest.csproj new file mode 100644 index 00000000000..c0afb02ae2f --- /dev/null +++ b/src/MSBuild.UnitTests/Microsoft.Build.CommandLine.UnitTests.VSTest.csproj @@ -0,0 +1,32 @@ + + + + + + $(MSBuildProjectName.Replace('.VSTest','')) + + + true + true + Library + false + + + true + + + + + + net10.0 + + + \ No newline at end of file diff --git a/src/StringTools.UnitTests/StringTools.UnitTests.VSTest.csproj b/src/StringTools.UnitTests/StringTools.UnitTests.VSTest.csproj new file mode 100644 index 00000000000..3460a56240b --- /dev/null +++ b/src/StringTools.UnitTests/StringTools.UnitTests.VSTest.csproj @@ -0,0 +1,33 @@ + + + + + + $(MSBuildProjectName.Replace('.VSTest','')) + + + true + true + Library + false + + + true + + + + + + + net10.0 + + + \ No newline at end of file diff --git a/src/Tasks.UnitTests/Microsoft.Build.Tasks.UnitTests.VSTest.csproj b/src/Tasks.UnitTests/Microsoft.Build.Tasks.UnitTests.VSTest.csproj new file mode 100644 index 00000000000..a60ab7a3e2b --- /dev/null +++ b/src/Tasks.UnitTests/Microsoft.Build.Tasks.UnitTests.VSTest.csproj @@ -0,0 +1,32 @@ + + + + + + $(MSBuildProjectName.Replace('.VSTest','')) + + + true + true + Library + false + + + true + + + + + + net10.0 + + + \ No newline at end of file diff --git a/src/UnitTests.Shared/TestEnvironment.cs b/src/UnitTests.Shared/TestEnvironment.cs index d1b68e4de72..79fb3c82c53 100644 --- a/src/UnitTests.Shared/TestEnvironment.cs +++ b/src/UnitTests.Shared/TestEnvironment.cs @@ -486,7 +486,10 @@ void AssertDictionaryInclusion(IDictionary superset, IDictionary subset, string { foreach (var key in subset.Keys) { - if (key is "_MSBUILDTLENABLED") + if (key is "_MSBUILDTLENABLED" + or "MSBuildLoadMicrosoftTargetsReadOnly" + or "MSBUILDLOADALLFILESASWRITEABLE" + || IsInstrumentationEnvironmentVariable((string)key)) { continue; } @@ -500,6 +503,19 @@ void AssertDictionaryInclusion(IDictionary superset, IDictionary subset, string } } } + + // Skip env vars injected by .NET profiler-based instrumentation tools (e.g. Clever Test Selection, + // code coverage). These are set by the host process for the test runner but are not present in the + // parent process, so they appear as "added" entries that would otherwise fail this invariant. + static bool IsInstrumentationEnvironmentVariable(string key) => + key.StartsWith("CORECLR_PROFILER", StringComparison.OrdinalIgnoreCase) || + key.StartsWith("COR_PROFILER", StringComparison.OrdinalIgnoreCase) || + key.StartsWith("CORECLR_ENABLE_PROFILING", StringComparison.OrdinalIgnoreCase) || + key.StartsWith("COR_ENABLE_PROFILING", StringComparison.OrdinalIgnoreCase) || + key.StartsWith("MicrosoftInstrumentationEngine_", StringComparison.OrdinalIgnoreCase) || + key.StartsWith("MicrosoftCodeCoverage", StringComparison.OrdinalIgnoreCase) || + key.StartsWith("CleverTestsSelection", StringComparison.OrdinalIgnoreCase) || + key.StartsWith("CTS_", StringComparison.OrdinalIgnoreCase); } } diff --git a/src/Utilities.UnitTests/Microsoft.Build.Utilities.UnitTests.VSTest.csproj b/src/Utilities.UnitTests/Microsoft.Build.Utilities.UnitTests.VSTest.csproj new file mode 100644 index 00000000000..15ff891eae2 --- /dev/null +++ b/src/Utilities.UnitTests/Microsoft.Build.Utilities.UnitTests.VSTest.csproj @@ -0,0 +1,32 @@ + + + + + + $(MSBuildProjectName.Replace('.VSTest','')) + + + true + true + Library + false + + + true + + + + + + net10.0 + + + \ No newline at end of file