diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index d27be338048..36167db3bfd 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -14,6 +14,11 @@ runs: 9.x 8.x + - uses: actions/setup-dotnet@v5 + if: env.DOCFX_PREVIEW_BUILD == 'true' + with: + dotnet-version: 11.0.100-rc.1.26425.128 + - name: Install fonts-noto-color-emoji package (Linux only) if: runner.os == 'Linux' shell: bash diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 840526544c8..827e87c6166 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -42,6 +42,9 @@ jobs: - name: dotnet test run: dotnet test -c Release -f net10.0 --no-build + + - name: dotnet test + run: dotnet test -c Release -f net11.0 --no-build - name: dotnet pack run: dotnet pack -c Release /p:Version=${{ steps.version.outputs.version }} /p:ApiCompatGenerateSuppressionFile=true -o drop/nuget @@ -149,7 +152,7 @@ jobs: GH_TOKEN: ${{ github.token }} TOOL_VERSION: ${{ needs.build-nightly-package.outputs.version }} run: | - ./test/compatibility/Measure-Compatibility.ps1 -MatrixPath drop/compatibility-matrix.json -OutputDirectory drop/compatibility-report -NightlyPackage "drop/nightly/docfx.$env:TOOL_VERSION.nupkg" + ./test/compatibility/Measure-Compatibility.ps1 -MatrixPath drop/compatibility-matrix.json -OutputDirectory drop/compatibility-report -NightlyPackage "drop/nightly/docfx.$env:TOOL_VERSION.nupkg" -NightlyFrameworks net8.0,net9.0,net10.0,net11.0 - name: Upload compatibility evidence, including failed cases if: always() uses: actions/upload-artifact@v7 diff --git a/Directory.Build.props b/Directory.Build.props index e2ae7061d6b..31d070b2d02 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,7 +2,7 @@ true net8.0;net9.0;net10.0 - + net8.0;net9.0;net10.0;net11.0 Preview enable true diff --git a/Directory.Packages.props b/Directory.Packages.props index 9dc17b22ea3..ccf90de4970 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -28,25 +28,26 @@ - - - - - - - - - - + + [5.6.0] + 5.9.0 + + 5.11.0-1.26425.128 + + + + + $(RestoreAdditionalProjectSources);https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + - - - - - - - - - + + + + + + + + + diff --git a/docs/docs/sdk-compatibility-maintenance.md b/docs/docs/sdk-compatibility-maintenance.md index bdfcda56964..13927eadc1a 100644 --- a/docs/docs/sdk-compatibility-maintenance.md +++ b/docs/docs/sdk-compatibility-maintenance.md @@ -10,13 +10,13 @@ This guide covers the measurement and publication pipeline behind the [SDK compa The nightly workflow builds the checked-out main commit, packs it, and tests **that exact package artifact**, verifying its version and package hash rather than installing whichever prerelease a feed returns. A separate released-package group discovers the latest non-prerelease version from the official GitHub release metadata, then installs and verifies that exact version using the configured package feed. Discovery or installation failure is an **infrastructure-error**; there is no fallback to an older release. GitHub release metadata is used only for version discovery, not package downloads. -Every case records the selected project SDK, project target framework, package version and hash, tool runtime TFM, OS, scenario, and timestamp. The report records the workflow's source commit and stable-version selection. Runtime TFM describes the packaged tool, not the project's framework or the exact installed runtime patch. +Every case records the selected project SDK, project target framework, package version and hash, tool runtime TFM, OS, scenario, and timestamp. The report records the workflow's source commit and stable-version selection. Runtime TFM describes the packaged tool, not the project's framework or the exact installed runtime patch. Schema 2 also records the requested tool targets, so failed installations retain their intended identity. Earlier schema 1 reports remain readable with only their originally measured coverage. -Both DocFX package groups use the same Linux matrix: the latest SDK from the explicitly configured 8.0, 9.0, 10.0, and 11.0 channels, resolved from Microsoft's official release metadata. Latest stable and current preview refer to **DocFX packages**, not SDK channels. The five SDK/project-TFM pairs are 8.0/`net8.0`, 9.0/`net9.0`, 10.0/`net10.0`, 11.0/`net10.0`, and 11.0/`net11.0`. With two packages and two scenarios, this configures **20 cases per run**. The same exact 11.0 SDK tests both `net10.0` for older-project regression coverage and `net11.0` for new-target coverage; their results and logs remain separate. This is configured coverage, not evidence that unmeasured combinations passed. +Both DocFX package groups use the same Linux matrix: the latest SDK from the explicitly configured 8.0, 9.0, 10.0, and 11.0 channels, resolved from Microsoft's official release metadata. Latest stable and current preview refer to **DocFX packages**, not SDK channels. The five SDK/project-TFM pairs are 8.0/`net8.0`, 9.0/`net9.0`, 10.0/`net10.0`, 11.0/`net10.0`, and 11.0/`net11.0`. Each pair runs both scenarios against the released net10.0 tool and the nightly net8.0/net9.0/net10.0/net11.0 tools: **50 cases per run** (10 released and 40 nightly). The same exact 11.0 SDK tests both `net10.0` for older-project regression coverage and `net11.0` for new-target coverage; their results and logs remain separate. This is configured coverage, not evidence that unmeasured combinations passed. -Each SDK is installed once, even when it tests multiple project TFMs. Each case pins `global.json` with roll-forward disabled and verifies `dotnet --version`. DocFX still runs from the package's `net10.0` assets regardless of project TFM; other tool TFMs and operating systems are not covered by this matrix. +Each SDK is installed once, even when it tests multiple project TFMs. Each case pins `global.json` with roll-forward disabled and verifies `dotnet --version`. The released package uses its `net10.0` tool assets. Each of the nightly package's net8.0/net9.0/net10.0/net11.0 tool targets runs the complete SDK/project matrix, with isolated installations, work directories, and logs. The tool target is recorded in every result and displayed in the column headers; it is independent of the project target. Other operating systems are not covered by this matrix. The strict PR smoke continues to exercise the net10.0 tool. -DocFX bundles a tested Roslyn dependency set. Selecting a newer SDK does not replace those assemblies. SDK source generators can require a newer compiler even when a project targets an older framework. This report measures that interaction; it does not change compiler loading or prescribe a workaround. +DocFX bundles a tested Roslyn dependency set for each tool target. Normal builds contain net8.0/net9.0/net10.0 assets with stable compiler dependencies. `DOCFX_PREVIEW_BUILD=true` also builds net11.0 assets with the preview compiler. Selecting a newer SDK does not replace the compiler bundled in an older tool target. SDK source generators can require a newer compiler even when a project targets an older framework. This report measures that interaction; it does not change compiler loading or prescribe a workaround. ## Scenarios and outcomes @@ -50,7 +50,7 @@ Main-branch CI publication and completed nightlies use the same whole-site publi ## Validating a workflow change -Manual runs of `nightly.yml` default to **validation only**. Select the branch to test and leave `validation_only` enabled. The workflow builds that checkout, runs the existing tests on all three tool target frameworks, packs it, and downloads the same versioned package for seed-site checks and the complete SDK compatibility matrix. Both this exact branch package and the latest stable release are tested against the same SDK matrix. This mode only restricts publication: it cannot publish NuGet packages or deploy Pages. It uploads the exact package and **sdk-compatibility-validation-v1** report/log artifact for inspection, retained for 14 days. Incompatible observations do not block the run; infrastructure failures still fail and retain available evidence. +Manual runs of `nightly.yml` default to **validation only**. Select the branch to test and leave `validation_only` enabled. The workflow builds that checkout, runs the existing tests on all four tool target frameworks, packs it, and downloads the same versioned package for seed-site checks and the complete SDK compatibility matrix. Both this exact branch package and the latest stable release are tested against the same SDK matrix. This mode only restricts publication: it cannot publish NuGet packages or deploy Pages. It uploads the exact package and **sdk-compatibility-validation-v1** report/log artifact for inspection, retained for 14 days. Incompatible observations do not block the run; infrastructure failures still fail and retain available evidence. Scheduled main runs keep the normal released-package comparison and publication behavior. A manual production run requires explicitly disabling `validation_only` and selecting upstream main. The website only refreshes for nightlies carrying the separate production report artifact; validation artifacts cannot authorize publication, even when validation runs on main. @@ -62,12 +62,12 @@ From the repository root, resolve the matrix and install the SDKs you intend to ```powershell node test/compatibility/report.mjs resolve drop/compatibility/matrix.json -./test/compatibility/Measure-Compatibility.ps1 -MatrixPath drop/compatibility/matrix.json -OutputDirectory drop/compatibility/report -NightlyPackage path/to/exact/docfx.nupkg +./test/compatibility/Measure-Compatibility.ps1 -MatrixPath drop/compatibility/matrix.json -OutputDirectory drop/compatibility/report -NightlyPackage path/to/exact/docfx.nupkg -NightlyFrameworks net8.0,net9.0,net10.0,net11.0 node test/compatibility/report.mjs prepare drop/compatibility/report/compatibility-report.json docs/obj/sdk-compatibility.json docfx docs/docfx.json ``` -Omit `-NightlyPackage` for released-package-only local measurements, or use `-SkipStable` for a bounded packaged-tool smoke. A report without a nightly package does **not** validate current main. For a deliberately older local comparison, supply both `-StableVersion` and `-StableVersionReason`; this is labeled an explicit local version, never the latest stable release, and is not allowed in GitHub Actions. `-NuGetConfig` applies an explicit feed configuration to tool installation and fixture restores; reuse your organization's approved feeds. `-WorkDirectory` retains isolated package/fixture files for inspection and must name a new directory. The harness validates the completed report and explicitly returns its check result, rather than inheriting the last fixture's native exit code. By default, incompatible observations produce warnings without failing; unavailable cases, infrastructure errors, and invalid or incomplete reports always fail **after** writing available evidence. `-FailOnIncompatible` selects strict mode for PR smoke or local gates. +Omit `-NightlyPackage` for released-package-only local measurements, or use `-SkipStable` for a bounded packaged-tool smoke. `-NightlyFrameworks` defaults to `net10.0` for bounded smoke tests; pass all four targets as above for nightly coverage. Each requested target must exist in the supplied package, and its runtime must be installed. The released comparison always uses net10.0. A report without a nightly package does **not** validate current main. For a deliberately older local comparison, supply both `-StableVersion` and `-StableVersionReason`; this is labeled an explicit local version, never the latest stable release, and is not allowed in GitHub Actions. `-NuGetConfig` applies an explicit feed configuration to tool installation and fixture restores; reuse your organization's approved feeds. `-WorkDirectory` retains isolated package/fixture files for inspection and must name a new directory. The harness validates the completed report and explicitly returns its check result, rather than inheriting the last fixture's native exit code. By default, incompatible observations produce warnings without failing; unavailable cases, infrastructure errors, and invalid or incomplete reports always fail **after** writing available evidence. `-FailOnIncompatible` selects strict mode for PR smoke or local gates. To recheck a saved report without rerunning measurements, use `node test/compatibility/report.mjs check path/to/compatibility-report.json`. Add `--strict` to require every row to pass. Neither mode rewrites observations or package provenance. diff --git a/docs/template/public/sdk-compatibility.mjs b/docs/template/public/sdk-compatibility.mjs index 3860387b8ff..95ab2912bca 100644 --- a/docs/template/public/sdk-compatibility.mjs +++ b/docs/template/public/sdk-compatibility.mjs @@ -10,7 +10,7 @@ const text = (value, max = 2000) => typeof value === 'string' && value.length > const date = value => typeof value === 'string' && /^\d{4}-\d\d-\d\dT/.test(value) && Number.isFinite(Date.parse(value)) export function validateReport(report, expected, now = Date.now()) { - assert(report?.schemaVersion === 1, 'Unsupported report schema.') + assert([1, 2].includes(report?.schemaVersion), 'Unsupported report schema.') assert(date(report.generatedAt) && Date.parse(report.generatedAt) <= now + 300000, 'Invalid report timestamp.') const source = report.source assert(source && (source.repository === 'local' || /^[a-zA-Z0-9][a-zA-Z0-9-]{0,38}\/[a-zA-Z0-9][a-zA-Z0-9_.-]{0,99}$/.test(source.repository)) && /^[a-f0-9]{40}$/.test(source.sha), 'Invalid report source.') @@ -37,11 +37,20 @@ export function validateReport(report, expected, now = Date.now()) { if (expected) assert(selection.mode === 'latest-release', 'Nightly must resolve the latest stable release.') } else { assert(report.stableSelection === null, 'Unexpected stable release selection.') } assert(JSON.stringify(report.scenarios) === '["basic","razor"]', 'Missing fixture scenarios.') - const wanted = new Set(targets.flatMap(t => report.channels.flatMap(c => report.scenarios.map(s => `${t}/${c}/${s}`)))) + const multipleTools = report.schemaVersion === 2 + const toolTargets = multipleTools ? report.toolTargets : report.channels.map(channel => ({ channel, framework: null })) + assert(Array.isArray(toolTargets) && toolTargets.length > 0 && toolTargets.length <= 8, 'Invalid tool targets.') + const toolKeys = toolTargets.map(target => { + assert(target && report.channels.includes(target.channel) && (!multipleTools || tfmPattern.test(target.framework)), 'Invalid tool target.') + return multipleTools ? `${target.channel}/${target.framework}` : target.channel + }) + assert(new Set(toolKeys).size === toolKeys.length && report.channels.every(channel => toolTargets.some(t => t.channel === channel)), 'Duplicate or missing tool target.') + const wanted = new Set(targets.flatMap(t => toolKeys.flatMap(tool => report.scenarios.map(s => `${t}/${tool}/${s}`)))) assert(Array.isArray(report.results) && report.results.length === wanted.size, 'Incomplete report.') const packageIdentities = new Map() for (const row of report.results) { - const key = `${row.sdk}/${row.projectTfm}/${row.channel}/${row.scenario}` + const toolKey = multipleTools ? `${row.channel}/${row.toolFramework}` : row.channel + const key = `${row.sdk}/${row.projectTfm}/${toolKey}/${row.scenario}` assert(wanted.delete(key), 'Unexpected or duplicate result.') assert(outcomes.includes(row.outcome), 'Unknown outcome.') assert(text(row.os, 300) && text(row.diagnostics) && /^logs\/[a-zA-Z0-9.-]+\.log$/.test(row.log), 'Invalid diagnostic evidence.') @@ -52,11 +61,12 @@ export function validateReport(report, expected, now = Date.now()) { assert(row.packageSha256 === null || /^[a-f0-9]{64}$/.test(row.packageSha256), 'Invalid package hash.') if (['passed', 'incompatible'].includes(row.outcome)) { assert(row.selectedSdk === row.sdk && row.toolVersion && row.toolRuntimeTfm && row.packageSha256, 'Measured result is missing SDK/package identity.') + if (multipleTools) assert(row.toolRuntimeTfm === row.toolFramework, 'Measured tool runtime differs from the requested framework.') if (row.channel === 'stable') assert(row.toolVersion === report.stableSelection.requestedVersion, 'Measured package differs from the requested stable release.') } if (row.channel === 'stable' && row.toolVersion) assert(!row.toolVersion.includes('-'), 'Stable channel contains a prerelease.') if (row.packageSha256) { - const identity = `${row.toolVersion}/${row.toolRuntimeTfm}/${row.packageSha256}` + const identity = multipleTools ? `${row.toolVersion}/${row.packageSha256}` : `${row.toolVersion}/${row.toolRuntimeTfm}/${row.packageSha256}` assert(!packageIdentities.has(row.channel) || packageIdentities.get(row.channel) === identity, 'Package identity changes within a channel.') packageIdentities.set(row.channel, identity) } @@ -68,9 +78,10 @@ const escape = value => String(value ?? 'Not measured').replace(/[&<>"']/g, c => const reportUrl = new URL('../reports/sdk-compatibility.json', import.meta.url) const label = value => value[0].toUpperCase() + value.slice(1) const statusLabels = { passed: '✓ Passed', incompatible: '⚠ Incompatible', unavailable: '— Unavailable', 'infrastructure-error': '! Infrastructure error' } -const columnKey = row => JSON.stringify([row.channel, row.toolVersion, row.toolRuntimeTfm, row.scenario]) +const toolFramework = row => row.toolFramework ?? row.toolRuntimeTfm +const columnKey = row => JSON.stringify([row.channel, row.toolVersion, toolFramework(row), row.scenario]) const caseTitle = row => `${label(row.channel)} & ${label(row.scenario)}` -const caseIdentity = row => `${caseTitle(row)}; SDK ${row.sdk}; project ${row.projectTfm}; DocFX ${row.toolVersion ?? 'not measured'}; tool target ${row.toolRuntimeTfm ?? 'not measured'}` +const caseIdentity = row => `${caseTitle(row)}; SDK ${row.sdk}; project ${row.projectTfm}; DocFX ${row.toolVersion ?? 'not measured'}; tool target ${toolFramework(row) ?? 'not measured'}` const runLink = source => source.repository === 'local' ? null : `https://github.com/${source.repository}/actions/runs/${source.runId}/attempts/${source.runAttempt}` export function renderReport(report, now = Date.now()) { @@ -85,7 +96,7 @@ export function renderReport(report, now = Date.now()) { const columns = [...new Map(report.results.map(row => [columnKey(row), row])).values()].sort((a, b) => ['stable', 'nightly'].indexOf(a.channel) - ['stable', 'nightly'].indexOf(b.channel) || String(a.toolVersion ?? '~').localeCompare(String(b.toolVersion ?? '~'), 'en', { numeric: true }) || - String(a.toolRuntimeTfm ?? '~').localeCompare(String(b.toolRuntimeTfm ?? '~'), 'en', { numeric: true }) || + String(toolFramework(a) ?? '~').localeCompare(String(toolFramework(b) ?? '~'), 'en', { numeric: true }) || report.scenarios.indexOf(a.scenario) - report.scenarios.indexOf(b.scenario)) const cells = new Map(report.results.map((row, index) => [`${row.sdk}/${row.projectTfm}/${columnKey(row)}`, index])) let html = `

Last tested

` @@ -94,7 +105,7 @@ export function renderReport(report, now = Date.now()) { if (!report.channels.includes('nightly')) html += '

Current-main package not measured in this report.

' if (report.stableSelection?.mode === 'explicit-version') html += `

Explicit local version: ${escape(report.stableSelection.requestedVersion)} — not a measurement of the latest stable release. ${escape(report.stableSelection.reason)}

` html += '
' - for (const column of columns) html += `` + for (const column of columns) html += `` html += '' for (const target of report.matrix) { html += `` @@ -205,7 +216,9 @@ function bindDetails(element, report, fetchLog) { identity.replaceChildren() for (const [name, value] of [ ['Requested SDK', row.sdk], ['Selected SDK', row.selectedSdk], ['Project target', row.projectTfm], - ['DocFX version', row.toolVersion], ['Tool target framework', row.toolRuntimeTfm], ['OS', row.os], + ['DocFX version', row.toolVersion], + ...(row.toolFramework ? [['Requested tool target', row.toolFramework]] : []), + ['Tool target framework', row.toolRuntimeTfm], ['OS', row.os], ['Tested', row.testedAt], ['Package SHA-256', row.packageSha256], ]) { const term = element.ownerDocument.createElement('dt') diff --git a/src/Docfx.Dotnet/SymbolFormatter.Symbols.cs b/src/Docfx.Dotnet/SymbolFormatter.Symbols.cs index a918eea53ed..13217d70091 100644 --- a/src/Docfx.Dotnet/SymbolFormatter.Symbols.cs +++ b/src/Docfx.Dotnet/SymbolFormatter.Symbols.cs @@ -82,6 +82,10 @@ private class ParameterSymbol : IParameterSymbol public bool IsExtern => false; +#if NET11_0_OR_GREATER + public bool RequiresUnsafeContext => false; +#endif + public bool IsImplicitlyDeclared => false; public bool CanBeReferencedByName => false; @@ -186,6 +190,12 @@ private class PropertySymbol : IPropertySymbol public bool IsExtern => Inner.IsExtern; +#if NET11_0_OR_GREATER +#pragma warning disable RSEXPERIMENTAL006 // Required by Roslyn 5.11's ISymbol interface. + public bool RequiresUnsafeContext => Inner.RequiresUnsafeContext; +#pragma warning restore RSEXPERIMENTAL006 +#endif + public bool IsImplicitlyDeclared => Inner.IsImplicitlyDeclared; public bool CanBeReferencedByName => Inner.CanBeReferencedByName; @@ -335,6 +345,12 @@ public class MethodSymbol : IMethodSymbol public bool IsExtern => Inner.IsExtern; +#if NET11_0_OR_GREATER +#pragma warning disable RSEXPERIMENTAL006 // Required by Roslyn 5.11's ISymbol interface. + public bool RequiresUnsafeContext => Inner.RequiresUnsafeContext; +#pragma warning restore RSEXPERIMENTAL006 +#endif + public bool IsImplicitlyDeclared => Inner.IsImplicitlyDeclared; public bool CanBeReferencedByName => Inner.CanBeReferencedByName; diff --git a/test/compatibility/Measure-Compatibility.ps1 b/test/compatibility/Measure-Compatibility.ps1 index a63851b4d18..39f41bc39a4 100644 --- a/test/compatibility/Measure-Compatibility.ps1 +++ b/test/compatibility/Measure-Compatibility.ps1 @@ -6,6 +6,7 @@ param( [Parameter(Mandatory)][string] $MatrixPath, [Parameter(Mandatory)][string] $OutputDirectory, [string] $NightlyPackage, + [ValidateSet('net8.0', 'net9.0', 'net10.0', 'net11.0')][string[]] $NightlyFrameworks = @('net10.0'), [string] $NuGetConfig, [string] $StableVersion, [string] $StableVersionReason, @@ -17,6 +18,7 @@ param( $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $false if ($StableVersion -and ($StableVersion -notmatch '^\d+\.\d+\.\d+$' -or -not $StableVersionReason -or $SkipStable -or $env:GITHUB_ACTIONS -eq 'true')) { throw 'An explicit stable version requires a reason and is allowed only for local released-package measurements.' } +if (@($NightlyFrameworks | Select-Object -Unique).Count -ne $NightlyFrameworks.Count) { throw 'Duplicate nightly tool frameworks.' } $matrix = @(Get-Content -LiteralPath $MatrixPath -Raw | ConvertFrom-Json) $OutputDirectory = [IO.Path]::GetFullPath($OutputDirectory) if ((Test-Path -LiteralPath $OutputDirectory) -and @(Get-ChildItem -LiteralPath $OutputDirectory -Force).Count -gt 0) { throw 'Output directory must be empty to prevent mixing evidence from different runs.' } @@ -53,12 +55,13 @@ function Invoke-Logged([string[]] $Arguments, [string] $Log) { return @{ code = $code; text = $output } } -function Install-Tool([string] $Channel, [string] $Package) { - $tool = @{ channel = $Channel; version = $null; runtimeTfm = $null; packageSha256 = $null; dll = $null; error = $null; selection = $null } - $log = Join-Path $logs "$Channel-install.log" +function Install-Tool([string] $Channel, [string] $Package, [string] $Framework) { + $tool = @{ channel = $Channel; framework = $Framework; version = $null; runtimeTfm = $null; packageSha256 = $null; dll = $null; error = $null; selection = $null } + $identity = "$Channel-$Framework" + $log = Join-Path $logs "$identity-install.log" try { $expectedVersion = $null - $install = @('tool', 'install', 'docfx', '--framework', 'net10.0', '--tool-path', (Join-Path $work $Channel)) + $install = @('tool', 'install', 'docfx', '--framework', $Framework, '--tool-path', (Join-Path $work $identity)) if ($Channel -eq 'stable') { $tool.selection = @{ mode = 'latest-release'; requestedVersion = $null; reason = 'Latest stable GitHub release; no older-version fallback.' } if ($StableVersion) { @@ -83,7 +86,7 @@ function Install-Tool([string] $Channel, [string] $Package) { if ($manifest.package.metadata.id -ne 'docfx') { throw 'Not a DocFX tool package.' } $expectedVersion = [string] $manifest.package.metadata.version } finally { $archive.Dispose() } - $config = Join-Path $work "$Channel.config" + $config = Join-Path $work "$identity.config" $feed = [Security.SecurityElement]::Escape([IO.Path]::GetDirectoryName($Package)) "" | Set-Content $config $install += @('--version', $expectedVersion, '--configfile', $config) @@ -92,15 +95,17 @@ function Install-Tool([string] $Channel, [string] $Package) { } $result = Invoke-Logged $install $log if ($result.code -ne 0) { throw 'Tool installation failed; see installation log.' } - $store = Join-Path $work "$Channel/.store/docfx" + $store = Join-Path $work "$identity/.store/docfx" $versions = @(Get-ChildItem -LiteralPath $store -Directory) if ($versions.Count -ne 1) { throw 'Expected exactly one installed package version.' } if ($expectedVersion -ne $versions[0].Name) { throw 'Installed package version differs from the exact requested version.' } $tool.version = $versions[0].Name if ($Channel -eq 'stable' -and $tool.version.Contains('-')) { throw 'Stable installation selected a prerelease.' } - $configs = @(Get-ChildItem -LiteralPath $store -Filter docfx.runtimeconfig.json -Recurse | Where-Object { $_.FullName -match '[\\/]tools[\\/]net10\.0[\\/]any[\\/]' }) - if ($configs.Count -ne 1) { throw 'Expected one installed net10.0 tool runtime configuration.' } + $frameworkPath = '[\\/]tools[\\/]' + [Regex]::Escape($Framework) + '[\\/]any[\\/]' + $configs = @(Get-ChildItem -LiteralPath $store -Filter docfx.runtimeconfig.json -Recurse | Where-Object { $_.FullName -match $frameworkPath }) + if ($configs.Count -ne 1) { throw "Expected one installed $Framework tool runtime configuration." } $tool.runtimeTfm = (Get-Content $configs[0].FullName -Raw | ConvertFrom-Json).runtimeOptions.tfm + if ($tool.runtimeTfm -ne $Framework) { throw 'Installed tool target framework differs from the requested framework.' } $tool.dll = Join-Path $configs[0].DirectoryName 'docfx.dll' $nupkg = Join-Path $store "$($tool.version)/docfx/$($tool.version)/docfx.$($tool.version).nupkg" if (-not (Test-Path -LiteralPath $nupkg)) { throw 'Installed package archive is missing from the isolated tool store.' } @@ -119,8 +124,10 @@ try { Push-Location $work try { Invoke-Logged @('--info') (Join-Path $logs 'environment.log') | Out-Null - if (-not $SkipStable) { $tools.Add((Install-Tool 'stable' '')) } - if ($NightlyPackage) { $tools.Add((Install-Tool 'nightly' $NightlyPackage)) } + if (-not $SkipStable) { $tools.Add((Install-Tool 'stable' '' 'net10.0')) } + if ($NightlyPackage) { + foreach ($framework in $NightlyFrameworks) { $tools.Add((Install-Tool 'nightly' $NightlyPackage $framework)) } + } if ($tools.Count -eq 0) { throw 'Select at least one tool package.' } } finally { Pop-Location } @@ -128,12 +135,12 @@ try { if ($target.sdk -notmatch '^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?$' -or $target.projectTfm -notmatch '^net\d+\.0$') { throw 'Invalid SDK matrix entry.' } foreach ($tool in $tools) { foreach ($scenario in $scenarios) { - $id = "$($tool.channel)-$($target.sdk)-$($target.projectTfm)-$scenario" + $id = "$($tool.channel)-$($tool.framework)-$($target.sdk)-$($target.projectTfm)-$scenario" $logName = "logs/$measurementId-$id.log" $log = Join-Path $OutputDirectory $logName $row = [ordered]@{ sdk = $target.sdk; selectedSdk = $null; projectTfm = $target.projectTfm - channel = $tool.channel; toolVersion = $tool.version; toolRuntimeTfm = $tool.runtimeTfm + channel = $tool.channel; toolFramework = $tool.framework; toolVersion = $tool.version; toolRuntimeTfm = $tool.runtimeTfm packageSha256 = $tool.packageSha256; os = [Runtime.InteropServices.RuntimeInformation]::OSDescription scenario = $scenario; testedAt = [DateTime]::UtcNow.ToString('o'); outcome = 'infrastructure-error' diagnostics = ''; log = $logName @@ -155,7 +162,7 @@ try { } $row.selectedSdk = $selection.text.Trim() if ($row.selectedSdk -ne $target.sdk) { throw 'Actual selected SDK does not match requested SDK.' } - if ($tool.error) { throw "$($tool.error) See logs/$($tool.channel)-install.log." } + if ($tool.error) { throw "$($tool.error) See logs/$($tool.channel)-$($tool.framework)-install.log." } $result = Invoke-Logged @('restore', '--nologo') $log if ($result.code -ne 0) { throw 'Fixture restore failed; compatibility was not measured.' } $result = Invoke-Logged @('build', '--no-restore', '--nologo', '-warnaserror') $log @@ -196,12 +203,13 @@ try { } } finally { $report = [ordered]@{ - schemaVersion = 1; generatedAt = [DateTime]::UtcNow.ToString('o') + schemaVersion = 2; generatedAt = [DateTime]::UtcNow.ToString('o') source = @{ repository = $(if ($env:GITHUB_REPOSITORY) { $env:GITHUB_REPOSITORY } else { 'local' }); sha = $sha dirty = $dirty runId = $(if ($env:GITHUB_RUN_ID) { $env:GITHUB_RUN_ID } else { $null }); runAttempt = $(if ($env:GITHUB_RUN_ATTEMPT) { $env:GITHUB_RUN_ATTEMPT } else { $null }) } stableSelection = $(if ($SkipStable) { $null } else { ($tools | Where-Object { $_.channel -eq 'stable' } | Select-Object -First 1).selection }) - matrix = $matrix; channels = @($tools | ForEach-Object { $_.channel }); scenarios = $scenarios; results = @($rows.ToArray()) + toolTargets = @($tools | ForEach-Object { @{ channel = $_.channel; framework = $_.framework } }) + matrix = $matrix; channels = @($tools | ForEach-Object { $_.channel } | Select-Object -Unique); scenarios = $scenarios; results = @($rows.ToArray()) } $report | ConvertTo-Json -Depth 10 | Set-Content (Join-Path $OutputDirectory 'compatibility-report.json') -Encoding utf8 $env:DOTNET_ROLL_FORWARD = $previousRollForward diff --git a/test/compatibility/Read-ReportArchive.ps1 b/test/compatibility/Read-ReportArchive.ps1 index ca970d217e2..077455ad9a1 100644 --- a/test/compatibility/Read-ReportArchive.ps1 +++ b/test/compatibility/Read-ReportArchive.ps1 @@ -32,7 +32,8 @@ try { if ($EvidenceDirectory) { if (Test-Path $EvidenceDirectory) { throw 'Evidence directory must be new.' } $report = $json | ConvertFrom-Json - if (!$report.results -or $report.results.Count -gt 48) { throw 'Invalid case log manifest.' } + # At most 12 SDK/project pairs, 8 package/tool targets, and 2 scenarios. + if (!$report.results -or $report.results.Count -gt 192) { throw 'Invalid case log manifest.' } $names = [Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) $destinations = [Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) $logs = @() diff --git a/test/compatibility/report.mjs b/test/compatibility/report.mjs index cbdbf6f382f..db528115e9f 100644 --- a/test/compatibility/report.mjs +++ b/test/compatibility/report.mjs @@ -70,6 +70,11 @@ export async function latestReport(api, readArchive, expectedMatrix) { const key = t => `${t.channel}/${t.projectTfm}` if (JSON.stringify(report.matrix.map(key).sort()) !== JSON.stringify(expectedMatrix.map(key).sort()) || report.matrix.some(t => !t.sdk.startsWith(`${t.channel}.`))) throw new Error('Report differs from the reviewed SDK channel matrix.') } + if (expectedMatrix && report.schemaVersion === 2) { + const expectedTools = ['stable/net10.0', ...['net8.0', 'net9.0', 'net10.0', 'net11.0'].map(tfm => `nightly/${tfm}`)] + const actualTools = report.toolTargets.map(t => `${t.channel}/${t.framework}`) + if (JSON.stringify(actualTools.sort()) !== JSON.stringify(expectedTools.sort())) throw new Error('Report differs from the reviewed tool framework matrix.') + } return report } if (runs.length < 100) break @@ -232,8 +237,8 @@ export async function main([command, path, output, ...extra]) { console.error(process.env.GITHUB_ACTIONS === 'true' ? `::error title=SDK compatibility::${error}` : `Error: ${error}`) } if (process.env.GITHUB_STEP_SUMMARY) { - const rows = report.results.map(r => `| ${r.channel} | ${r.toolVersion ?? 'Not measured'} | ${r.sdk} | ${r.projectTfm} | ${r.scenario} | ${r.outcome} |`).join('\n') - await writeFile(process.env.GITHUB_STEP_SUMMARY, `### SDK compatibility\n\n${summary}\n\n${policy}\n\nSource: ${report.source.sha}\n\n| DocFX channel | Package version | Project SDK | Project TFM | Scenario | Outcome |\n| --- | --- | --- | --- | --- | --- |\n${rows}\n\n`, { flag: 'a' }) + const rows = report.results.map(r => `| ${r.channel} | ${r.toolVersion ?? 'Not measured'} | ${r.toolFramework ?? r.toolRuntimeTfm ?? 'Not measured'} | ${r.sdk} | ${r.projectTfm} | ${r.scenario} | ${r.outcome} |`).join('\n') + await writeFile(process.env.GITHUB_STEP_SUMMARY, `### SDK compatibility\n\n${summary}\n\n${policy}\n\nSource: ${report.source.sha}\n\n| DocFX channel | Package version | Tool TFM | Project SDK | Project TFM | Scenario | Outcome |\n| --- | --- | --- | --- | --- | --- | --- |\n${rows}\n\n`, { flag: 'a' }) } process.exitCode = unmeasured || (strict && incompatible) ? 1 : 0 break diff --git a/test/compatibility/report.test.mjs b/test/compatibility/report.test.mjs index 0cddd7f6db2..753caadaec7 100644 --- a/test/compatibility/report.test.mjs +++ b/test/compatibility/report.test.mjs @@ -36,6 +36,19 @@ function report(matrix = [{ channel: '10.0', sdk: '10.0.401', projectTfm: 'net10 })))) return data } +function multiToolReport(matrix = resolveMatrix(matrixConfig, sdkIndex)) { + const data = report(matrix) + data.schemaVersion = 2 + data.toolTargets = [ + { channel: 'stable', framework: 'net10.0' }, + ...['net8.0', 'net9.0', 'net10.0', 'net11.0'].map(framework => ({ channel: 'nightly', framework })), + ] + data.results = data.results.flatMap(row => data.toolTargets.filter(t => t.channel === row.channel).map(({ framework }) => ({ + ...row, toolFramework: framework, toolRuntimeTfm: framework, + log: row.log.replace(`${row.channel}-`, `${row.channel}-${framework}-`), + }))) + return data +} function run(overrides = {}) { return { id: 42, workflow_id: 7, path: '.github/workflows/nightly.yml', head_repository: { full_name: 'dotnet/docfx' }, head_branch: 'main', head_sha: sha, event: 'schedule', status: 'completed', conclusion: 'failure', run_attempt: 1, ...overrides } } @@ -416,6 +429,70 @@ test('validates and renders 20 distinct cases in matrix order with net10.0 tool assert.match(html, /20 passed<\/strong> · 0 incompatible<\/p>/) }) +test('validates and renders all 50 cases across four nightly tool targets', async () => { + const data = multiToolReport() + assert.equal(data.results.length, 50) + assert.equal(validateReport(data, data.source, now), data) + assert.equal(await latestReport(adapter(), async () => data, matrixConfig.channels), data) + const html = renderReport(data, now) + const rows = [...html.matchAll(/(.*?)<\/tbody>/gs)][0][1].matchAll(/(.*?)<\/tr>/gs) + for (const [i, row] of [...rows].entries()) { + const indices = [...row[1].matchAll(/data-case-index="(\d+)"/g)].map(match => Number(match[1])) + assert.equal(indices.length, 10) + assert.ok(indices.every(index => data.results[index].sdk === data.matrix[i].sdk && data.results[index].projectTfm === data.matrix[i].projectTfm)) + assert.equal(new Set(indices.map(index => `${data.results[index].channel}/${data.results[index].toolFramework}/${data.results[index].scenario}`)).size, 10) + } + for (const { channel, framework } of data.toolTargets) { + const version = channel === 'stable' ? '2.80.1' : '2.80.2-preview.1' + assert.equal(html.split(`${version} & ${framework}`).length - 1, 2) + } + assert.equal((html.match(/data-case-index=/g) ?? []).length, 50) +}) + +test('requires each requested tool target and rejects fallback runtimes and mixed package bytes', () => { + for (const framework of ['net8.0', 'net9.0', 'net10.0', 'net11.0']) { + const complete = multiToolReport() + const index = complete.results.findIndex(r => r.channel === 'nightly' && r.toolFramework === framework) + for (const mutate of [ + r => r.results.splice(index, 1), + r => { r.results[index] = r.results.find(row => row.channel === 'nightly' && row.toolFramework !== framework) }, + r => { r.results[index].toolRuntimeTfm = framework === 'net8.0' ? 'net10.0' : 'net8.0' }, + r => { r.results[index].packageSha256 = 'c'.repeat(64) }, + r => { r.results[index].toolVersion = '2.80.3-preview.1' }, + ]) { + const data = structuredClone(complete); mutate(data) + assert.throws(() => validateReport(data, undefined, now)) + } + } + for (const mutate of [ + r => { delete r.toolTargets }, r => r.toolTargets.push(r.toolTargets[0]), + r => { r.toolTargets[0].channel = 'unknown' }, r => { r.toolTargets[0].framework = 'invalid' }, + ]) { + const data = multiToolReport(); mutate(data) + assert.throws(() => validateReport(data, undefined, now)) + } +}) + +test('trusted retrieval rejects an omitted nightly target while bounded local reports remain valid', async () => { + const data = multiToolReport() + data.toolTargets = data.toolTargets.filter(t => t.framework !== 'net9.0') + data.results = data.results.filter(r => r.toolFramework !== 'net9.0') + assert.equal(validateReport(data, undefined, now), data) + await assert.rejects(latestReport(adapter(), async () => data, matrixConfig.channels), /reviewed tool framework matrix/) +}) + +test('failed tool installations retain their requested framework in the report and headers', () => { + const data = multiToolReport() + for (const row of data.results.filter(r => r.channel === 'nightly' && ['net8.0', 'net9.0'].includes(r.toolFramework))) { + row.toolRuntimeTfm = null; row.toolVersion = null; row.packageSha256 = null; row.outcome = 'infrastructure-error' + } + assert.equal(validateReport(data, data.source, now), data) + const html = renderReport(data, now) + for (const framework of ['net8.0', 'net9.0']) assert.equal(html.split(`Not measured & ${framework}`).length - 1, 2) + assert.equal((html.match(/data-case-index=/g) ?? []).length, 50) + assert.equal((html.match(/data-outcome="infrastructure-error"/g) ?? []).length, 20) +}) + test('each net11.0 observation is required and cannot be replaced with the SDK 11 net10.0 row', async () => { const complete = report(resolveMatrix(matrixConfig, sdkIndex)) const indices = complete.results.flatMap((r, i) => r.projectTfm === 'net11.0' ? [i] : []) @@ -465,11 +542,15 @@ function harnessIdentifiers(contexts) { if ($measurement) { Invoke-Expression $measurement.Extent.Text } $cases = foreach ($target in ('${matrix}' | ConvertFrom-Json)) { foreach ($channel in @('stable', 'nightly')) { - $tool = @{ channel = $channel } - foreach ($scenario in @('basic', 'razor')) { - Invoke-Expression ($assignments -join [Environment]::NewLine) - if (!$id.Contains($target.projectTfm) -or $directory -ne (Join-Path $work $id)) { throw 'Incorrect case identity' } - @{ id = $id; log = $logName; directory = $directory } + $frameworks = if ($channel -eq 'nightly') { @('net8.0', 'net9.0', 'net10.0', 'net11.0') } else { @('net10.0') } + foreach ($framework in $frameworks) { + $tool = @{ channel = $channel; framework = $framework } + foreach ($scenario in @('basic', 'razor')) { + Invoke-Expression ($assignments -join [Environment]::NewLine) + if (!$id.Contains($target.projectTfm) -or $directory -ne (Join-Path $work $id)) { throw 'Incorrect case identity' } + if (!$id.Contains($framework)) { throw 'Missing tool framework in case identity' } + @{ id = $id; log = $logName; directory = $directory } + } } } } @@ -479,11 +560,11 @@ function harnessIdentifiers(contexts) { `], { encoding: 'utf8', stdio: 'pipe' })) } -test('actual harness identifiers isolate all 20 work directories and log files by project TFM', () => { +test('actual harness identifiers isolate all 50 work directories and logs by tool and project TFM', () => { const [{ cases }] = harnessIdentifiers([{ runId: '42', runAttempt: '1' }]) - assert.equal(cases.length, 20) - assert.equal(new Set(cases.map(row => row.directory)).size, 20) - assert.equal(new Set(cases.map(row => row.log)).size, 20) + assert.equal(cases.length, 50) + assert.equal(new Set(cases.map(row => row.directory)).size, 50) + assert.equal(new Set(cases.map(row => row.log)).size, 50) for (const row of cases) assert.match(row.log, /^logs\/[a-zA-Z0-9.-]+\.log$/) }) @@ -494,7 +575,7 @@ test('actual harness logs never reuse URLs across runs, retries or repeated loca { runId: null, runAttempt: null }, { runId: null, runAttempt: null }, ]) const urls = runs.flatMap(({ cases }) => cases.map(row => caseLogUrl(row, 'https://example.test/reports/report.json').href)) - assert.equal(new Set(urls).size, 120, 'Different measurements must not serve different evidence at the same log URL') + assert.equal(new Set(urls).size, 300, 'Different measurements must not serve different evidence at the same log URL') }) test('discovers an exact official stable release and rejects previews or malformed versions', () => { @@ -540,12 +621,12 @@ test('exact installer refuses an older installed version and never retries a mis function Invoke-Logged($Arguments, $Log) { $calls.Add($Arguments); return @{ code = $script:exitCode; text = 'Synthetic installer boundary' } } try { $script:exitCode = 1 - $missing = Install-Tool 'stable' '' + $missing = Install-Tool 'stable' '' 'net10.0' if (!$missing.error -or $calls.Count -ne 1) { throw 'Missing package silently retried or passed' } if (($calls[0] -join ' ') -notmatch '--version 2.80.1') { throw 'Exact version was not requested' } - New-Item -ItemType Directory (Join-Path $work 'stable/.store/docfx/2.78.5') -Force | Out-Null + New-Item -ItemType Directory (Join-Path $work 'stable-net10.0/.store/docfx/2.78.5') -Force | Out-Null $script:exitCode = 0 - $wrong = Install-Tool 'stable' '' + $wrong = Install-Tool 'stable' '' 'net10.0' if ($wrong.error -notmatch 'differs from the exact requested version' -or $calls.Count -ne 2) { throw 'Wrong package identity accepted or retried' } } finally { Remove-Item $work -Recurse -Force } ` @@ -634,7 +715,7 @@ const readArchive = (archive, evidence) => execFileSync('pwsh', ['-NoProfile', ' test('archive and prepare export only named case logs and preserve JSON/log bytes', async () => { const temp = await mkdtemp(join(tmpdir(), 'docfx-evidence-test-')) try { - const data = report() + const data = multiToolReport() const original = Buffer.from('\ufeff' + JSON.stringify(data, null, 2).replaceAll('\n', '\r\n') + '\r\n') const bytes = Buffer.from('\ufeffunit log\r\n\u001b[31mwarning XX123: unit text\u001b[0m\r\n') const archive = join(temp, 'evidence.zip'); const evidence = join(temp, 'evidence'); const output = join(temp, 'site', 'sdk-compatibility.json') @@ -800,7 +881,7 @@ test('reporting check warns without failing incompatible observations; strict ch assert.equal(result.status, strict && outcome === 'incompatible' ? 1 : 0, result.stderr) const markdown = await readFile(summary, 'utf8') assert.match(markdown, strict ? /Strict mode/ : /Reporting mode/) - assert.ok(markdown.includes(`| nightly | 2.80.2-preview.1 | 10.0.401 | net10.0 | razor | ${outcome} |`)) + assert.ok(markdown.includes(`| nightly | 2.80.2-preview.1 | net10.0 | 10.0.401 | net10.0 | razor | ${outcome} |`)) if (outcome === 'incompatible') { assert.match(result.stderr, /::warning title=SDK compatibility::1 incompatible result/) assert.match(result.stdout, /3 passed, 1 incompatible, 0 unavailable, 0 infrastructure-error/)
SDK versionProject target${escape(caseTitle(column))}${escape(column.toolVersion)} & ${escape(column.toolRuntimeTfm)}${escape(caseTitle(column))}${escape(column.toolVersion)} & ${escape(toolFramework(column))}
${escape(target.sdk)}${escape(target.projectTfm)}