Skip to content

Migrate API compatibility test projects to MSTest.Sdk on MTP #54880

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Evangelink merged 3 commits into from
Jun 22, 2026
Merged

Migrate API compatibility test projects to MSTest.Sdk on MTP #54880

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
098ea13
Migrate API compatibility test projects to MSTest.Sdk on MTP
Jun 19, 2026
288984b
Make SymbolFactory test helper framework-neutral
Evangelink Jun 19, 2026
4e20605
Fix reversed string-assertion argument order from xUnit->MSTest conve…
Evangelink Jun 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions ...soft.DotNet.ApiCompat.IntegrationTests/Microsoft.DotNet.ApiCompat.IntegrationTests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="MSTest.Sdk">

<PropertyGroup>
<TargetFrameworks>$(SdkTargetFramework);$(NetFrameworkToolCurrent)</TargetFrameworks>
<OutputType Condition="'$(TargetFramework)' == '$(SdkTargetFramework)'">Exe</OutputType>
<AppendTargetFrameworkToOutputPath>true</AppendTargetFrameworkToOutputPath>
</PropertyGroup>

<ItemGroup>
Expand All @@ -14,7 +12,7 @@
<!-- Build-ordering dependencies on the tools and tasks under test. ReferenceOutputAssembly="false" so we don't take a compile-time dependency on netcore-only types; SetTargetFramework pins both inner builds to consume the netcore TFM since these projects are netcore-only. -->
<ProjectReference Include="$(RepoRoot)src\Compatibility\ApiCompat\Microsoft.DotNet.ApiCompat.Task\Microsoft.DotNet.ApiCompat.Task.csproj" ReferenceOutputAssembly="false" SetTargetFramework="TargetFramework=$(NetMinimum)" />
<ProjectReference Include="$(RepoRoot)src\Compatibility\ApiCompat\Microsoft.DotNet.ApiCompat.Tool\Microsoft.DotNet.ApiCompat.Tool.csproj" ReferenceOutputAssembly="false" SetTargetFramework="TargetFramework=$(NetMinimum)" />
<ProjectReference Include="..\Microsoft.NET.TestFramework\Microsoft.NET.TestFramework.csproj" />
<ProjectReference Include="..\Microsoft.NET.TestFramework.MSTest\Microsoft.NET.TestFramework.MSTest.csproj" />
</ItemGroup>

<!-- Stage the apicompat CLI binary closure under tools\Microsoft.DotNet.ApiCompat.Tool\ so the
Expand All @@ -33,4 +31,12 @@
</ItemGroup>
</Target>

<ItemGroup>
<Using Include="Microsoft.NET.TestFramework" />
<Using Include="Microsoft.NET.TestFramework.Assertions" />
<Using Include="Microsoft.NET.TestFramework.Commands" />
<Using Include="Microsoft.NET.TestFramework.ProjectConstruction" />
<Using Include="Microsoft.NET.TestFramework.Utilities" />
</ItemGroup>

</Project>
13 changes: 7 additions & 6 deletions ...rosoft.DotNet.ApiCompat.IntegrationTests/Task/ValidateAssembliesTargetIntegrationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
// Licensed to the .NET Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

namespace Microsoft.DotNet.ApiCompat.Task.IntegrationTests
{
public class ValidateAssembliesTargetIntegrationTests(ITestOutputHelper log) : SdkTest(log)
[TestClass]
public class ValidateAssembliesTargetIntegrationTests() : SdkTest()
{
private const string TestAssetName = "ApiCompatValidateAssembliesTestProject";

[Fact]
[TestMethod]
public void ValidateAssemblies_NoBreakingChanges_Succeeds()
{
// Build the asset twice with the same source to produce a "contract" DLL and an "implementation" DLL
Expand All @@ -25,7 +26,7 @@ public void ValidateAssemblies_NoBreakingChanges_Succeeds()
result.StdOut.Should().NotContain("error CP0002");
}

[Fact]
[TestMethod]
public void ValidateAssemblies_BreakingChange_FailsWithCP0002()
{
// Contract has Goodbye(string); implementation removes it via -p:ForceBreakingChange=true.
Expand All @@ -45,7 +46,7 @@ public void ValidateAssemblies_BreakingChange_FailsWithCP0002()
.And.Contain("Goodbye");
}

[Fact]
[TestMethod]
public void ValidateAssemblies_StrictMode_FailsOnAddition()
{
// Implementation adds Welcome(string) (-p:AddNewMember=true). Without strict mode, ApiCompat tolerates additions.
Expand All @@ -66,7 +67,7 @@ public void ValidateAssemblies_StrictMode_FailsOnAddition()
.And.Contain("Welcome");
}

[Fact]
[TestMethod]
public void ValidateAssemblies_GeneratesAndConsumesSuppressionFile()
{
// 1) Generate a suppression file for the breaking change.
Expand Down
48 changes: 23 additions & 25 deletions ...Microsoft.DotNet.ApiCompat.IntegrationTests/Task/ValidatePackageTargetIntegrationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,11 @@

namespace Microsoft.DotNet.ApiCompat.Task.IntegrationTests
{
[TestClass]
public class ValidatePackageTargetIntegrationTests : SdkTest
{
public ValidatePackageTargetIntegrationTests(ITestOutputHelper log) : base(log)
{
}

[Fact]
[TestMethod]
public void InvalidPackage()
{
var testAsset = TestAssetsManager
Expand All @@ -24,11 +22,11 @@ public void InvalidPackage()
.Execute($"-p:ForceValidationProblem=true");

// No failures while running the package validation on a simple assembly.
Assert.Equal(1, result.ExitCode);
Assert.AreEqual(1, result.ExitCode);
Assert.Contains("error CP0002: Member 'void PackageValidationTestProject.Program.SomeAPINotInCore()' exists on lib/netstandard2.0/PackageValidationTestProject.dll but not on lib/net8.0/PackageValidationTestProject.dll", result.StdOut);
}

[Fact]
[TestMethod]
public void ValidatePackageTargetRunsSuccessfully()
{
var testAsset = TestAssetsManager
Expand All @@ -39,10 +37,10 @@ public void ValidatePackageTargetRunsSuccessfully()
.Execute();

// No failures while running the package validation on a simple assembly.
Assert.Equal(0, result.ExitCode);
Assert.AreEqual(0, result.ExitCode);
}

[Fact]
[TestMethod]
public void ValidatePackageTargetRunsSuccessfullyWithBaselineCheck()
{
var testAsset = TestAssetsManager
Expand All @@ -52,17 +50,17 @@ public void ValidatePackageTargetRunsSuccessfullyWithBaselineCheck()
var result = new PackCommand(Log, Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.csproj"))
.Execute($"-p:PackageOutputPath={testAsset.TestRoot}");

Assert.Equal(0, result.ExitCode);
Assert.AreEqual(0, result.ExitCode);

string packageValidationBaselinePath = Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.1.0.0.nupkg");
result = new PackCommand(Log, Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.csproj"))
.Execute($"-p:PackageVersion=2.0.0;PackageValidationBaselinePath={packageValidationBaselinePath}");

// No failures while running the package validation on a simple assembly.
Assert.Equal(0, result.ExitCode);
Assert.AreEqual(0, result.ExitCode);
}

[Fact]
[TestMethod]
public void ValidatePackageTargetRunsSuccessfullyWithBaselineVersion()
{
var testAsset = TestAssetsManager
Expand All @@ -72,16 +70,16 @@ public void ValidatePackageTargetRunsSuccessfullyWithBaselineVersion()
var result = new PackCommand(Log, Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.csproj"))
.Execute($"-p:PackageOutputPath={testAsset.TestRoot}");

Assert.Equal(0, result.ExitCode);
Assert.AreEqual(0, result.ExitCode);

result = new PackCommand(Log, Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.csproj"))
.Execute($"-p:PackageVersion=2.0.0;PackageValidationBaselineVersion=1.0.0;PackageValidationBaselineName=PackageValidationTestProject");

// No failures while running the package validation on a simple assembly.
Assert.Equal(0, result.ExitCode);
Assert.AreEqual(0, result.ExitCode);
}

[Fact]
[TestMethod]
public void ValidatePackageTargetFailsWithBaselineVersion()
{
var testAsset = TestAssetsManager
Expand All @@ -91,18 +89,18 @@ public void ValidatePackageTargetFailsWithBaselineVersion()
var result = new PackCommand(Log, Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.csproj"))
.Execute($"-p:PackageOutputPath={testAsset.TestRoot}");

Assert.Equal(0, result.ExitCode);
Assert.AreEqual(0, result.ExitCode);

string packageValidationBaselinePath = Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.1.0.0.nupkg");
result = new PackCommand(Log, Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.csproj"))
.Execute($"-p:PackageVersion=2.0.0;AddBreakingChange=true;PackageValidationBaselinePath={packageValidationBaselinePath}");

Assert.Equal(1, result.ExitCode);
Assert.AreEqual(1, result.ExitCode);
Assert.Contains("error CP0002: Member 'void PackageValidationTestProject.Program.SomeApiNotInLatestVersion()' exists on [Baseline] lib/net8.0/PackageValidationTestProject.dll but not on lib/net8.0/PackageValidationTestProject.dll", result.StdOut);
Assert.Contains("error CP0002: Member 'void PackageValidationTestProject.Program.SomeApiNotInLatestVersion()' exists on [Baseline] lib/netstandard2.0/PackageValidationTestProject.dll but not on lib/netstandard2.0/PackageValidationTestProject.dll", result.StdOut);
}

[Fact]
[TestMethod]
public void ValidatePackageTargetWithIncorrectBaselinePackagePath()
{
var testAsset = TestAssetsManager
Expand All @@ -113,16 +111,16 @@ public void ValidatePackageTargetWithIncorrectBaselinePackagePath()
var result = new PackCommand(Log, Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.csproj"))
.Execute($"-p:PackageVersion=2.0.0;PackageValidationBaselinePath={nonExistentPackageBaselinePath}");

Assert.Equal(1, result.ExitCode);
Assert.AreEqual(1, result.ExitCode);
Assert.Contains(string.Format(Resources.NonExistentPackagePath, nonExistentPackageBaselinePath), result.StdOut);

// Disables package baseline validation.
result = new PackCommand(Log, Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.csproj"))
.Execute($"-p:PackageVersion=2.0.0;DisablePackageBaselineValidation=true;PackageValidationBaselinePath={nonExistentPackageBaselinePath}");
Assert.Equal(0, result.ExitCode);
Assert.AreEqual(0, result.ExitCode);
}

[Fact]
[TestMethod]
public void ValidatePackageTargetFailsWithBaselineVersionInStrictMode()
{
var testAsset = TestAssetsManager
Expand All @@ -132,18 +130,18 @@ public void ValidatePackageTargetFailsWithBaselineVersionInStrictMode()
var result = new PackCommand(Log, Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.csproj"))
.Execute($"-p:PackageOutputPath={testAsset.TestRoot}");

Assert.Equal(0, result.ExitCode);
Assert.AreEqual(0, result.ExitCode);

string packageValidationBaselinePath = Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.1.0.0.nupkg");
result = new PackCommand(Log, Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.csproj"))
.Execute($"-p:PackageVersion=2.0.0;ForceStrictModeBaselineValidationProblem=true;EnableStrictModeForBaselineValidation=true;PackageValidationBaselinePath={packageValidationBaselinePath}");

Assert.Equal(1, result.ExitCode);
Assert.AreEqual(1, result.ExitCode);
Assert.Contains("error CP0002: Member 'void PackageValidationTestProject.Program.SomeApiOnlyInLatestVersion()' exists on lib/net8.0/PackageValidationTestProject.dll but not on [Baseline] lib/net8.0/PackageValidationTestProject.dll", result.StdOut);
Assert.Contains("error CP0002: Member 'void PackageValidationTestProject.Program.SomeApiOnlyInLatestVersion()' exists on lib/netstandard2.0/PackageValidationTestProject.dll but not on [Baseline] lib/netstandard2.0/PackageValidationTestProject.dll", result.StdOut);
}

[Fact]
[TestMethod]
public void ValidatePackageTargetSucceedsWithBaselineVersionNotInStrictMode()
{
var testAsset = TestAssetsManager
Expand All @@ -153,13 +151,13 @@ public void ValidatePackageTargetSucceedsWithBaselineVersionNotInStrictMode()
var result = new PackCommand(Log, Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.csproj"))
.Execute($"-p:PackageOutputPath={testAsset.TestRoot}");

Assert.Equal(0, result.ExitCode);
Assert.AreEqual(0, result.ExitCode);

string packageValidationBaselinePath = Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.1.0.0.nupkg");
result = new PackCommand(Log, Path.Combine(testAsset.TestRoot, "PackageValidationTestProject.csproj"))
.Execute($"-p:PackageVersion=2.0.0;ForceStrictModeBaselineValidationProblem=true;PackageValidationBaselinePath={packageValidationBaselinePath}");

Assert.Equal(0, result.ExitCode);
Assert.AreEqual(0, result.ExitCode);
}
}
}
15 changes: 6 additions & 9 deletions test/Microsoft.DotNet.ApiCompat.IntegrationTests/Tool/ApiCompatToolIntegrationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Licensed to the .NET Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using Microsoft.DotNet.Cli.Utils;
Expand All @@ -10,15 +10,12 @@ namespace Microsoft.DotNet.ApiCompat.IntegrationTests
/// <c>dotnet exec Microsoft.DotNet.ApiCompat.Tool.dll</c> with command-line arguments and
/// asserting on the exit code and stdout, exactly as a customer would.
/// </summary>
[TestClass]
public class ApiCompatToolIntegrationTests : SdkTest
{
private const string TestAssetName = "ApiCompatValidateAssembliesTestProject";

public ApiCompatToolIntegrationTests(ITestOutputHelper log) : base(log)
{
}

[Fact]
[TestMethod]
public void ApiCompatTool_AssembliesIdentical_ExitsZero()
{
string assembly = BuildAsset(nameof(ApiCompatTool_AssembliesIdentical_ExitsZero), forceBreakingChange: false);
Expand All @@ -28,7 +25,7 @@ public void ApiCompatTool_AssembliesIdentical_ExitsZero()
result.Should().Pass();
}

[Fact]
[TestMethod]
public void ApiCompatTool_BreakingChange_ReportsCP0002()
{
string contractAssembly = BuildAsset($"{nameof(ApiCompatTool_BreakingChange_ReportsCP0002)}_left", forceBreakingChange: false);
Expand All @@ -42,7 +39,7 @@ public void ApiCompatTool_BreakingChange_ReportsCP0002()
.And.Contain("Goodbye");
}

[Fact]
[TestMethod]
public void ApiCompatTool_SuppressionFile_RoundTrip()
{
string contractAssembly = BuildAsset($"{nameof(ApiCompatTool_SuppressionFile_RoundTrip)}_left", forceBreakingChange: false);
Expand Down Expand Up @@ -71,7 +68,7 @@ public void ApiCompatTool_SuppressionFile_RoundTrip()
consumeResult.StdOut.Should().NotContain("error CP0002");
}

[Fact]
[TestMethod]
public void ApiCompatTool_PackageMode_DetectsRemovedApi()
{
// Pack the existing PackageValidationTestProject twice to produce two .nupkg files
Expand Down
2 changes: 1 addition & 1 deletion test/Microsoft.DotNet.ApiCompat.IntegrationTests/ToolPaths.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Licensed to the .NET Foundation under one or more agreements.
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

namespace Microsoft.DotNet.ApiCompat.IntegrationTests
Expand Down
35 changes: 18 additions & 17 deletions test/Microsoft.DotNet.ApiCompatibility.Tests/CompatDifferenceTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

namespace Microsoft.DotNet.ApiCompatibility.Tests
{
[TestClass]
public class CompatDifferenceTests
{
public static IEnumerable<object[]> CompatDifferencesData =>
Expand All @@ -22,22 +23,22 @@ public class CompatDifferenceTests
}
};

[Theory]
[MemberData(nameof(CompatDifferencesData))]
[TestMethod]
[DynamicData(nameof(CompatDifferencesData))]
public void PropertiesAreCorrect(MetadataInformation left, MetadataInformation right, string diagId, string message, string memberId, DifferenceType type)
{
CompatDifference difference = new(left, right, diagId, message, type, memberId);
Assert.Equal(left, difference.Left);
Assert.Equal(right, difference.Right);
Assert.Equal(diagId, difference.DiagnosticId);
Assert.Equal(message, difference.Message);
Assert.Equal(memberId, difference.ReferenceId);
Assert.Equal(type, difference.Type);
Assert.AreEqual(left, difference.Left);
Assert.AreEqual(right, difference.Right);
Assert.AreEqual(diagId, difference.DiagnosticId);
Assert.AreEqual(message, difference.Message);
Assert.AreEqual(memberId, difference.ReferenceId);
Assert.AreEqual(type, difference.Type);

Assert.Equal($"{diagId} : {message}", difference.ToString());
Assert.AreEqual($"{diagId} : {message}", difference.ToString());
}

[Fact]
[TestMethod]
public void IsEquatableWorksAsExpected()
{
CompatDifference difference = CompatDifference.CreateWithDefaultMetadata(DiagnosticIds.TypeMustExist, string.Empty, DifferenceType.Removed, "T:Foo");
Expand All @@ -47,13 +48,13 @@ public void IsEquatableWorksAsExpected()
CompatDifference differentMemberId = CompatDifference.CreateWithDefaultMetadata(DiagnosticIds.TypeMustExist, string.Empty, DifferenceType.Removed, "T:FooBar");
CompatDifference differentMessage = CompatDifference.CreateWithDefaultMetadata(DiagnosticIds.TypeMustExist, "Hello", DifferenceType.Removed, "T:Foo");

Assert.False(difference.Equals(null));
Assert.True(difference.Equals(otherEqual));
Assert.True(difference.Equals((object)otherEqual));
Assert.False(difference.Equals(differentDiagId));
Assert.False(difference.Equals(differentType));
Assert.False(difference.Equals(differentMemberId));
Assert.True(difference.Equals(differentMessage));
Assert.IsFalse(difference.Equals(null));
Assert.IsTrue(difference.Equals(otherEqual));
Assert.IsTrue(difference.Equals((object)otherEqual));
Assert.IsFalse(difference.Equals(differentDiagId));
Assert.IsFalse(difference.Equals(differentType));
Assert.IsFalse(difference.Equals(differentMemberId));
Assert.IsTrue(difference.Equals(differentMessage));
}
}
}
Loading
Loading