Skip to content

Corrected test discovery with flaky retries #3073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jeremydmiller merged 6 commits into from
Jun 11, 2026
Merged

Corrected test discovery with flaky retries #3073

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f898356
Correct test discovery
dmytro-pryvedeniuk Jun 3, 2026
5cf3ff4
Run tests with flaky retries
dmytro-pryvedeniuk Jun 8, 2026
ea3ac3d
Fix CosmosDb tests
dmytro-pryvedeniuk Jun 8, 2026
02f5318
Disable Pulsar compliance tests
dmytro-pryvedeniuk Jun 8, 2026
94c44cf
Fix ASB tests
dmytro-pryvedeniuk Jun 8, 2026
0152d0c
Disable RabbitMQ CircuitBreaker tests as flaky
dmytro-pryvedeniuk Jun 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions Directory.Packages.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,13 +104,14 @@
<PackageVersion Include="Swashbuckle.AspNetCore.SwaggerUI" Version="9.0.1" />
<PackageVersion Include="System.ComponentModel" Version="4.3.0" />
<PackageVersion Include="System.Security.Cryptography.Xml" Version="10.0.7" />
<PackageVersion Include="Testcontainers" Version="4.11.0" />
<PackageVersion Include="Testcontainers.LocalStack" Version="4.11.0" />
<PackageVersion Include="Testcontainers.MsSql" Version="4.11.0" />
<PackageVersion Include="Testcontainers.Nats" Version="4.10.0" />
<PackageVersion Include="Testcontainers.Pulsar" Version="4.10.0" />
<PackageVersion Include="Testcontainers.Redis" Version="4.10.0" />
<PackageVersion Include="Testcontainers.ServiceBus" Version="4.10.0" />
<PackageVersion Include="Testcontainers" Version="4.12.0" />
<PackageVersion Include="Testcontainers.CosmosDb" Version="4.12.0" />
<PackageVersion Include="Testcontainers.LocalStack" Version="4.12.0" />
<PackageVersion Include="Testcontainers.MsSql" Version="4.12.0" />
<PackageVersion Include="Testcontainers.Nats" Version="4.12.0" />
<PackageVersion Include="Testcontainers.Pulsar" Version="4.12.0" />
<PackageVersion Include="Testcontainers.Redis" Version="4.12.0" />
<PackageVersion Include="Testcontainers.ServiceBus" Version="4.12.0" />
<PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="9.0.5" />
<PackageVersion Include="System.Net.NameResolution" Version="4.3.0" />
<PackageVersion Include="System.Threading.Tasks.Dataflow" Version="9.0.5" />
Expand Down
58 changes: 23 additions & 35 deletions build/CITargets.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,8 +240,8 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
BuildTestProjectsWithFramework("net9.0", persistenceTests);
StartDockerServices("postgresql", "sqlserver", "rabbitmq");

RunSingleProjectOneClassAtATime(persistenceTests, frameworkOverride: "net9.0");
RunSingleProjectOneClassAtATime(postgresqlTests);
RunTestProject(persistenceTests, frameworkOverride: "net9.0");
RunTestProject(postgresqlTests);
});

Target CISqlite => _ => _
Expand All @@ -252,7 +252,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat

BuildTestProjects(sqliteTests);

RunSingleProjectOneClassAtATime(sqliteTests);
RunTestProject(sqliteTests);
});

Target CISqlServer => _ => _
Expand All @@ -264,7 +264,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
BuildTestProjects(sqlServerTests);
StartDockerServices("sqlserver");

RunSingleProjectOneClassAtATime(sqlServerTests);
RunTestProject(sqlServerTests);
});

Target CIMarten => _ => _
Expand All @@ -277,15 +277,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
BuildTestProjects(martenTests, martenSubscriptionTests);
StartDockerServices("postgresql");

// #2810: run each project in a single invocation rather than one
// dotnet-test spawn per test class. MartenTests has 111 test files;
// the per-class spawn overhead (process start + assembly load +
// xUnit discovery + per-fixture Postgres/daemon warm-up) dominated
// the ~22 min wall clock. Execution stays serial via the project's
// CollectionPerAssembly attribute, so isolation between classes is
// unchanged at the concurrency level — see RunWholeProjectWithRetry.
RunWholeProjectWithRetry(martenTests);
RunWholeProjectWithRetry(martenSubscriptionTests);
RunTestProjects([martenTests, martenSubscriptionTests]);
});

Target CIMySql => _ => _
Expand All @@ -297,7 +289,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
BuildTestProjects(mySqlTests);
StartDockerServices("mysql");

RunSingleProjectOneClassAtATime(mySqlTests);
RunTestProject(mySqlTests);
});

Target CIOracle => _ => _
Expand All @@ -309,7 +301,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
BuildTestProjects(oracleTests);
StartDockerServices("oracle");

RunSingleProjectOneClassAtATime(oracleTests);
RunTestProject(oracleTests);
});

Target CIEfCore => _ => _
Expand All @@ -325,8 +317,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
// See GH-2588.
StartDockerServices("postgresql", "sqlserver", "rabbitmq");

RunSingleProjectOneClassAtATime(efCoreTests);
RunSingleProjectOneClassAtATime(efCoreMultiTenancy);
RunTestProjects([efCoreTests, efCoreMultiTenancy]);
});

// ─── Transport CI Targets ──────────────────────────────────────────
Expand All @@ -341,8 +332,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
BuildTestProjects(sqsTests, snsTests);
StartDockerServices("localstack", "postgresql");

RunSingleProjectOneClassAtATime(sqsTests);
RunSingleProjectOneClassAtATime(snsTests);
RunTestProjects([sqsTests, snsTests]);
});

Target CIKafka => _ => _
Expand All @@ -354,7 +344,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
BuildTestProjects(tests);
StartDockerServices("kafka", "postgresql");

RunSingleProjectOneClassAtATime(tests);
RunTestProject(tests);
});

Target CIMQTT => _ => _
Expand All @@ -366,7 +356,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
BuildTestProjects(tests);
StartDockerServices("postgresql", "sqlserver");

RunSingleProjectOneClassAtATime(tests);
RunTestProject(tests);
});

Target CINATS => _ => _
Expand All @@ -378,7 +368,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
BuildTestProjects(tests);
StartDockerServices("postgresql");

RunSingleProjectOneClassAtATime(tests);
RunTestProject(tests);
});

Target CIPulsar => _ => _
Expand All @@ -389,7 +379,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat

BuildTestProjects(tests);

RunSingleProjectOneClassAtATime(tests);
RunTestProject(tests);
});

Target CIRedis => _ => _
Expand All @@ -401,7 +391,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
BuildTestProjects(tests);
StartDockerServices("postgresql");

RunSingleProjectOneClassAtATime(tests);
RunTestProject(tests);
});

Target CIHttp => _ => _
Expand Down Expand Up @@ -430,8 +420,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
BuildTestProjects(rabbitTests, circuitTests);
StartDockerServices("rabbitmq", "postgresql", "sqlserver");

RunSingleProjectOneClassAtATime(rabbitTests);
RunSingleProjectOneClassAtATime(circuitTests);
RunTestProjects([rabbitTests, circuitTests]);
});

/// <summary>
Expand All @@ -452,7 +441,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
BuildTestProjects(tests);
StartDockerServices("rabbitmq");

RunSingleProjectOneClassAtATime(tests);
RunTestProject(tests);
});

Target CICosmosDb => _ => _
Expand All @@ -464,8 +453,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat

BuildTestProjects(cosmosDbTests, leaderElectionTests);

RunSingleProjectOneClassAtATime(cosmosDbTests);
RunSingleProjectOneClassAtATime(leaderElectionTests);
RunTestProjects([cosmosDbTests, leaderElectionTests]);
});

Target CIRavenDb => _ => _
Expand All @@ -477,8 +465,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat

BuildTestProjectsWithFramework("net9.0", ravenDbTests, leaderElectionTests);

RunSingleProjectOneClassAtATime(ravenDbTests, frameworkOverride: "net9.0");
RunSingleProjectOneClassAtATime(leaderElectionTests, frameworkOverride: "net9.0");
RunTestProjects([ravenDbTests, leaderElectionTests], frameworkOverride: "net9.0");
});

Target CIGrpc => _ => _
Expand All @@ -489,7 +476,7 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat

BuildTestProjects(tests);

RunSingleProjectOneClassAtATime(tests);
RunTestProject(tests);
});

// ─── AOT Smoke ──────────────────────────────────────────────────────
Expand Down Expand Up @@ -533,9 +520,10 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
var tests = RootDirectory / "src" / "Transports" / "Azure" / "Wolverine.AzureServiceBus.Tests" / "Wolverine.AzureServiceBus.Tests.csproj";

BuildTestProjects(tests);
StartDockerServices("asb-emulator");
// Postgres is needed for leader election tests
StartDockerServices("asb-emulator", "postgresql");

RunSingleProjectOneClassAtATime(tests);
RunTestProject(tests);
});

Target CIPolecat => _ => _
Expand All @@ -547,6 +535,6 @@ void BuildTestProjectsWithFramework(string frameworkOverride, params AbsolutePat
BuildTestProjectsWithFramework("net10.0", polecatTests);
StartDockerServices("sqlserver");

RunSingleProjectOneClassAtATime(polecatTests, frameworkOverride: "net10.0");
RunTestProject(polecatTests, frameworkOverride: "net10.0");
});
}
Loading
Loading