diff --git a/TUnit.Analyzers.Tests/DataSourceGeneratorAnalyzerTests.cs b/TUnit.Analyzers.Tests/DataSourceGeneratorAnalyzerTests.cs index 2715ab44b8..0dc060f8f2 100644 --- a/TUnit.Analyzers.Tests/DataSourceGeneratorAnalyzerTests.cs +++ b/TUnit.Analyzers.Tests/DataSourceGeneratorAnalyzerTests.cs @@ -141,6 +141,8 @@ public class CustomDataAttribute : Attribute, IDataSourceAttribute { public bool SkipIfEmpty { get; set; } + public bool DeferEnumeration { get; set; } + public async IAsyncEnumerable>> GetDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata) { // Returns Foo instances as the test expects, not T diff --git a/TUnit.Core.SourceGenerator.Tests/DeferEnumerationTests.Test.verified.txt b/TUnit.Core.SourceGenerator.Tests/DeferEnumerationTests.Test.verified.txt new file mode 100644 index 0000000000..e741c62792 --- /dev/null +++ b/TUnit.Core.SourceGenerator.Tests/DeferEnumerationTests.Test.verified.txt @@ -0,0 +1,198 @@ +// +#pragma warning disable + +#nullable enable +namespace TUnit.Generated; +[global::System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverageAttribute] +[global::System.CodeDom.Compiler.GeneratedCode("TUnit", "VERSION_SCRUBBED")] +internal static class TUnit_TestProject_DeferEnumerationTests_DeferEnumerationTests__TestSource +{ + private static readonly global::TUnit.Core.ClassMetadata __classMetadata = global::TUnit.Core.ClassMetadata.GetOrAdd("TestsBase`1:global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests", new global::TUnit.Core.ClassMetadata + { + Type = typeof(global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests), + TypeInfo = new global::TUnit.Core.ConcreteType(typeof(global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests)), + Name = "DeferEnumerationTests", + Namespace = "TUnit.TestProject.DeferEnumerationTests", + Assembly = global::TUnit.Core.AssemblyMetadata.GetOrAdd("TestsBase`1", "TestsBase`1"), + Parameters = global::System.Array.Empty(), + Properties = global::System.Array.Empty(), + Parent = null + }); + private static readonly global::System.Type __classType = typeof(global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests); + private static readonly global::TUnit.Core.MethodMetadata __mm_0 = global::TUnit.Core.MethodMetadataFactory.Create("Deferred", __classType, typeof(global::System.Threading.Tasks.Task), __classMetadata, parameters: new global::TUnit.Core.ParameterMetadata[] +{ +global::TUnit.Core.ParameterMetadataFactory.Create(typeof(int), "value", new global::TUnit.Core.ConcreteType(typeof(int)), false, reflectionInfoFactory: static () => typeof(global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests).GetMethod("Deferred", global::System.Reflection.BindingFlags.Public | global::System.Reflection.BindingFlags.NonPublic | global::System.Reflection.BindingFlags.Instance, null, new global::System.Type[] { typeof(int) }, null)!.GetParameters()[0]) +}); + private static readonly global::TUnit.Core.MethodMetadata __mm_1 = global::TUnit.Core.MethodMetadataFactory.Create("DeferredWithRepeat", __classType, typeof(global::System.Threading.Tasks.Task), __classMetadata, parameters: new global::TUnit.Core.ParameterMetadata[] +{ +global::TUnit.Core.ParameterMetadataFactory.Create(typeof(int), "value", new global::TUnit.Core.ConcreteType(typeof(int)), false, reflectionInfoFactory: static () => typeof(global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests).GetMethod("DeferredWithRepeat", global::System.Reflection.BindingFlags.Public | global::System.Reflection.BindingFlags.NonPublic | global::System.Reflection.BindingFlags.Instance, null, new global::System.Type[] { typeof(int) }, null)!.GetParameters()[0]) +}); + private static global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests __CreateInstance(global::System.Type[] typeArgs, object?[] args) + { + return new global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests(); + } + private static global::System.Threading.Tasks.ValueTask __Invoke(global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests instance, int methodIndex, object?[] args, global::System.Threading.CancellationToken cancellationToken) + { + switch (methodIndex) + { + case 0: + { + try + { + switch (args.Length) + { + case 1: + { + return new global::System.Threading.Tasks.ValueTask(instance.Deferred(global::TUnit.Core.Helpers.CastHelper.Cast(args[0]))); + } + default: + throw new global::System.ArgumentException($"Expected exactly 1 argument, but got {args.Length}"); + } + } + catch (global::System.Exception ex) + { + return new global::System.Threading.Tasks.ValueTask(global::System.Threading.Tasks.Task.FromException(ex)); + } + } + case 1: + { + try + { + switch (args.Length) + { + case 1: + { + return new global::System.Threading.Tasks.ValueTask(instance.DeferredWithRepeat(global::TUnit.Core.Helpers.CastHelper.Cast(args[0]))); + } + default: + throw new global::System.ArgumentException($"Expected exactly 1 argument, but got {args.Length}"); + } + } + catch (global::System.Exception ex) + { + return new global::System.Threading.Tasks.ValueTask(global::System.Threading.Tasks.Task.FromException(ex)); + } + } + default: + throw new global::System.ArgumentOutOfRangeException(nameof(methodIndex)); + } + } + private static global::System.Attribute[] __Attributes(int groupIndex) + { + switch (groupIndex) + { + case 0: + { + return + [ + new global::TUnit.Core.TestAttribute() + ]; + } + case 1: + { + return + [ + new global::TUnit.Core.TestAttribute(), + new global::TUnit.Core.RepeatAttribute(2) + ]; + } + default: + throw new global::System.ArgumentOutOfRangeException(nameof(groupIndex)); + } + } + public static readonly global::TUnit.Core.TestEntry[] Entries = new global::TUnit.Core.TestEntry[] + { + new global::TUnit.Core.TestEntry + { + MethodName = "Deferred", + FullyQualifiedName = "TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests.Deferred", + FilePath = "", + LineNumber = 9, + Categories = global::System.Array.Empty(), + Properties = global::System.Array.Empty(), + HasDataSource = true, + RepeatCount = 0, + DependsOn = global::System.Array.Empty(), + MethodMetadata = __mm_0, + CreateInstance = __CreateInstance, + InvokeBody = __Invoke, + MethodIndex = 0, + CreateAttributes = __Attributes, + AttributeGroupIndex = 0, + TestDataSources = new global::TUnit.Core.IDataSourceAttribute[] +{ + new global::TUnit.Core.MethodDataSourceAttribute("TenValues") + { + DeferEnumeration = true, + Factory = (dataGeneratorMetadata) => + { + async global::System.Collections.Generic.IAsyncEnumerable>> Factory() + { + var result = global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests.TenValues(); + if (result is global::System.Collections.IEnumerable enumerable && !(result is string)) + { + foreach (var item in enumerable) + { + yield return () => global::System.Threading.Tasks.Task.FromResult(global::TUnit.Core.Helpers.DataSourceHelpers.ToObjectArray(item)); + } + } + else + { + yield return () => global::System.Threading.Tasks.Task.FromResult(global::TUnit.Core.Helpers.DataSourceHelpers.ToObjectArray(result)); + } + } + return Factory(); + } + }, +}, + }, + new global::TUnit.Core.TestEntry + { + MethodName = "DeferredWithRepeat", + FullyQualifiedName = "TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests.DeferredWithRepeat", + FilePath = "", + LineNumber = 16, + Categories = global::System.Array.Empty(), + Properties = global::System.Array.Empty(), + HasDataSource = true, + RepeatCount = 2, + DependsOn = global::System.Array.Empty(), + MethodMetadata = __mm_1, + CreateInstance = __CreateInstance, + InvokeBody = __Invoke, + MethodIndex = 1, + CreateAttributes = __Attributes, + AttributeGroupIndex = 1, + TestDataSources = new global::TUnit.Core.IDataSourceAttribute[] +{ + new global::TUnit.Core.MethodDataSourceAttribute("TenValues") + { + DeferEnumeration = true, + Factory = (dataGeneratorMetadata) => + { + async global::System.Collections.Generic.IAsyncEnumerable>> Factory() + { + var result = global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests.TenValues(); + if (result is global::System.Collections.IEnumerable enumerable && !(result is string)) + { + foreach (var item in enumerable) + { + yield return () => global::System.Threading.Tasks.Task.FromResult(global::TUnit.Core.Helpers.DataSourceHelpers.ToObjectArray(item)); + } + } + else + { + yield return () => global::System.Threading.Tasks.Task.FromResult(global::TUnit.Core.Helpers.DataSourceHelpers.ToObjectArray(result)); + } + } + return Factory(); + } + }, +}, + }, + }; +} +internal static partial class TUnit_TestRegistration +{ + static readonly int _r_TUnit_TestProject_DeferEnumerationTests_DeferEnumerationTests__TestSource = global::TUnit.Core.SourceRegistrar.RegisterEntries(static () => TUnit_TestProject_DeferEnumerationTests_DeferEnumerationTests__TestSource.Entries); +} diff --git a/TUnit.Core.SourceGenerator.Tests/DeferEnumerationTests.cs b/TUnit.Core.SourceGenerator.Tests/DeferEnumerationTests.cs new file mode 100644 index 0000000000..dcdb494d8e --- /dev/null +++ b/TUnit.Core.SourceGenerator.Tests/DeferEnumerationTests.cs @@ -0,0 +1,16 @@ + +namespace TUnit.Core.SourceGenerator.Tests; + +internal class DeferEnumerationTests : TestsBase +{ + [Test] + public Task Test() => RunTest(Path.Combine(Git.RootDirectory.FullName, + "TUnit.TestProject", + "DeferEnumerationTests", + "DeferEnumerationTests.cs"), + async generatedFiles => + { + await Assert.That(generatedFiles).IsNotEmpty(); + await Assert.That(string.Join("\n", generatedFiles)).Contains("DeferEnumeration = true"); + }); +} diff --git a/TUnit.Core.SourceGenerator/Generators/TestMetadataGenerator.cs b/TUnit.Core.SourceGenerator/Generators/TestMetadataGenerator.cs index 9dc62c4058..76eaf84af6 100644 --- a/TUnit.Core.SourceGenerator/Generators/TestMetadataGenerator.cs +++ b/TUnit.Core.SourceGenerator/Generators/TestMetadataGenerator.cs @@ -1636,6 +1636,15 @@ private static void GenerateMethodDataSourceAttribute(CodeWriter writer, Attribu writer.AppendLine(","); } + // Copy over DeferEnumeration if set so the test is enumerated at runtime instead of discovery. + // The hand-built initializer above only copies the named args it knows about, so this must be + // threaded explicitly (unlike the method-not-found path which round-trips all named arguments). + var deferProperty = attr.NamedArguments.FirstOrDefault(x => x.Key == "DeferEnumeration"); + if (deferProperty is { Key: not null, Value.IsNull: false } && deferProperty.Value.Value is true) + { + writer.AppendLine("DeferEnumeration = true,"); + } + // Set the Factory property with a strongly-typed function writer.AppendLine("Factory = (dataGeneratorMetadata) =>"); writer.AppendLine("{"); diff --git a/TUnit.Core/Attributes/TestData/ArgumentsAttribute.cs b/TUnit.Core/Attributes/TestData/ArgumentsAttribute.cs index bce2979eb6..311b67cec6 100644 --- a/TUnit.Core/Attributes/TestData/ArgumentsAttribute.cs +++ b/TUnit.Core/Attributes/TestData/ArgumentsAttribute.cs @@ -74,6 +74,10 @@ public sealed class ArgumentsAttribute : Attribute, IDataSourceAttribute, ITestR /// public bool SkipIfEmpty { get; set; } + /// + // [Arguments] yields a single row, so deferring its enumeration would be pure overhead - always false. + public bool DeferEnumeration { get => false; set { } } + /// /// Initializes a new instance of the class with the specified test argument values. /// @@ -166,6 +170,10 @@ public sealed class ArgumentsAttribute(T value) : TypedDataSourceAttribute /// public override bool SkipIfEmpty { get; set; } + /// + // [Arguments] yields a single row, so deferring its enumeration would be pure overhead - always false. + public override bool DeferEnumeration { get => false; set { } } + public override async IAsyncEnumerable>> GetTypedDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata) { yield return () => Task.FromResult(value); diff --git a/TUnit.Core/Attributes/TestData/AsyncUntypedDataSourceSourceGeneratorAttribute.cs b/TUnit.Core/Attributes/TestData/AsyncUntypedDataSourceSourceGeneratorAttribute.cs index 88ae04b9c0..0b2b83121f 100644 --- a/TUnit.Core/Attributes/TestData/AsyncUntypedDataSourceSourceGeneratorAttribute.cs +++ b/TUnit.Core/Attributes/TestData/AsyncUntypedDataSourceSourceGeneratorAttribute.cs @@ -8,6 +8,9 @@ public abstract class AsyncUntypedDataSourceGeneratorAttribute : Attribute, IAsy /// public virtual bool SkipIfEmpty { get; set; } + /// + public virtual bool DeferEnumeration { get; set; } + protected abstract IAsyncEnumerable>> GenerateDataSourcesAsync(DataGeneratorMetadata dataGeneratorMetadata); public IAsyncEnumerable>> GenerateAsync(DataGeneratorMetadata dataGeneratorMetadata) diff --git a/TUnit.Core/Attributes/TestData/DelegateDataSourceAttribute.cs b/TUnit.Core/Attributes/TestData/DelegateDataSourceAttribute.cs index 45a07b9795..9b687a3025 100644 --- a/TUnit.Core/Attributes/TestData/DelegateDataSourceAttribute.cs +++ b/TUnit.Core/Attributes/TestData/DelegateDataSourceAttribute.cs @@ -13,6 +13,9 @@ internal sealed class DelegateDataSourceAttribute : Attribute, IDataSourceAttrib /// public bool SkipIfEmpty { get; set; } + /// + public bool DeferEnumeration { get; set; } + public DelegateDataSourceAttribute(Func> factory, bool isShared = false) { _factory = factory ?? throw new ArgumentNullException(nameof(factory)); diff --git a/TUnit.Core/Attributes/TestData/EmptyDataSourceAttribute.cs b/TUnit.Core/Attributes/TestData/EmptyDataSourceAttribute.cs index a80ea2f2d2..4a838a651f 100644 --- a/TUnit.Core/Attributes/TestData/EmptyDataSourceAttribute.cs +++ b/TUnit.Core/Attributes/TestData/EmptyDataSourceAttribute.cs @@ -9,6 +9,10 @@ internal sealed class EmptyDataSourceAttribute : Attribute, IDataSourceAttribute /// public bool SkipIfEmpty { get; set; } + /// + // Always a single (empty) row, so deferring its enumeration would be pure overhead. + public bool DeferEnumeration { get => false; set { } } + public async IAsyncEnumerable>> GetDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata) { yield return () => Task.FromResult([ diff --git a/TUnit.Core/Attributes/TestData/IDataSourceAttribute.cs b/TUnit.Core/Attributes/TestData/IDataSourceAttribute.cs index 9dd5122933..2a5b187181 100644 --- a/TUnit.Core/Attributes/TestData/IDataSourceAttribute.cs +++ b/TUnit.Core/Attributes/TestData/IDataSourceAttribute.cs @@ -18,4 +18,19 @@ public interface IDataSourceAttribute /// When true, if the data source returns no data, the test will be skipped instead of failing. /// bool SkipIfEmpty { get; set; } + + /// + /// When true, this data source is not enumerated during test discovery. Instead, the test + /// appears as a single placeholder node, and the data rows are enumerated and executed at runtime + /// (each reported as a result nested under the placeholder). This avoids the IDE/test-explorer + /// overhead of expanding a data source that produces a large number of cases. + /// + /// + /// If any data source on a test sets this to true, the entire test's case expansion is + /// deferred to runtime. Tests deferred this way cannot be targeted individually by a filter, and + /// other tests cannot [DependsOn] their rows (the rows do not exist until runtime). + /// Single-row sources (such as [Arguments]) ignore this flag — there is nothing to defer — + /// so setting it on them has no effect. + /// + bool DeferEnumeration { get; set; } } diff --git a/TUnit.Core/Attributes/TestData/MethodDataSourceAttribute.cs b/TUnit.Core/Attributes/TestData/MethodDataSourceAttribute.cs index 582e750493..6d5aebb08d 100644 --- a/TUnit.Core/Attributes/TestData/MethodDataSourceAttribute.cs +++ b/TUnit.Core/Attributes/TestData/MethodDataSourceAttribute.cs @@ -104,6 +104,9 @@ public class MethodDataSourceAttribute : Attribute, IDataSourceAttribute /// public bool SkipIfEmpty { get; set; } + /// + public bool DeferEnumeration { get; set; } + public MethodDataSourceAttribute(string methodNameProvidingDataSource) { if (methodNameProvidingDataSource is null or { Length: < 1 }) @@ -143,6 +146,7 @@ internal InstanceMethodDataSourceAttribute ToInstanceVariant() converted.Arguments = Arguments; converted.SkipIfEmpty = SkipIfEmpty; + converted.DeferEnumeration = DeferEnumeration; return converted; } diff --git a/TUnit.Core/Attributes/TestData/NoDataSource.cs b/TUnit.Core/Attributes/TestData/NoDataSource.cs index 87052cb3bc..899065107d 100644 --- a/TUnit.Core/Attributes/TestData/NoDataSource.cs +++ b/TUnit.Core/Attributes/TestData/NoDataSource.cs @@ -8,6 +8,9 @@ internal class NoDataSource : IDataSourceAttribute /// public bool SkipIfEmpty { get; set; } + /// + public bool DeferEnumeration { get; set; } + public async IAsyncEnumerable>> GetDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata) { yield return static () => _emptyRowTask; diff --git a/TUnit.Core/Attributes/TestData/StaticDataSourceAttribute.cs b/TUnit.Core/Attributes/TestData/StaticDataSourceAttribute.cs index d520ba51ce..85ce484a74 100644 --- a/TUnit.Core/Attributes/TestData/StaticDataSourceAttribute.cs +++ b/TUnit.Core/Attributes/TestData/StaticDataSourceAttribute.cs @@ -11,6 +11,9 @@ internal sealed class StaticDataSourceAttribute : Attribute, IDataSourceAttribut /// public bool SkipIfEmpty { get; set; } + /// + public bool DeferEnumeration { get; set; } + public StaticDataSourceAttribute(params object?[][] data) { _data = data ?? throw new ArgumentNullException(nameof(data)); diff --git a/TUnit.Core/Attributes/TestData/TypedDataSourceAttribute.cs b/TUnit.Core/Attributes/TestData/TypedDataSourceAttribute.cs index 9ebf5c96c7..53b828bf4f 100644 --- a/TUnit.Core/Attributes/TestData/TypedDataSourceAttribute.cs +++ b/TUnit.Core/Attributes/TestData/TypedDataSourceAttribute.cs @@ -7,6 +7,9 @@ public abstract class TypedDataSourceAttribute : Attribute, ITypedDataSourceA /// public virtual bool SkipIfEmpty { get; set; } + /// + public virtual bool DeferEnumeration { get; set; } + public abstract IAsyncEnumerable>> GetTypedDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata); public async IAsyncEnumerable>> GetDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata) diff --git a/TUnit.Engine.Tests/DeferEnumerationTests.cs b/TUnit.Engine.Tests/DeferEnumerationTests.cs new file mode 100644 index 0000000000..d8a8e939bb --- /dev/null +++ b/TUnit.Engine.Tests/DeferEnumerationTests.cs @@ -0,0 +1,68 @@ +using Shouldly; +using TUnit.Engine.Tests.Enums; + +namespace TUnit.Engine.Tests; + +/// +/// Validates DeferEnumeration: a data source marked with it is NOT expanded during discovery (one +/// placeholder node is shown) and is expanded into the real cases at runtime. The placeholder is reported +/// as a container whose result aggregates its cases, so the run-time counts below are the number of data +/// cases plus one for the placeholder (e.g. a 10-row source => 10 cases + 1 placeholder = 11). Discovery-time +/// collapse is covered by manual verification / list-tests. +/// +public class DeferEnumerationTests(TestMode testMode) : InvokableTestBase(testMode) +{ + [Test] + public async Task Single_Deferred_Test_Expands_To_All_Cases_At_Runtime() + { + await RunTestsWithFilter( + "/*/*/DeferEnumerationTests/Deferred", + [ + result => result.ResultSummary.Outcome.ShouldBe("Completed"), + result => result.ResultSummary.Counters.Total.ShouldBe(11), + result => result.ResultSummary.Counters.Passed.ShouldBe(11), + result => result.ResultSummary.Counters.Failed.ShouldBe(0) + ]); + } + + [Test] + public async Task Deferred_Test_Honours_Repeat() + { + // [Repeat(2)] => 3 runs per case; 10 cases => 30 cases + 1 placeholder container = 31. + await RunTestsWithFilter( + "/*/*/DeferEnumerationTests/DeferredWithRepeat", + [ + result => result.ResultSummary.Outcome.ShouldBe("Completed"), + result => result.ResultSummary.Counters.Total.ShouldBe(31), + result => result.ResultSummary.Counters.Passed.ShouldBe(31), + result => result.ResultSummary.Counters.Failed.ShouldBe(0) + ]); + } + + [Test] + public async Task Deferred_Class_Data_Source_Expands_At_Runtime() + { + // Deferral on a class-level (constructor) data source: 5 class instances x 1 method = 5 cases + // + 1 placeholder container = 6. + await RunTestsWithFilter( + "/*/*/DeferEnumerationClassDataTests/*", + [ + result => result.ResultSummary.Outcome.ShouldBe("Completed"), + result => result.ResultSummary.Counters.Total.ShouldBe(6), + result => result.ResultSummary.Counters.Passed.ShouldBe(6), + result => result.ResultSummary.Counters.Failed.ShouldBe(0) + ]); + } + + [Test] + public async Task Deferred_Data_Source_Error_Surfaces_At_Runtime_Without_Crashing_Discovery() + { + // The throwing data source must not crash discovery; the error surfaces as a failed case, and the + // placeholder container aggregates to failed too (failed case + failed container = 2). + await RunTestsWithFilter( + "/*/*/DeferEnumerationErrorTests/*", + [ + result => result.ResultSummary.Counters.Failed.ShouldBe(2) + ]); + } +} diff --git a/TUnit.Engine/Building/TestBuilder.cs b/TUnit.Engine/Building/TestBuilder.cs index 8a9847b0a8..b41cff79e4 100644 --- a/TUnit.Engine/Building/TestBuilder.cs +++ b/TUnit.Engine/Building/TestBuilder.cs @@ -189,6 +189,16 @@ public async Task> BuildTestsFromMetadataAsy var contextAccessor = new TestBuilderContextAccessor(testBuilderContext); + // DeferEnumeration: emit a single placeholder node instead of enumerating the data source(s). + // The placeholder is built during both discovery and the execution build (so it matches the + // IDE's UID filter); DeferredTestExpander later re-runs this method with IgnoreDeferral set to + // produce the real cases. Skipped entirely when IgnoreDeferral is set (the expansion pass). + if (!buildingContext.IgnoreDeferral && HasDeferredDataSource(metadata)) + { + tests.Add(await BuildDeferredPlaceholderAsync(metadata, testBuilderContext, cancellationToken)); + return tests; + } + var classDataAttributeIndex = 0; foreach (var classDataSource in await GetDataSourcesAsync(metadata.ClassDataSources, cancellationToken)) { @@ -1098,6 +1108,67 @@ private Task InvokeDiscoveryEventReceiversAsync(TestContext context) return _eventReceiverOrchestrator.InvokeTestDiscoveryEventReceiversAsync(context, discoveredContext, CancellationToken.None); } + private static bool HasDeferredDataSource(TestMetadata metadata) + { + foreach (var dataSource in metadata.DataSources) + { + if (dataSource.DeferEnumeration) + { + return true; + } + } + + foreach (var dataSource in metadata.ClassDataSources) + { + if (dataSource.DeferEnumeration) + { + return true; + } + } + + return false; + } + + /// + /// Builds the single placeholder node for a test whose data source enumeration is deferred. + /// The placeholder carries no class/method data and is never executed as a real test — it exists + /// so discovery shows one node, and DeferredTestExpander uses its to build + /// the real cases at runtime. + /// + private async Task BuildDeferredPlaceholderAsync( + TestMetadata metadata, + TestBuilderContext testBuilderContext, + CancellationToken cancellationToken) + { + var testId = TestIdentifierService.GenerateDeferredPlaceholderTestId(metadata); + + var testData = new TestData + { + TestClassInstanceFactory = static () => Task.FromResult(PlaceholderInstance.Instance), + ClassDataSourceAttributeIndex = 0, + ClassDataLoopIndex = 0, + ClassData = [], + MethodDataSourceAttributeIndex = 0, + MethodDataLoopIndex = 0, + MethodData = [], + RepeatIndex = 0, + InheritanceDepth = metadata.InheritanceDepth, + ResolvedClassGenericArguments = Type.EmptyTypes, + ResolvedMethodGenericArguments = Type.EmptyTypes + }; + + var context = await CreateTestContextAsync(testId, metadata, testData, testBuilderContext, cancellationToken); + + return new DeferredEnumerationExecutableTest + { + TestId = testId, + Metadata = metadata, + Arguments = [], + ClassArguments = [], + Context = context + }; + } + private AbstractExecutableTest CreateFailedTestForDataGenerationError(TestMetadata metadata, Exception exception) { return CreateFailedTestForDataGenerationError(metadata, exception, new TestDataCombination()); diff --git a/TUnit.Engine/Building/TestBuildingContext.cs b/TUnit.Engine/Building/TestBuildingContext.cs index b48669f9af..06e29073c3 100644 --- a/TUnit.Engine/Building/TestBuildingContext.cs +++ b/TUnit.Engine/Building/TestBuildingContext.cs @@ -15,5 +15,12 @@ internal record TestBuildingContext( /// /// The filter to apply during test building. Only relevant when IsForExecution is true. /// - ITestExecutionFilter? Filter + ITestExecutionFilter? Filter, + + /// + /// When true, data sources marked with DeferEnumeration are expanded eagerly instead of + /// producing a single placeholder node. Set during runtime expansion of a deferred placeholder so + /// the real test cases are built (see DeferredTestExpander). + /// + bool IgnoreDeferral = false ); diff --git a/TUnit.Engine/DeferredEnumerationExecutableTest.cs b/TUnit.Engine/DeferredEnumerationExecutableTest.cs new file mode 100644 index 0000000000..36cbf95986 --- /dev/null +++ b/TUnit.Engine/DeferredEnumerationExecutableTest.cs @@ -0,0 +1,22 @@ +using TUnit.Core; + +namespace TUnit.Engine; + +/// +/// Placeholder test that stands in for a data-driven test whose data source is marked +/// DeferEnumeration = true. A single one of these is produced during discovery so the IDE shows +/// one node instead of expanding potentially thousands of cases. At execution time it is never run as a +/// real test — expands it into the real test cases, which are +/// scheduled and reported nested under this placeholder. The Create/Invoke members therefore throw: if one +/// is ever reached it means the placeholder leaked into the normal execution path, which is a bug. +/// +internal sealed class DeferredEnumerationExecutableTest : AbstractExecutableTest +{ + public override Task CreateInstanceAsync() => + throw new InvalidOperationException( + $"Deferred enumeration placeholder '{TestId}' was scheduled directly. It must be expanded by DeferredTestExpander before execution."); + + public override Task InvokeTestAsync(object instance, CancellationToken cancellationToken) => + throw new InvalidOperationException( + $"Deferred enumeration placeholder '{TestId}' was invoked directly. It must be expanded by DeferredTestExpander before execution."); +} diff --git a/TUnit.Engine/EmptyDataSourceAttribute.cs b/TUnit.Engine/EmptyDataSourceAttribute.cs index 5978bea49c..818a1c6233 100644 --- a/TUnit.Engine/EmptyDataSourceAttribute.cs +++ b/TUnit.Engine/EmptyDataSourceAttribute.cs @@ -7,6 +7,10 @@ internal class EmptyDataSourceAttribute : IDataSourceAttribute /// public bool SkipIfEmpty { get; set; } + /// + // Always a single (empty) row, so deferring its enumeration would be pure overhead. + public bool DeferEnumeration { get => false; set { } } + public async IAsyncEnumerable>> GetDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata) { yield return () => Task.FromResult([]); diff --git a/TUnit.Engine/Framework/TUnitServiceProvider.cs b/TUnit.Engine/Framework/TUnitServiceProvider.cs index cd937f037d..94edfe3850 100644 --- a/TUnit.Engine/Framework/TUnitServiceProvider.cs +++ b/TUnit.Engine/Framework/TUnitServiceProvider.cs @@ -263,6 +263,8 @@ public TUnitServiceProvider(IExtension extension, var dynamicTestQueue = Register(new DynamicTestQueue(MessageBus)); + var deferredTestExpander = Register(new DeferredTestExpander(TestBuilderPipeline, TestFilterService)); + var testScheduler = Register(new TestScheduler( Logger, testGroupingService, @@ -286,7 +288,8 @@ public TUnitServiceProvider(IExtension extension, lifecycleCoordinator, MessageBus, staticPropertyInitializer, - objectTracker)); + objectTracker, + deferredTestExpander)); Register(new TestRegistry(TestBuilderPipeline, testCoordinator, dynamicTestQueue, TestFilterService, TestSessionId, CancellationToken.Token)); diff --git a/TUnit.Engine/Services/DeferredTestExpander.cs b/TUnit.Engine/Services/DeferredTestExpander.cs new file mode 100644 index 0000000000..35e9d1121d --- /dev/null +++ b/TUnit.Engine/Services/DeferredTestExpander.cs @@ -0,0 +1,64 @@ +using System.Diagnostics.CodeAnalysis; +using TUnit.Core; +using TUnit.Core.Enums; +using TUnit.Engine.Building; + +namespace TUnit.Engine.Services; + +/// +/// Expands a deferred-enumeration placeholder (see ) into +/// its real test cases at the start of execution. The data source is only deferred for discovery +/// (so the IDE shows one node instead of thousands); at run time the rows are enumerated normally and the +/// resulting tests are scheduled through the standard pipeline, so they get correct hooks and lifecycle +/// counting. The children are reported nested under the placeholder via their ParentTestId. +/// +internal sealed class DeferredTestExpander +{ + private readonly TestBuilderPipeline _testBuilderPipeline; + private readonly TestFilterService _testFilterService; + + public DeferredTestExpander(TestBuilderPipeline testBuilderPipeline, TestFilterService testFilterService) + { + _testBuilderPipeline = testBuilderPipeline; + _testFilterService = testFilterService; + } + + /// + /// Builds the real test cases for a deferred placeholder, registers them for execution (event + /// receivers + argument/reference tracking), and returns them. Each child is parented to the + /// placeholder so reporters can nest them underneath it. + /// +#if NET8_0_OR_GREATER + [RequiresUnreferencedCode("Building tests in reflection mode uses generic type resolution which requires unreferenced code")] +#endif + public async Task> ExpandAsync( + AbstractExecutableTest placeholder, + CancellationToken cancellationToken) + { + // IgnoreDeferral re-runs the normal class x method x repeat expansion that discovery skipped. + // Filter is null: these cases were selected by the placeholder matching the run filter, so the + // children must not be re-filtered (they have different ids than the placeholder, like dynamic tests). + // IsForExecution: false matches the runtime-built-test precedent (TestRegistry / BuildTestsAsync); + // in the build path it only gates a pre-filter optimisation that is a no-op when Filter is null. + // Execution registration is done explicitly below via RegisterTestsAsync(isForExecution: true). + var buildingContext = new TestBuildingContext(IsForExecution: false, Filter: null, IgnoreDeferral: true); + + var built = await _testBuilderPipeline + .BuildTestsFromMetadataAsync([placeholder.Metadata], buildingContext, cancellationToken) + .ConfigureAwait(false); + + var children = built as IReadOnlyList ?? built.ToList(); + + foreach (var child in children) + { + child.Context.ParentTestId = placeholder.TestId; + child.Context.Relationship = TestRelationship.Generated; + } + + // Children bypassed the discovery pipeline's post-filter registration (they didn't exist then), + // so register them here before they are scheduled - mirrors TestRegistry's dynamic-test path. + await _testFilterService.RegisterTestsAsync(children, isForExecution: true).ConfigureAwait(false); + + return children; + } +} diff --git a/TUnit.Engine/Services/TestIdentifierService.cs b/TUnit.Engine/Services/TestIdentifierService.cs index 54652f1432..63e7b4c260 100644 --- a/TUnit.Engine/Services/TestIdentifierService.cs +++ b/TUnit.Engine/Services/TestIdentifierService.cs @@ -70,6 +70,50 @@ public static string GenerateTestId(TestMetadata metadata, TestBuilder.TestData } } + /// + /// Generates a stable id for a deferred-enumeration placeholder (a single node that stands in for a + /// data source whose rows are expanded at runtime). Deterministic from metadata alone so it is + /// identical across the discovery build and the execution build (the IDE "run" filter carries this id). + /// + /// + /// Cannot collide with a real test id despite the shared _Deferred token: every real id from + /// has the form ...(ctorParams).{classIdx}.{classLoop}.Method(params).{methodIdx}.{methodLoop}.{repeat} + /// — i.e. the method name is always preceded by two numeric data-index segments and followed by three + /// more. The placeholder id places the method name directly after the constructor params and ends in + /// _Deferred with none of those numeric segments, so the two id shapes can never be equal (a + /// method literally named Foo_Deferred still produces ...Foo_Deferred(params).i.j.k, not + /// ...Foo(params)_Deferred). Two placeholders only match if their method+params match, which + /// means the same method — impossible to declare twice. + /// + public static string GenerateDeferredPlaceholderTestId(TestMetadata metadata) + { + var methodMetadata = metadata.MethodMetadata; + var classMetadata = methodMetadata.Class; + + var constructorParameters = classMetadata.Parameters; + var methodParameters = methodMetadata.Parameters; + + var vsb = new ValueStringBuilder(stackalloc char[256]); + + try + { + vsb.Append(methodMetadata.Class.Namespace); + vsb.Append('.'); + WriteTypeNameWithGenerics(ref vsb, metadata.TestClassType); + WriteTypeWithParameters(ref vsb, constructorParameters); + vsb.Append('.'); + vsb.Append(metadata.TestMethodName); + WriteTypeWithParameters(ref vsb, methodParameters); + vsb.Append("_Deferred"); + + return vsb.ToString(); + } + finally + { + vsb.Dispose(); + } + } + public static string GenerateFailedTestId(TestMetadata metadata) { // For backward compatibility, use default combination values diff --git a/TUnit.Engine/TUnitMessageBus.cs b/TUnit.Engine/TUnitMessageBus.cs index c229367024..9608d8be7e 100644 --- a/TUnit.Engine/TUnitMessageBus.cs +++ b/TUnit.Engine/TUnitMessageBus.cs @@ -25,6 +25,17 @@ internal class TUnitMessageBus(IExtension extension, ICommandLineOptions command private bool? _isConsole; private bool IsConsole => _isConsole ??= serviceProvider.GetClientInfo().Id.Contains("console", StringComparison.InvariantCultureIgnoreCase); + // Tests created by expanding a deferred-enumeration placeholder (or runtime variants) carry a + // ParentTestId; surfacing it as the MTP parentTestNodeUid makes IDEs nest them under that node. + private TestNodeUpdateMessage CreateUpdateMessage(TestContext testContext, TestNode testNode) + { + var parentTestId = testContext.ParentTestId; + + return parentTestId is null + ? new TestNodeUpdateMessage(_sessionSessionUid, testNode) + : new TestNodeUpdateMessage(_sessionSessionUid, testNode, new TestNodeUid(parentTestId)); + } + public ValueTask Discovered(TestContext testContext) { if (testContext.IsNotDiscoverable) @@ -32,18 +43,14 @@ public ValueTask Discovered(TestContext testContext) return ValueTask.CompletedTask; } - return new ValueTask(context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( - sessionUid: _sessionSessionUid, - testNode: testContext.ToTestNode(DiscoveredTestNodeStateProperty.CachedInstance) - ))); + return new ValueTask(context.MessageBus.PublishAsync(this, + CreateUpdateMessage(testContext, testContext.ToTestNode(DiscoveredTestNodeStateProperty.CachedInstance)))); } public ValueTask InProgress(TestContext testContext) { - return new ValueTask(context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( - sessionUid: _sessionSessionUid, - testNode: testContext.ToTestNode(InProgressTestNodeStateProperty.CachedInstance) - ))); + return new ValueTask(context.MessageBus.PublishAsync(this, + CreateUpdateMessage(testContext, testContext.ToTestNode(InProgressTestNodeStateProperty.CachedInstance)))); } public ValueTask Passed(TestContext testContext, DateTimeOffset start) @@ -55,10 +62,7 @@ public ValueTask Passed(TestContext testContext, DateTimeOffset start) var testNode = testContext.ToTestNode(PassedTestNodeStateProperty.CachedInstance); - return new ValueTask(context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( - sessionUid: _sessionSessionUid, - testNode: testNode - ))); + return new ValueTask(context.MessageBus.PublishAsync(this, CreateUpdateMessage(testContext, testNode))); } public ValueTask Failed(TestContext testContext, Exception exception, DateTimeOffset start) @@ -76,10 +80,7 @@ public ValueTask Failed(TestContext testContext, Exception exception, DateTimeOf var testNode = testContext.ToTestNode(updateType); - return new ValueTask(context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( - sessionUid: _sessionSessionUid, - testNode: testNode - ))); + return new ValueTask(context.MessageBus.PublishAsync(this, CreateUpdateMessage(testContext, testNode))); } private Exception SimplifyStacktrace(Exception exception) @@ -106,10 +107,7 @@ public ValueTask Skipped(TestContext testContext, string reason) { var testNode = testContext.ToTestNode(new SkippedTestNodeStateProperty(reason)); - return new ValueTask(context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( - sessionUid: _sessionSessionUid, - testNode: testNode - ))); + return new ValueTask(context.MessageBus.PublishAsync(this, CreateUpdateMessage(testContext, testNode))); } public ValueTask Cancelled(TestContext testContext, DateTimeOffset start) @@ -118,10 +116,7 @@ public ValueTask Cancelled(TestContext testContext, DateTimeOffset start) var testNode = testContext.ToTestNode(new CancelledTestNodeStateProperty()); #pragma warning restore CS0618 - return new ValueTask(context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage( - sessionUid: _sessionSessionUid, - testNode: testNode - ))); + return new ValueTask(context.MessageBus.PublishAsync(this, CreateUpdateMessage(testContext, testNode))); } public ValueTask SessionArtifact(Artifact artifact) diff --git a/TUnit.Engine/TestSessionCoordinator.cs b/TUnit.Engine/TestSessionCoordinator.cs index c53a47ddfb..2a306319a9 100644 --- a/TUnit.Engine/TestSessionCoordinator.cs +++ b/TUnit.Engine/TestSessionCoordinator.cs @@ -24,6 +24,7 @@ internal sealed class TestSessionCoordinator : ITestExecutor, IDisposable, IAsyn private readonly ITUnitMessageBus _messageBus; private readonly IStaticPropertyInitializer _staticPropertyInitializer; private readonly ObjectTracker _objectTracker; + private readonly DeferredTestExpander _deferredTestExpander; public TestSessionCoordinator(EventReceiverOrchestrator eventReceiverOrchestrator, TUnitFrameworkLogger logger, @@ -33,7 +34,8 @@ public TestSessionCoordinator(EventReceiverOrchestrator eventReceiverOrchestrato TestLifecycleCoordinator lifecycleCoordinator, ITUnitMessageBus messageBus, IStaticPropertyInitializer staticPropertyInitializer, - ObjectTracker objectTracker) + ObjectTracker objectTracker, + DeferredTestExpander deferredTestExpander) { _eventReceiverOrchestrator = eventReceiverOrchestrator; _logger = logger; @@ -44,6 +46,7 @@ public TestSessionCoordinator(EventReceiverOrchestrator eventReceiverOrchestrato _testScheduler = testScheduler; _staticPropertyInitializer = staticPropertyInitializer; _objectTracker = objectTracker; + _deferredTestExpander = deferredTestExpander; } public async Task ExecuteTests( @@ -54,12 +57,20 @@ public async Task ExecuteTests( { var testList = tests.ToList(); + // Expand any deferred-enumeration placeholders into their real cases before counting/scheduling, + // so the children flow through the normal pipeline (correct hooks + lifecycle counting). + var expandedPlaceholders = await ExpandDeferredPlaceholdersAsync(testList, cancellationToken); + InitializeEventReceivers(testList, cancellationToken); try { await PrepareTestOrchestrator(testList, cancellationToken); await ExecuteTestsCore(testList, cancellationToken); + + // Children have now run: resolve each placeholder container to the aggregate of its cases so the + // IDE node the user ran gets a result (rather than "not run") that reflects its children. + await ReportDeferredPlaceholderResultsAsync(expandedPlaceholders); } finally { @@ -92,6 +103,116 @@ private void InitializeEventReceivers(List testList, Can _eventReceiverOrchestrator.InitializeTestCounts(testList); } + /// + /// Replaces every in the list with the real test cases + /// produced by enumerating its data source. The placeholder is reported as a running container now (so the + /// IDE node the user ran shows as in-progress rather than "not run") and resolved to the aggregate of its + /// children later by ; the children are added to the + /// list, scheduled like any other test, and nested under the placeholder via their ParentTestId. If the + /// expansion itself throws, the placeholder is reported failed immediately. (Per-row data errors surface + /// as their own failed child via the standard data-generation-error path.) The returned list pairs each + /// successfully-expanded placeholder with its children so its final result can be reported post-run. + /// +#if NET8_0_OR_GREATER + [UnconditionalSuppressMessage("Trimming", "IL2026", Justification = "Reflection mode is not used in AOT/trimmed scenarios")] + [UnconditionalSuppressMessage("AOT", "IL3050", Justification = "Reflection mode is not used in AOT scenarios")] +#endif + private async Task Children)>> ExpandDeferredPlaceholdersAsync( + List testList, CancellationToken cancellationToken) + { + List? placeholders = null; + foreach (var test in testList) + { + if (test is DeferredEnumerationExecutableTest placeholder) + { + (placeholders ??= []).Add(placeholder); + } + } + + if (placeholders is null) + { + return []; + } + + // Drop all placeholders in a single pass so none are scheduled as real tests (their Create/Invoke + // throw); their children are added back below. + testList.RemoveAll(static t => t is DeferredEnumerationExecutableTest); + + var expanded = new List<(AbstractExecutableTest, IReadOnlyList)>(placeholders.Count); + + foreach (var placeholder in placeholders) + { + placeholder.StartTime = DateTimeOffset.UtcNow; + placeholder.State = TestState.Running; + await _messageBus.InProgress(placeholder.Context); + + try + { + var children = await _deferredTestExpander.ExpandAsync(placeholder, cancellationToken); + testList.AddRange(children); + expanded.Add((placeholder, children)); + } + catch (Exception ex) + { + // Expansion itself failed (as opposed to a per-row data error, which becomes a failed + // child). Surface it on the placeholder node so the failure is visible. + await _logger.LogErrorAsync($"Failed to expand deferred test '{placeholder.TestId}': {ex}"); + placeholder.EndTime = DateTimeOffset.UtcNow; + placeholder.SetResult(TestState.Failed, ex); + await _messageBus.Failed(placeholder.Context, ex, placeholder.StartTime.GetValueOrDefault()); + } + } + + return expanded; + } + + /// + /// Reports the final result for each deferred placeholder once its children have executed: the placeholder + /// is a container whose outcome is the aggregate of its cases — failed if any case failed, skipped if every + /// case was skipped, otherwise passed. This resolves the IDE node the user ran without masking child + /// failures (which a fixed "passed" would). + /// + private async Task ReportDeferredPlaceholderResultsAsync( + List<(AbstractExecutableTest Placeholder, IReadOnlyList Children)> expandedPlaceholders) + { + foreach (var (placeholder, children) in expandedPlaceholders) + { + placeholder.EndTime = DateTimeOffset.UtcNow; + + var failedCount = 0; + var skippedCount = 0; + foreach (var child in children) + { + switch (child.State) + { + case TestState.Failed or TestState.Timeout or TestState.Cancelled: + failedCount++; + break; + case TestState.Skipped: + skippedCount++; + break; + } + } + + if (failedCount > 0) + { + var exception = new Exception($"{failedCount} of {children.Count} deferred test case(s) failed."); + placeholder.SetResult(TestState.Failed, exception); + await _messageBus.Failed(placeholder.Context, exception, placeholder.StartTime.GetValueOrDefault()); + } + else if (children.Count > 0 && skippedCount == children.Count) + { + placeholder.State = TestState.Skipped; + await _messageBus.Skipped(placeholder.Context, "All deferred test cases were skipped"); + } + else + { + placeholder.SetResult(TestState.Passed); + await _messageBus.Passed(placeholder.Context, placeholder.StartTime.GetValueOrDefault()); + } + } + } + private async Task PrepareTestOrchestrator(List testList, CancellationToken cancellationToken) { // Register all tests upfront so orchestrator knows total counts per class/assembly for lifecycle management diff --git a/TUnit.FsCheck/FsCheckPropertyAttribute.cs b/TUnit.FsCheck/FsCheckPropertyAttribute.cs index 7433a2618a..d75c38ff61 100644 --- a/TUnit.FsCheck/FsCheckPropertyAttribute.cs +++ b/TUnit.FsCheck/FsCheckPropertyAttribute.cs @@ -78,6 +78,12 @@ public class FsCheckPropertyAttribute : Attribute, ITestRegisteredEventReceiver, /// public bool SkipIfEmpty { get; set; } + /// + /// Not used - FsCheck generates its own data during test execution (there are no discovery-time rows + /// to defer), so this is always false. Exists to satisfy the IDataSourceAttribute interface. + /// + public bool DeferEnumeration { get => false; set { } } + /// /// Called when the test is registered. Sets up the FsCheck property executor. /// diff --git a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt index 083a180c91..2c994e0f87 100644 --- a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt +++ b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet10_0.verified.txt @@ -79,12 +79,13 @@ namespace { public ArgumentsAttribute(params object?[]? values) { } public string[]? Categories { get; set; } + public bool DeferEnumeration { get; set; } public string? DisplayName { get; set; } public int Order { get; } public string? Skip { get; set; } public bool SkipIfEmpty { get; set; } public object?[] Values { get; } - [.(typeof(.ArgumentsAttribute.d__20))] + [.(typeof(.ArgumentsAttribute.d__23))] public .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } public . OnTestRegistered(.TestRegisteredContext context) { } } @@ -93,11 +94,12 @@ namespace { public ArgumentsAttribute(T value) { } public string[]? Categories { get; set; } + public override bool DeferEnumeration { get; set; } public string? DisplayName { get; set; } public int Order { get; } public string? Skip { get; set; } public override bool SkipIfEmpty { get; set; } - [.(typeof(.ArgumentsAttribute.d__18))] + [.(typeof(.ArgumentsAttribute.d__21))] public override .<<.>> GetTypedDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } public . OnTestRegistered(.TestRegisteredContext context) { } } @@ -194,6 +196,7 @@ namespace public abstract class AsyncUntypedDataSourceGeneratorAttribute : , .IDataSourceAttribute { protected AsyncUntypedDataSourceGeneratorAttribute() { } + public virtual bool DeferEnumeration { get; set; } public virtual bool SkipIfEmpty { get; set; } public .<<.>> GenerateAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } protected abstract .<<.>> GenerateDataSourcesAsync(.DataGeneratorMetadata dataGeneratorMetadata); @@ -883,6 +886,7 @@ namespace public interface IAccessesInstanceData { } public interface IDataSourceAttribute { + bool DeferEnumeration { get; set; } bool SkipIfEmpty { get; set; } .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata); } @@ -1010,6 +1014,7 @@ namespace public object?[] Arguments { get; set; } [.(..None | ..PublicParameterlessConstructor | ..PublicConstructors | ..PublicMethods | ..NonPublicMethods | ..PublicProperties)] public ? ClassProvidingDataSource { get; } + public bool DeferEnumeration { get; set; } public <.DataGeneratorMetadata, .<<.>>>? Factory { get; set; } public string MethodNameProvidingDataSource { get; } public bool SkipIfEmpty { get; set; } @@ -1017,7 +1022,7 @@ namespace "rty.")] [.("Trimming", "IL2075", Justification="Method data sources require runtime discovery. AOT users should use Factory prope" + "rty.")] - [.(typeof(.MethodDataSourceAttribute.d__22))] + [.(typeof(.MethodDataSourceAttribute.d__26))] public .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } } [(.Class | .Method | .Property | .Parameter, AllowMultiple=true)] @@ -1755,8 +1760,9 @@ namespace public abstract class TypedDataSourceAttribute : , .IDataSourceAttribute, .ITypedDataSourceAttribute { protected TypedDataSourceAttribute() { } + public virtual bool DeferEnumeration { get; set; } public virtual bool SkipIfEmpty { get; set; } - [.(typeof(.TypedDataSourceAttribute.d__5))] + [.(typeof(.TypedDataSourceAttribute.d__9))] public .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } public abstract .<<.>> GetTypedDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata); } diff --git a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt index 4bcd0f9540..32f9dfdd4d 100644 --- a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt +++ b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet8_0.verified.txt @@ -79,12 +79,13 @@ namespace { public ArgumentsAttribute(params object?[]? values) { } public string[]? Categories { get; set; } + public bool DeferEnumeration { get; set; } public string? DisplayName { get; set; } public int Order { get; } public string? Skip { get; set; } public bool SkipIfEmpty { get; set; } public object?[] Values { get; } - [.(typeof(.ArgumentsAttribute.d__20))] + [.(typeof(.ArgumentsAttribute.d__23))] public .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } public . OnTestRegistered(.TestRegisteredContext context) { } } @@ -93,11 +94,12 @@ namespace { public ArgumentsAttribute(T value) { } public string[]? Categories { get; set; } + public override bool DeferEnumeration { get; set; } public string? DisplayName { get; set; } public int Order { get; } public string? Skip { get; set; } public override bool SkipIfEmpty { get; set; } - [.(typeof(.ArgumentsAttribute.d__18))] + [.(typeof(.ArgumentsAttribute.d__21))] public override .<<.>> GetTypedDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } public . OnTestRegistered(.TestRegisteredContext context) { } } @@ -194,6 +196,7 @@ namespace public abstract class AsyncUntypedDataSourceGeneratorAttribute : , .IDataSourceAttribute { protected AsyncUntypedDataSourceGeneratorAttribute() { } + public virtual bool DeferEnumeration { get; set; } public virtual bool SkipIfEmpty { get; set; } public .<<.>> GenerateAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } protected abstract .<<.>> GenerateDataSourcesAsync(.DataGeneratorMetadata dataGeneratorMetadata); @@ -883,6 +886,7 @@ namespace public interface IAccessesInstanceData { } public interface IDataSourceAttribute { + bool DeferEnumeration { get; set; } bool SkipIfEmpty { get; set; } .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata); } @@ -1010,6 +1014,7 @@ namespace public object?[] Arguments { get; set; } [.(..None | ..PublicParameterlessConstructor | ..PublicConstructors | ..PublicMethods | ..NonPublicMethods | ..PublicProperties)] public ? ClassProvidingDataSource { get; } + public bool DeferEnumeration { get; set; } public <.DataGeneratorMetadata, .<<.>>>? Factory { get; set; } public string MethodNameProvidingDataSource { get; } public bool SkipIfEmpty { get; set; } @@ -1017,7 +1022,7 @@ namespace "rty.")] [.("Trimming", "IL2075", Justification="Method data sources require runtime discovery. AOT users should use Factory prope" + "rty.")] - [.(typeof(.MethodDataSourceAttribute.d__22))] + [.(typeof(.MethodDataSourceAttribute.d__26))] public .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } } [(.Class | .Method | .Property | .Parameter, AllowMultiple=true)] @@ -1755,8 +1760,9 @@ namespace public abstract class TypedDataSourceAttribute : , .IDataSourceAttribute, .ITypedDataSourceAttribute { protected TypedDataSourceAttribute() { } + public virtual bool DeferEnumeration { get; set; } public virtual bool SkipIfEmpty { get; set; } - [.(typeof(.TypedDataSourceAttribute.d__5))] + [.(typeof(.TypedDataSourceAttribute.d__9))] public .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } public abstract .<<.>> GetTypedDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata); } diff --git a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt index 4367ec7d53..78903c50f7 100644 --- a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt +++ b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.DotNet9_0.verified.txt @@ -79,12 +79,13 @@ namespace { public ArgumentsAttribute(params object?[]? values) { } public string[]? Categories { get; set; } + public bool DeferEnumeration { get; set; } public string? DisplayName { get; set; } public int Order { get; } public string? Skip { get; set; } public bool SkipIfEmpty { get; set; } public object?[] Values { get; } - [.(typeof(.ArgumentsAttribute.d__20))] + [.(typeof(.ArgumentsAttribute.d__23))] public .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } public . OnTestRegistered(.TestRegisteredContext context) { } } @@ -93,11 +94,12 @@ namespace { public ArgumentsAttribute(T value) { } public string[]? Categories { get; set; } + public override bool DeferEnumeration { get; set; } public string? DisplayName { get; set; } public int Order { get; } public string? Skip { get; set; } public override bool SkipIfEmpty { get; set; } - [.(typeof(.ArgumentsAttribute.d__18))] + [.(typeof(.ArgumentsAttribute.d__21))] public override .<<.>> GetTypedDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } public . OnTestRegistered(.TestRegisteredContext context) { } } @@ -194,6 +196,7 @@ namespace public abstract class AsyncUntypedDataSourceGeneratorAttribute : , .IDataSourceAttribute { protected AsyncUntypedDataSourceGeneratorAttribute() { } + public virtual bool DeferEnumeration { get; set; } public virtual bool SkipIfEmpty { get; set; } public .<<.>> GenerateAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } protected abstract .<<.>> GenerateDataSourcesAsync(.DataGeneratorMetadata dataGeneratorMetadata); @@ -883,6 +886,7 @@ namespace public interface IAccessesInstanceData { } public interface IDataSourceAttribute { + bool DeferEnumeration { get; set; } bool SkipIfEmpty { get; set; } .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata); } @@ -1010,6 +1014,7 @@ namespace public object?[] Arguments { get; set; } [.(..None | ..PublicParameterlessConstructor | ..PublicConstructors | ..PublicMethods | ..NonPublicMethods | ..PublicProperties)] public ? ClassProvidingDataSource { get; } + public bool DeferEnumeration { get; set; } public <.DataGeneratorMetadata, .<<.>>>? Factory { get; set; } public string MethodNameProvidingDataSource { get; } public bool SkipIfEmpty { get; set; } @@ -1017,7 +1022,7 @@ namespace "rty.")] [.("Trimming", "IL2075", Justification="Method data sources require runtime discovery. AOT users should use Factory prope" + "rty.")] - [.(typeof(.MethodDataSourceAttribute.d__22))] + [.(typeof(.MethodDataSourceAttribute.d__26))] public .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } } [(.Class | .Method | .Property | .Parameter, AllowMultiple=true)] @@ -1755,8 +1760,9 @@ namespace public abstract class TypedDataSourceAttribute : , .IDataSourceAttribute, .ITypedDataSourceAttribute { protected TypedDataSourceAttribute() { } + public virtual bool DeferEnumeration { get; set; } public virtual bool SkipIfEmpty { get; set; } - [.(typeof(.TypedDataSourceAttribute.d__5))] + [.(typeof(.TypedDataSourceAttribute.d__9))] public .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } public abstract .<<.>> GetTypedDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata); } diff --git a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt index 249b80bc79..f29185fd84 100644 --- a/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt +++ b/TUnit.PublicAPI/Tests.Core_Library_Has_No_API_Changes.Net4_7.verified.txt @@ -79,12 +79,13 @@ namespace { public ArgumentsAttribute(params object?[]? values) { } public string[]? Categories { get; set; } + public bool DeferEnumeration { get; set; } public string? DisplayName { get; set; } public int Order { get; } public string? Skip { get; set; } public bool SkipIfEmpty { get; set; } public object?[] Values { get; } - [.(typeof(.ArgumentsAttribute.d__20))] + [.(typeof(.ArgumentsAttribute.d__23))] public .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } public . OnTestRegistered(.TestRegisteredContext context) { } } @@ -93,11 +94,12 @@ namespace { public ArgumentsAttribute(T value) { } public string[]? Categories { get; set; } + public override bool DeferEnumeration { get; set; } public string? DisplayName { get; set; } public int Order { get; } public string? Skip { get; set; } public override bool SkipIfEmpty { get; set; } - [.(typeof(.ArgumentsAttribute.d__18))] + [.(typeof(.ArgumentsAttribute.d__21))] public override .<<.>> GetTypedDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } public . OnTestRegistered(.TestRegisteredContext context) { } } @@ -191,6 +193,7 @@ namespace public abstract class AsyncUntypedDataSourceGeneratorAttribute : , .IDataSourceAttribute { protected AsyncUntypedDataSourceGeneratorAttribute() { } + public virtual bool DeferEnumeration { get; set; } public virtual bool SkipIfEmpty { get; set; } public .<<.>> GenerateAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } protected abstract .<<.>> GenerateDataSourcesAsync(.DataGeneratorMetadata dataGeneratorMetadata); @@ -860,6 +863,7 @@ namespace public interface IAccessesInstanceData { } public interface IDataSourceAttribute { + bool DeferEnumeration { get; set; } bool SkipIfEmpty { get; set; } .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata); } @@ -978,10 +982,11 @@ namespace public MethodDataSourceAttribute( classProvidingDataSource, string methodNameProvidingDataSource) { } public object?[] Arguments { get; set; } public ? ClassProvidingDataSource { get; } + public bool DeferEnumeration { get; set; } public <.DataGeneratorMetadata, .<<.>>>? Factory { get; set; } public string MethodNameProvidingDataSource { get; } public bool SkipIfEmpty { get; set; } - [.(typeof(.MethodDataSourceAttribute.d__22))] + [.(typeof(.MethodDataSourceAttribute.d__26))] public .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } } [(.Class | .Method | .Property | .Parameter, AllowMultiple=true)] @@ -1694,8 +1699,9 @@ namespace public abstract class TypedDataSourceAttribute : , .IDataSourceAttribute, .ITypedDataSourceAttribute { protected TypedDataSourceAttribute() { } + public virtual bool DeferEnumeration { get; set; } public virtual bool SkipIfEmpty { get; set; } - [.(typeof(.TypedDataSourceAttribute.d__5))] + [.(typeof(.TypedDataSourceAttribute.d__9))] public .<<.>> GetDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata) { } public abstract .<<.>> GetTypedDataRowsAsync(.DataGeneratorMetadata dataGeneratorMetadata); } diff --git a/TUnit.SourceGenerator.Benchmarks/TUnit.SourceGenerator.Benchmarks.csproj b/TUnit.SourceGenerator.Benchmarks/TUnit.SourceGenerator.Benchmarks.csproj index 776b8a3035..c0ba473746 100644 --- a/TUnit.SourceGenerator.Benchmarks/TUnit.SourceGenerator.Benchmarks.csproj +++ b/TUnit.SourceGenerator.Benchmarks/TUnit.SourceGenerator.Benchmarks.csproj @@ -12,6 +12,7 @@ + diff --git a/TUnit.TestProject/ClassDataSourceTupleTests.cs b/TUnit.TestProject/ClassDataSourceTupleTests.cs index 4986fc235e..74fc00e079 100644 --- a/TUnit.TestProject/ClassDataSourceTupleTests.cs +++ b/TUnit.TestProject/ClassDataSourceTupleTests.cs @@ -5,6 +5,8 @@ public class TupleDataSource : IDataSourceAttribute { public bool SkipIfEmpty { get; set; } + public bool DeferEnumeration { get; set; } + public async IAsyncEnumerable>> GetDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata) { // Return tuples that should be unwrapped into constructor parameters diff --git a/TUnit.TestProject/DeferEnumerationTests/DeferEnumerationClassDataTests.cs b/TUnit.TestProject/DeferEnumerationTests/DeferEnumerationClassDataTests.cs new file mode 100644 index 0000000000..b471ae3dc3 --- /dev/null +++ b/TUnit.TestProject/DeferEnumerationTests/DeferEnumerationClassDataTests.cs @@ -0,0 +1,15 @@ +namespace TUnit.TestProject.DeferEnumerationTests; + +// Exercises DeferEnumeration on a CLASS-level (constructor) data source, so the deferral flows through +// the metadata.ClassDataSources path. 5 class instances x 1 method = 5 cases (+ 1 placeholder container). +[MethodDataSource(nameof(ClassValues), DeferEnumeration = true)] +public class DeferEnumerationClassDataTests(int value) +{ + public static IEnumerable ClassValues() => Enumerable.Range(0, 5); + + [Test] + public async Task ClassDeferred() + { + await Assert.That(value).IsGreaterThanOrEqualTo(0); + } +} diff --git a/TUnit.TestProject/DeferEnumerationTests/DeferEnumerationErrorTests.cs b/TUnit.TestProject/DeferEnumerationTests/DeferEnumerationErrorTests.cs new file mode 100644 index 0000000000..41a9ab0c04 --- /dev/null +++ b/TUnit.TestProject/DeferEnumerationTests/DeferEnumerationErrorTests.cs @@ -0,0 +1,15 @@ +namespace TUnit.TestProject.DeferEnumerationTests; + +// Designed to fail: a deferred data source that throws. The failure must surface as a single failed +// placeholder result at runtime, NOT crash the whole discovery phase. Filter explicitly to run. +public class DeferEnumerationErrorTests +{ + public static IEnumerable Throws() => throw new InvalidOperationException("Boom from data source"); + + [Test] + [MethodDataSource(nameof(Throws), DeferEnumeration = true)] + public async Task DeferredThrowing(int value) + { + await Assert.That(value).IsGreaterThanOrEqualTo(0); + } +} diff --git a/TUnit.TestProject/DeferEnumerationTests/DeferEnumerationTests.cs b/TUnit.TestProject/DeferEnumerationTests/DeferEnumerationTests.cs new file mode 100644 index 0000000000..46ec071284 --- /dev/null +++ b/TUnit.TestProject/DeferEnumerationTests/DeferEnumerationTests.cs @@ -0,0 +1,23 @@ +namespace TUnit.TestProject.DeferEnumerationTests; + +// Tests for DeferEnumeration: the data source is not enumerated during discovery (one placeholder node +// is shown) and is expanded into individual cases at runtime. Filter with /*/*DeferEnumeration*/*. +public class DeferEnumerationTests +{ + public static IEnumerable TenValues() => Enumerable.Range(0, 10); + + [Test] + [MethodDataSource(nameof(TenValues), DeferEnumeration = true)] + public async Task Deferred(int value) + { + await Assert.That(value).IsGreaterThanOrEqualTo(0); + } + + [Test] + [Repeat(2)] + [MethodDataSource(nameof(TenValues), DeferEnumeration = true)] + public async Task DeferredWithRepeat(int value) + { + await Assert.That(value).IsGreaterThanOrEqualTo(0); + } +} diff --git a/TUnit.UnitTests/TraceScopeRegistryTests.cs b/TUnit.UnitTests/TraceScopeRegistryTests.cs index e56bf4920b..14c58b390e 100644 --- a/TUnit.UnitTests/TraceScopeRegistryTests.cs +++ b/TUnit.UnitTests/TraceScopeRegistryTests.cs @@ -135,6 +135,8 @@ public FakeTraceScopeDataSource(SharedType[] sharedTypes) public bool SkipIfEmpty { get; set; } + public bool DeferEnumeration { get; set; } + public IAsyncEnumerable>> GetDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata) => throw new NotSupportedException("Not needed for registry tests"); @@ -148,6 +150,8 @@ private sealed class FakeNonTraceScopeDataSource : IDataSourceAttribute { public bool SkipIfEmpty { get; set; } + public bool DeferEnumeration { get; set; } + public IAsyncEnumerable>> GetDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata) => throw new NotSupportedException("Not needed for registry tests"); } diff --git a/TestProject.props b/TestProject.props index ff3eb49139..0ffff9b0ee 100644 --- a/TestProject.props +++ b/TestProject.props @@ -9,11 +9,17 @@ Condition="'$(MSBuildProjectExtension)' == '.fsproj'" /> - - net10.0 + + net8.0;net9.0;net10.0 Exe diff --git a/docs/docs/writing-tests/data-driven-overview.md b/docs/docs/writing-tests/data-driven-overview.md index 0931d51452..7a9a2bf09e 100644 --- a/docs/docs/writing-tests/data-driven-overview.md +++ b/docs/docs/writing-tests/data-driven-overview.md @@ -14,6 +14,7 @@ TUnit offers several ways to provide data to your tests. Use this guide to pick | Multiple sources on one method | Combined attributes | [Combined Data Sources](combined-data-source.md) | | Hierarchical injection | Nested properties | [Nested Data Sources](nested-data-sources.md) | | Custom generic attributes | `[GenerateGenericTest(typeof(...))]` | [Generic Attributes](generic-attributes.md) | +| Huge data set (reduce IDE overhead) | `DeferEnumeration = true` | [Defer Enumeration](defer-enumeration.md) | ## Quick Examples diff --git a/docs/docs/writing-tests/defer-enumeration.md b/docs/docs/writing-tests/defer-enumeration.md new file mode 100644 index 0000000000..739288496f --- /dev/null +++ b/docs/docs/writing-tests/defer-enumeration.md @@ -0,0 +1,40 @@ +# Deferring Data Source Enumeration + +By default, data sources are enumerated during **test discovery** — every row becomes its own test node, so a data source that produces thousands of cases produces thousands of nodes in your IDE's test explorer. For very large data sets this can make discovery slow and the test tree unwieldy. + +Setting `DeferEnumeration = true` on a data source tells TUnit **not** to enumerate it during discovery. The test then appears as a **single placeholder node**, and the data source is enumerated into individual cases only when the test is **run**. This is similar to xUnit's `DisableDiscoveryEnumeration`. + +```csharp +public static IEnumerable ManyCases() => Enumerable.Range(0, 10_000); + +public class MyTests +{ + [Test] + [MethodDataSource(nameof(ManyCases), DeferEnumeration = true)] + public async Task MyTest(int input) + { + await Assert.That(input).IsGreaterThanOrEqualTo(0); + } +} +``` + +With the flag set: + +- **Discovery** shows one node for `MyTest` instead of 10,000. +- **Running** `MyTest` enumerates the data source and reports each case as a result nested under the placeholder. + +`DeferEnumeration` is available on any data source attribute (`[MethodDataSource]`, `[ClassDataSource]`, custom `DataSourceGenerator` attributes, etc.). If **any** data source on a test sets it, the entire test's case expansion is deferred. It has no effect on `[Arguments]` (a single inline row, so there is nothing to defer). + +:::info +The placeholder is reported as a **container**: the individual cases (nested under it) carry the real pass/fail results, and the placeholder's own result aggregates them — it passes only if every case passes, and fails if any case fails. Because it is reported, it adds one extra entry to flat result counts (TRX/console) per deferred test. If the data source itself throws while enumerating, the error surfaces as a failed result at run time (just as a non-deferred data source error would) instead of failing discovery for the whole assembly. +::: + +:::warning Trade-offs +Because the individual cases do not exist until runtime, a deferred test: + +- cannot have its individual rows selected/filtered from the IDE — you can only run the whole test; +- cannot be targeted by another test's `[DependsOn]`; +- adds **one extra entry** to flat result totals (TRX/console) — the placeholder container — per deferred test, so a 10‑row deferred source reports 11 results. Keep this in mind for CI dashboards or count-based quality gates that compare totals across runs. + +Use it for large data sets where reducing discovery overhead matters more than per-row selection. +::: diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 1ae3b4d312..516e8cdd63 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -29,6 +29,7 @@ const sidebars: SidebarsConfig = { 'writing-tests/method-data-source', 'writing-tests/class-data-source', 'writing-tests/test-data-row', + 'writing-tests/defer-enumeration', 'writing-tests/matrix-tests', 'writing-tests/combined-data-source', 'writing-tests/nested-data-sources',