feat: add DeferEnumeration to defer data-source expansion to runtime (#5833)

TUnit.Analyzers.Tests/DataSourceGeneratorAnalyzerTests.cs

@@ -141,6 +141,8 @@ public class CustomDataAttribute<T> : Attribute, IDataSourceAttribute
                 {
                     public bool SkipIfEmpty { get; set; }
 
+                    public bool DeferEnumeration { get; set; }
+
                     public async IAsyncEnumerable<Func<Task<object?[]?>>> GetDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata)
                     {
                         // Returns Foo instances as the test expects, not T

TUnit.Core.SourceGenerator.Tests/DeferEnumerationTests.Test.verified.txt

@@ -0,0 +1,198 @@
+// <auto-generated/>
+#pragma warning disable
+
+#nullable enable
+namespace TUnit.Generated;
+[global::System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverageAttribute]
+[global::System.CodeDom.Compiler.GeneratedCode("TUnit", "VERSION_SCRUBBED")]
+internal static class TUnit_TestProject_DeferEnumerationTests_DeferEnumerationTests__TestSource
+{
+    private static readonly global::TUnit.Core.ClassMetadata __classMetadata = global::TUnit.Core.ClassMetadata.GetOrAdd("TestsBase`1:global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests", new global::TUnit.Core.ClassMetadata
+    {
+        Type = typeof(global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests),
+        TypeInfo = new global::TUnit.Core.ConcreteType(typeof(global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests)),
+        Name = "DeferEnumerationTests",
+        Namespace = "TUnit.TestProject.DeferEnumerationTests",
+        Assembly = global::TUnit.Core.AssemblyMetadata.GetOrAdd("TestsBase`1", "TestsBase`1"),
+        Parameters = global::System.Array.Empty<global::TUnit.Core.ParameterMetadata>(),
+        Properties = global::System.Array.Empty<global::TUnit.Core.PropertyMetadata>(),
+        Parent = null
+    });
+    private static readonly global::System.Type __classType = typeof(global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests);
+    private static readonly global::TUnit.Core.MethodMetadata __mm_0 = global::TUnit.Core.MethodMetadataFactory.Create("Deferred", __classType, typeof(global::System.Threading.Tasks.Task), __classMetadata, parameters: new global::TUnit.Core.ParameterMetadata[]
+{
+global::TUnit.Core.ParameterMetadataFactory.Create(typeof(int), "value", new global::TUnit.Core.ConcreteType(typeof(int)), false, reflectionInfoFactory: static () => typeof(global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests).GetMethod("Deferred", global::System.Reflection.BindingFlags.Public | global::System.Reflection.BindingFlags.NonPublic | global::System.Reflection.BindingFlags.Instance, null, new global::System.Type[] { typeof(int) }, null)!.GetParameters()[0])
+});
+    private static readonly global::TUnit.Core.MethodMetadata __mm_1 = global::TUnit.Core.MethodMetadataFactory.Create("DeferredWithRepeat", __classType, typeof(global::System.Threading.Tasks.Task), __classMetadata, parameters: new global::TUnit.Core.ParameterMetadata[]
+{
+global::TUnit.Core.ParameterMetadataFactory.Create(typeof(int), "value", new global::TUnit.Core.ConcreteType(typeof(int)), false, reflectionInfoFactory: static () => typeof(global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests).GetMethod("DeferredWithRepeat", global::System.Reflection.BindingFlags.Public | global::System.Reflection.BindingFlags.NonPublic | global::System.Reflection.BindingFlags.Instance, null, new global::System.Type[] { typeof(int) }, null)!.GetParameters()[0])
+});
+    private static global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests __CreateInstance(global::System.Type[] typeArgs, object?[] args)
+    {
+        return new global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests();
+    }
+    private static global::System.Threading.Tasks.ValueTask __Invoke(global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests instance, int methodIndex, object?[] args, global::System.Threading.CancellationToken cancellationToken)
+    {
+        switch (methodIndex)
+        {
+            case 0:
+            {
+                try
+                {
+                    switch (args.Length)
+                    {
+                        case 1:
+                            {
+                                return new global::System.Threading.Tasks.ValueTask(instance.Deferred(global::TUnit.Core.Helpers.CastHelper.Cast<int>(args[0])));
+                            }
+                        default:
+                            throw new global::System.ArgumentException($"Expected exactly 1 argument, but got {args.Length}");
+                    }
+                }
+                catch (global::System.Exception ex)
+                {
+                    return new global::System.Threading.Tasks.ValueTask(global::System.Threading.Tasks.Task.FromException(ex));
+                }
+            }
+            case 1:
+            {
+                try
+                {
+                    switch (args.Length)
+                    {
+                        case 1:
+                            {
+                                return new global::System.Threading.Tasks.ValueTask(instance.DeferredWithRepeat(global::TUnit.Core.Helpers.CastHelper.Cast<int>(args[0])));
+                            }
+                        default:
+                            throw new global::System.ArgumentException($"Expected exactly 1 argument, but got {args.Length}");
+                    }
+                }
+                catch (global::System.Exception ex)
+                {
+                    return new global::System.Threading.Tasks.ValueTask(global::System.Threading.Tasks.Task.FromException(ex));
+                }
+            }
+            default:
+                throw new global::System.ArgumentOutOfRangeException(nameof(methodIndex));
+        }
+    }
+    private static global::System.Attribute[] __Attributes(int groupIndex)
+    {
+        switch (groupIndex)
+        {
+            case 0:
+            {
+                return
+                [
+                    new global::TUnit.Core.TestAttribute()
+                ];
+            }
+            case 1:
+            {
+                return
+                [
+                    new global::TUnit.Core.TestAttribute(),
+                    new global::TUnit.Core.RepeatAttribute(2)
+                ];
+            }
+            default:
+                throw new global::System.ArgumentOutOfRangeException(nameof(groupIndex));
+        }
+    }
+    public static readonly global::TUnit.Core.TestEntry<global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests>[] Entries = new global::TUnit.Core.TestEntry<global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests>[]
+    {
+        new global::TUnit.Core.TestEntry<global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests>
+        {
+            MethodName = "Deferred",
+            FullyQualifiedName = "TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests.Deferred",
+            FilePath = "",
+            LineNumber = 9,
+            Categories = global::System.Array.Empty<string>(),
+            Properties = global::System.Array.Empty<string>(),
+            HasDataSource = true,
+            RepeatCount = 0,
+            DependsOn = global::System.Array.Empty<string>(),
+            MethodMetadata = __mm_0,
+            CreateInstance = __CreateInstance,
+            InvokeBody = __Invoke,
+            MethodIndex = 0,
+            CreateAttributes = __Attributes,
+            AttributeGroupIndex = 0,
+            TestDataSources = new global::TUnit.Core.IDataSourceAttribute[]
+{
+    new global::TUnit.Core.MethodDataSourceAttribute("TenValues")
+    {
+        DeferEnumeration = true,
+        Factory = (dataGeneratorMetadata) =>
+        {
+            async global::System.Collections.Generic.IAsyncEnumerable<global::System.Func<global::System.Threading.Tasks.Task<object?[]?>>> Factory()
+            {
+                var result = global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests.TenValues();
+                if (result is global::System.Collections.IEnumerable enumerable && !(result is string))
+                {
+                    foreach (var item in enumerable)
+                    {
+                        yield return () => global::System.Threading.Tasks.Task.FromResult(global::TUnit.Core.Helpers.DataSourceHelpers.ToObjectArray(item));
+                    }
+                }
+                else
+                {
+                    yield return () => global::System.Threading.Tasks.Task.FromResult(global::TUnit.Core.Helpers.DataSourceHelpers.ToObjectArray(result));
+                }
+            }
+            return Factory();
+        }
+    },
+},
+        },
+        new global::TUnit.Core.TestEntry<global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests>
+        {
+            MethodName = "DeferredWithRepeat",
+            FullyQualifiedName = "TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests.DeferredWithRepeat",
+            FilePath = "",
+            LineNumber = 16,
+            Categories = global::System.Array.Empty<string>(),
+            Properties = global::System.Array.Empty<string>(),
+            HasDataSource = true,
+            RepeatCount = 2,
+            DependsOn = global::System.Array.Empty<string>(),
+            MethodMetadata = __mm_1,
+            CreateInstance = __CreateInstance,
+            InvokeBody = __Invoke,
+            MethodIndex = 1,
+            CreateAttributes = __Attributes,
+            AttributeGroupIndex = 1,
+            TestDataSources = new global::TUnit.Core.IDataSourceAttribute[]
+{
+    new global::TUnit.Core.MethodDataSourceAttribute("TenValues")
+    {
+        DeferEnumeration = true,
+        Factory = (dataGeneratorMetadata) =>
+        {
+            async global::System.Collections.Generic.IAsyncEnumerable<global::System.Func<global::System.Threading.Tasks.Task<object?[]?>>> Factory()
+            {
+                var result = global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests.TenValues();
+                if (result is global::System.Collections.IEnumerable enumerable && !(result is string))
+                {
+                    foreach (var item in enumerable)
+                    {
+                        yield return () => global::System.Threading.Tasks.Task.FromResult(global::TUnit.Core.Helpers.DataSourceHelpers.ToObjectArray(item));
+                    }
+                }
+                else
+                {
+                    yield return () => global::System.Threading.Tasks.Task.FromResult(global::TUnit.Core.Helpers.DataSourceHelpers.ToObjectArray(result));
+                }
+            }
+            return Factory();
+        }
+    },
+},
+        },
+    };
+}
+internal static partial class TUnit_TestRegistration
+{
+    static readonly int _r_TUnit_TestProject_DeferEnumerationTests_DeferEnumerationTests__TestSource = global::TUnit.Core.SourceRegistrar.RegisterEntries<global::TUnit.TestProject.DeferEnumerationTests.DeferEnumerationTests>(static () => TUnit_TestProject_DeferEnumerationTests_DeferEnumerationTests__TestSource.Entries);
+}

TUnit.Core.SourceGenerator.Tests/DeferEnumerationTests.cs

@@ -0,0 +1,16 @@
+
+namespace TUnit.Core.SourceGenerator.Tests;
+
+internal class DeferEnumerationTests : TestsBase
+{
+    [Test]
+    public Task Test() => RunTest(Path.Combine(Git.RootDirectory.FullName,
+            "TUnit.TestProject",
+            "DeferEnumerationTests",
+            "DeferEnumerationTests.cs"),
+        async generatedFiles =>
+        {
+            await Assert.That(generatedFiles).IsNotEmpty();
+            await Assert.That(string.Join("\n", generatedFiles)).Contains("DeferEnumeration = true");
+        });
+}

TUnit.Core.SourceGenerator/Generators/TestMetadataGenerator.cs

@@ -1636,6 +1636,15 @@ private static void GenerateMethodDataSourceAttribute(CodeWriter writer, Attribu
             writer.AppendLine(",");
         }
 
+        // Copy over DeferEnumeration if set so the test is enumerated at runtime instead of discovery.
+        // The hand-built initializer above only copies the named args it knows about, so this must be
+        // threaded explicitly (unlike the method-not-found path which round-trips all named arguments).
+        var deferProperty = attr.NamedArguments.FirstOrDefault(x => x.Key == "DeferEnumeration");
+        if (deferProperty is { Key: not null, Value.IsNull: false } && deferProperty.Value.Value is true)
+        {
+            writer.AppendLine("DeferEnumeration = true,");
+        }
+
         // Set the Factory property with a strongly-typed function
         writer.AppendLine("Factory = (dataGeneratorMetadata) =>");
         writer.AppendLine("{");

TUnit.Core/Attributes/TestData/ArgumentsAttribute.cs

@@ -74,6 +74,10 @@ public sealed class ArgumentsAttribute : Attribute, IDataSourceAttribute, ITestR
     /// <inheritdoc />
     public bool SkipIfEmpty { get; set; }
 
+    /// <inheritdoc />
+    // [Arguments] yields a single row, so deferring its enumeration would be pure overhead - always false.
+    public bool DeferEnumeration { get => false; set { } }
+
     /// <summary>
     /// Initializes a new instance of the <see cref="ArgumentsAttribute"/> class with the specified test argument values.
     /// </summary>
@@ -166,6 +170,10 @@ public sealed class ArgumentsAttribute<T>(T value) : TypedDataSourceAttribute<T>
     /// <inheritdoc />
     public override bool SkipIfEmpty { get; set; }
 
+    /// <inheritdoc />
+    // [Arguments] yields a single row, so deferring its enumeration would be pure overhead - always false.
+    public override bool DeferEnumeration { get => false; set { } }
+
     public override async IAsyncEnumerable<Func<Task<T>>> GetTypedDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata)
     {
         yield return () => Task.FromResult(value);

TUnit.Core/Attributes/TestData/AsyncUntypedDataSourceSourceGeneratorAttribute.cs

@@ -8,6 +8,9 @@ public abstract class AsyncUntypedDataSourceGeneratorAttribute : Attribute, IAsy
     /// <inheritdoc />
     public virtual bool SkipIfEmpty { get; set; }
 
+    /// <inheritdoc />
+    public virtual bool DeferEnumeration { get; set; }
+
     protected abstract IAsyncEnumerable<Func<Task<object?[]?>>> GenerateDataSourcesAsync(DataGeneratorMetadata dataGeneratorMetadata);
 
     public IAsyncEnumerable<Func<Task<object?[]?>>> GenerateAsync(DataGeneratorMetadata dataGeneratorMetadata)

TUnit.Core/Attributes/TestData/DelegateDataSourceAttribute.cs

@@ -13,6 +13,9 @@ internal sealed class DelegateDataSourceAttribute : Attribute, IDataSourceAttrib
     /// <inheritdoc />
     public bool SkipIfEmpty { get; set; }
 
+    /// <inheritdoc />
+    public bool DeferEnumeration { get; set; }
+
     public DelegateDataSourceAttribute(Func<DataGeneratorMetadata, IAsyncEnumerable<object?[]>> factory, bool isShared = false)
     {
         _factory = factory ?? throw new ArgumentNullException(nameof(factory));

TUnit.Core/Attributes/TestData/EmptyDataSourceAttribute.cs

@@ -9,6 +9,10 @@ internal sealed class EmptyDataSourceAttribute : Attribute, IDataSourceAttribute
     /// <inheritdoc />
     public bool SkipIfEmpty { get; set; }
 
+    /// <inheritdoc />
+    // Always a single (empty) row, so deferring its enumeration would be pure overhead.
+    public bool DeferEnumeration { get => false; set { } }
+
     public async IAsyncEnumerable<Func<Task<object?[]?>>> GetDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata)
     {
         yield return () => Task.FromResult<object?[]?>([

TUnit.Core/Attributes/TestData/IDataSourceAttribute.cs

@@ -18,4 +18,19 @@ public interface IDataSourceAttribute
     /// When true, if the data source returns no data, the test will be skipped instead of failing.
     /// </summary>
     bool SkipIfEmpty { get; set; }
+
+    /// <summary>
+    /// When <c>true</c>, this data source is not enumerated during test discovery. Instead, the test
+    /// appears as a single placeholder node, and the data rows are enumerated and executed at runtime
+    /// (each reported as a result nested under the placeholder). This avoids the IDE/test-explorer
+    /// overhead of expanding a data source that produces a large number of cases.
+    /// </summary>
+    /// <remarks>
+    /// If any data source on a test sets this to <c>true</c>, the entire test's case expansion is
+    /// deferred to runtime. Tests deferred this way cannot be targeted individually by a filter, and
+    /// other tests cannot <c>[DependsOn]</c> their rows (the rows do not exist until runtime).
+    /// Single-row sources (such as <c>[Arguments]</c>) ignore this flag — there is nothing to defer —
+    /// so setting it on them has no effect.
+    /// </remarks>
+    bool DeferEnumeration { get; set; }
 }

TUnit.Core/Attributes/TestData/MethodDataSourceAttribute.cs

@@ -104,6 +104,9 @@ public class MethodDataSourceAttribute : Attribute, IDataSourceAttribute
     /// <inheritdoc />
     public bool SkipIfEmpty { get; set; }
 
+    /// <inheritdoc />
+    public bool DeferEnumeration { get; set; }
+
     public MethodDataSourceAttribute(string methodNameProvidingDataSource)
     {
         if (methodNameProvidingDataSource is null or { Length: < 1 })
@@ -143,6 +146,7 @@ internal InstanceMethodDataSourceAttribute ToInstanceVariant()
 
         converted.Arguments = Arguments;
         converted.SkipIfEmpty = SkipIfEmpty;
+        converted.DeferEnumeration = DeferEnumeration;
 
         return converted;
     }

TUnit.Core/Attributes/TestData/NoDataSource.cs

@@ -8,6 +8,9 @@ internal class NoDataSource : IDataSourceAttribute
     /// <inheritdoc />
     public bool SkipIfEmpty { get; set; }
 
+    /// <inheritdoc />
+    public bool DeferEnumeration { get; set; }
+
     public async IAsyncEnumerable<Func<Task<object?[]?>>> GetDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata)
     {
         yield return static () => _emptyRowTask;

TUnit.Core/Attributes/TestData/StaticDataSourceAttribute.cs

@@ -11,6 +11,9 @@ internal sealed class StaticDataSourceAttribute : Attribute, IDataSourceAttribut
     /// <inheritdoc />
     public bool SkipIfEmpty { get; set; }
 
+    /// <inheritdoc />
+    public bool DeferEnumeration { get; set; }
+
     public StaticDataSourceAttribute(params object?[][] data)
     {
         _data = data ?? throw new ArgumentNullException(nameof(data));

TUnit.Core/Attributes/TestData/TypedDataSourceAttribute.cs

@@ -7,6 +7,9 @@ public abstract class TypedDataSourceAttribute<T> : Attribute, ITypedDataSourceA
     /// <inheritdoc />
     public virtual bool SkipIfEmpty { get; set; }
 
+    /// <inheritdoc />
+    public virtual bool DeferEnumeration { get; set; }
+
     public abstract IAsyncEnumerable<Func<Task<T>>> GetTypedDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata);
 
     public async IAsyncEnumerable<Func<Task<object?[]?>>> GetDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata)