diff --git a/.gitignore b/.gitignore index dd1360bb1a..740dafc6d7 100644 --- a/.gitignore +++ b/.gitignore @@ -438,4 +438,4 @@ doc/plans/ *.nettrace # Git worktrees -.worktrees/ \ No newline at end of file +.worktrees/.worktrees/ diff --git a/TUnit.Engine.Tests/GenericMethodWithDataSourceTests.cs b/TUnit.Engine.Tests/GenericMethodWithDataSourceTests.cs new file mode 100644 index 0000000000..fae226043f --- /dev/null +++ b/TUnit.Engine.Tests/GenericMethodWithDataSourceTests.cs @@ -0,0 +1,88 @@ +using Shouldly; +using TUnit.Engine.Tests.Enums; + +namespace TUnit.Engine.Tests; + +/// +/// Tests that verify generic methods with [GenerateGenericTest] and [MethodDataSource] attributes +/// are properly discovered and executed. +/// +public class GenericMethodWithDataSourceTests(TestMode testMode) : InvokableTestBase(testMode) +{ + [Test] + public async Task NonGenericClassWithGenericMethodAndDataSource_Should_Generate_Tests() + { + // This test verifies that a non-generic class with a generic method that has both + // [GenerateGenericTest(typeof(int))] and [GenerateGenericTest(typeof(double))] + // combined with [MethodDataSource(nameof(GetStrings))] generates 4 tests: + // - GenericMethodWithDataSource("hello") + // - GenericMethodWithDataSource("world") + // - GenericMethodWithDataSource("hello") + // - GenericMethodWithDataSource("world") + await RunTestsWithFilter( + "/*/*/Bug4440_NonGenericClassWithGenericMethodAndDataSource/*", + [ + result => result.ResultSummary.Outcome.ShouldBe("Completed"), + result => result.ResultSummary.Counters.Total.ShouldBe(4), + result => result.ResultSummary.Counters.Passed.ShouldBe(4), + result => result.ResultSummary.Counters.Failed.ShouldBe(0), + result => result.ResultSummary.Counters.NotExecuted.ShouldBe(0) + ]); + } + + [Test] + public async Task GenericClassWithMethodDataSource_Should_Generate_Tests() + { + // Generic class with 2 types (string, object) and data source with 3 items (1, 2, 3) + // Expected: 2 class types × 3 data items = 6 tests + // The data source values differentiate the test names, so all 6 are unique. + await RunTestsWithFilter( + "/*/*/Bug4440_GenericClassWithMethodDataSource*/*", + [ + result => result.ResultSummary.Outcome.ShouldBe("Completed"), + result => result.ResultSummary.Counters.Total.ShouldBe(6), + result => result.ResultSummary.Counters.Passed.ShouldBe(6), + result => result.ResultSummary.Counters.Failed.ShouldBe(0), + result => result.ResultSummary.Counters.NotExecuted.ShouldBe(0) + ]); + } + + [Test] + public async Task FullyGenericWithDataSources_Should_Generate_Tests() + { + // Cartesian product: 2 class types × 2 method types × 2 data items = 8 tests + // - Bug4440_GenericClassGenericMethodWithDataSources.CartesianProduct(true) + // - Bug4440_GenericClassGenericMethodWithDataSources.CartesianProduct(false) + // - Bug4440_GenericClassGenericMethodWithDataSources.CartesianProduct(true) + // - Bug4440_GenericClassGenericMethodWithDataSources.CartesianProduct(false) + // - Bug4440_GenericClassGenericMethodWithDataSources.CartesianProduct(true) + // - Bug4440_GenericClassGenericMethodWithDataSources.CartesianProduct(false) + // - Bug4440_GenericClassGenericMethodWithDataSources.CartesianProduct(true) + // - Bug4440_GenericClassGenericMethodWithDataSources.CartesianProduct(false) + await RunTestsWithFilter( + "/*/*/Bug4440_GenericClassGenericMethodWithDataSources*/*", + [ + result => result.ResultSummary.Outcome.ShouldBe("Completed"), + result => result.ResultSummary.Counters.Total.ShouldBe(8), + result => result.ResultSummary.Counters.Passed.ShouldBe(8), + result => result.ResultSummary.Counters.Failed.ShouldBe(0), + result => result.ResultSummary.Counters.NotExecuted.ShouldBe(0) + ]); + } + + [Test] + public async Task GenericMethodWithoutDataSource_Should_Work() + { + // Generic method with 3 type arguments (int, string, object) but NO data source. + // Expected: 3 tests with unique names: GenericMethod_Should_Work, , + await RunTestsWithFilter( + "/*/*/Bug4440_NonGenericClassWithGenericMethod/*", + [ + result => result.ResultSummary.Outcome.ShouldBe("Completed"), + result => result.ResultSummary.Counters.Total.ShouldBe(3), + result => result.ResultSummary.Counters.Passed.ShouldBe(3), + result => result.ResultSummary.Counters.Failed.ShouldBe(0), + result => result.ResultSummary.Counters.NotExecuted.ShouldBe(0) + ]); + } +} diff --git a/TUnit.Engine/Building/TestBuilder.cs b/TUnit.Engine/Building/TestBuilder.cs index a4f36c1d5f..4c767710e4 100644 --- a/TUnit.Engine/Building/TestBuilder.cs +++ b/TUnit.Engine/Building/TestBuilder.cs @@ -1165,6 +1165,30 @@ private static bool IsDataCompatibleWithExpectedTypes(TestMetadata metadata, obj return true; // No specific types expected, allow all data } + // Check if any method parameter actually uses the method's generic type parameters. + // If none of the parameters use T, then data compatibility with the generic type doesn't matter. + // This is important for methods like GenericMethod(string input) where the parameter + // is a concrete type (string) and doesn't depend on the generic type T. + if (metadata.GenericMethodTypeArguments is { Length: > 0 }) + { + var anyParameterUsesMethodGeneric = false; + foreach (var param in metadata.MethodMetadata.Parameters) + { + if (ParameterUsesMethodGenericType(param.TypeInfo)) + { + anyParameterUsesMethodGeneric = true; + break; + } + } + + if (!anyParameterUsesMethodGeneric) + { + // None of the method parameters use the method's generic type parameters. + // The data doesn't need to match the generic types - allow all data. + return true; + } + } + // For generic methods, we need to check if the data types match the expected types // The key is to determine what type of data this data source produces @@ -1260,6 +1284,21 @@ private static bool IsDataCompatibleWithExpectedTypes(TestMetadata metadata, obj return false; } + /// + /// Checks if a parameter's type involves a method generic type parameter. + /// Returns true for parameters like T, List<T>, etc. where T is a method generic parameter. + /// Returns false for concrete types like string, int, etc. + /// + private static bool ParameterUsesMethodGenericType(TypeInfo? typeInfo) + { + return typeInfo switch + { + GenericParameter { IsMethodParameter: true } => true, + ConstructedGeneric cg => cg.TypeArguments.Any(ParameterUsesMethodGenericType), + _ => false + }; + } + private static Type? GetExpectedTypeForParameter(ParameterMetadata param, Type[] genericTypeArgs) { // If it's a direct generic parameter (e.g., T) diff --git a/TUnit.Engine/Discovery/ReflectionTestDataCollector.cs b/TUnit.Engine/Discovery/ReflectionTestDataCollector.cs index 868f9924f9..ee1a0663f0 100644 --- a/TUnit.Engine/Discovery/ReflectionTestDataCollector.cs +++ b/TUnit.Engine/Discovery/ReflectionTestDataCollector.cs @@ -85,6 +85,7 @@ public async Task> CollectTestsAsync(string testSessio #endif var allAssemblies = Assemblies; + var assembliesList = new List(allAssemblies.Length); foreach (var assembly in allAssemblies) { @@ -95,7 +96,6 @@ public async Task> CollectTestsAsync(string testSessio } var assemblies = assembliesList; - var maxConcurrency = Math.Min(assemblies.Count, Environment.ProcessorCount * 2); var semaphore = new SemaphoreSlim(maxConcurrency, maxConcurrency); var tasks = new Task>[assemblies.Count]; @@ -409,14 +409,25 @@ private static async Task> DiscoverTestsInAssembly(Assembly a foreach (var method in testMethods) { - try + // Resolve method-level generic instantiations + foreach (var (resolvedMethod, failedMetadata) in ResolveMethodInstantiations(type, method)) { - discoveredTests.Add(await BuildTestMetadata(type, method).ConfigureAwait(false)); - } - catch (Exception ex) - { - var failedTest = CreateFailedTestMetadata(type, method, ex); - discoveredTests.Add(failedTest); + if (failedMetadata != null) + { + discoveredTests.Add(failedMetadata); + continue; + } + + try + { + var metadata = await BuildTestMetadata(type, resolvedMethod).ConfigureAwait(false); + discoveredTests.Add(metadata); + } + catch (Exception ex) + { + var failedTest = CreateFailedTestMetadata(type, resolvedMethod, ex); + discoveredTests.Add(failedTest); + } } } } @@ -531,32 +542,42 @@ private static async IAsyncEnumerable DiscoverTestsInAssemblyStrea { cancellationToken.ThrowIfCancellationRequested(); - TestMetadata? testMetadata = null; - TestMetadata? failedMetadata = null; + // Prevent duplicate test metadata for inherited tests + if (method.DeclaringType != type && !type.IsDefined(typeof(InheritsTestsAttribute), inherit: false)) + { + continue; + } - try + // Resolve method-level generic instantiations + foreach (var (resolvedMethod, methodFailedMetadata) in ResolveMethodInstantiations(type, method)) { - // Prevent duplicate test metadata for inherited tests - if (method.DeclaringType != type && !type.IsDefined(typeof(InheritsTestsAttribute), inherit: false)) + if (methodFailedMetadata != null) { + yield return methodFailedMetadata; continue; } - testMetadata = await BuildTestMetadata(type, method).ConfigureAwait(false); - } - catch (Exception ex) - { - // Create a failed test metadata for discovery failures - failedMetadata = CreateFailedTestMetadata(type, method, ex); - } + TestMetadata? testMetadata = null; + TestMetadata? failedMetadata = null; - if (testMetadata != null) - { - yield return testMetadata; - } - else if (failedMetadata != null) - { - yield return failedMetadata; + try + { + testMetadata = await BuildTestMetadata(type, resolvedMethod).ConfigureAwait(false); + } + catch (Exception ex) + { + // Create a failed test metadata for discovery failures + failedMetadata = CreateFailedTestMetadata(type, resolvedMethod, ex); + } + + if (testMetadata != null) + { + yield return testMetadata; + } + else if (failedMetadata != null) + { + yield return failedMetadata; + } } } } @@ -607,10 +628,20 @@ private static async Task> DiscoverGenericTests(Type genericT if (concreteMethod != null) { - // Build test metadata for the concrete type - // No class data for GenerateGenericTest - it just provides type arguments - var testMetadata = await BuildTestMetadata(concreteType, concreteMethod, null).ConfigureAwait(false); - discoveredTests.Add(testMetadata); + // Resolve method-level generic instantiations + foreach (var (resolvedMethod, methodFailedMetadata) in ResolveMethodInstantiations(concreteType, concreteMethod)) + { + if (methodFailedMetadata != null) + { + discoveredTests.Add(methodFailedMetadata); + continue; + } + + // Build test metadata for the concrete type + // No class data for GenerateGenericTest - it just provides type arguments + var testMetadata = await BuildTestMetadata(concreteType, resolvedMethod, null).ConfigureAwait(false); + discoveredTests.Add(testMetadata); + } } } } @@ -665,13 +696,23 @@ private static async Task> DiscoverGenericTests(Type genericT if (concreteMethod != null) { - // Build test metadata for the concrete type - // The concrete type already has its generic arguments resolved - // For generic types with primary constructors that were resolved from class-level data sources, - // we need to ensure the class data sources contain the specific data for this instantiation - var testMetadata = await BuildTestMetadata(concreteType, concreteMethod, dataRow).ConfigureAwait(false); + // Resolve method-level generic instantiations + foreach (var (resolvedMethod, methodFailedMetadata) in ResolveMethodInstantiations(concreteType, concreteMethod)) + { + if (methodFailedMetadata != null) + { + discoveredTests.Add(methodFailedMetadata); + continue; + } - discoveredTests.Add(testMetadata); + // Build test metadata for the concrete type + // The concrete type already has its generic arguments resolved + // For generic types with primary constructors that were resolved from class-level data sources, + // we need to ensure the class data sources contain the specific data for this instantiation + var testMetadata = await BuildTestMetadata(concreteType, resolvedMethod, dataRow).ConfigureAwait(false); + + discoveredTests.Add(testMetadata); + } } } } @@ -740,15 +781,23 @@ private static async IAsyncEnumerable DiscoverGenericTestsStreamin if (concreteMethod != null) { - // Build test metadata for the concrete type - // No class data for GenerateGenericTest - it just provides type arguments - var testMetadata = await BuildTestMetadata(concreteType, concreteMethod, null).ConfigureAwait(false); - - if (successfulTests == null) + // Resolve method-level generic instantiations + foreach (var (resolvedMethod, methodFailedMetadata) in ResolveMethodInstantiations(concreteType, concreteMethod)) { - successfulTests = []; + if (methodFailedMetadata != null) + { + successfulTests ??= []; + successfulTests.Add(methodFailedMetadata); + continue; + } + + // Build test metadata for the concrete type + // No class data for GenerateGenericTest - it just provides type arguments + var testMetadata = await BuildTestMetadata(concreteType, resolvedMethod, null).ConfigureAwait(false); + + successfulTests ??= []; + successfulTests.Add(testMetadata); } - successfulTests.Add(testMetadata); } } } @@ -836,19 +885,25 @@ private static async IAsyncEnumerable DiscoverGenericTestsStreamin if (concreteMethod != null) { - // Build test metadata for the concrete type - // The concrete type already has its generic arguments resolved - // For generic types with primary constructors that were resolved from class-level data sources, - // we need to ensure the class data sources contain the specific data for this instantiation - var testMetadata = await BuildTestMetadata(concreteType, concreteMethod, dataRow).ConfigureAwait(false); - - if (successfulTests == null) + // Resolve method-level generic instantiations + foreach (var (resolvedMethod, methodFailedMetadata) in ResolveMethodInstantiations(concreteType, concreteMethod)) { - successfulTests = - [ - ]; + if (methodFailedMetadata != null) + { + successfulTests ??= []; + successfulTests.Add(methodFailedMetadata); + continue; + } + + // Build test metadata for the concrete type + // The concrete type already has its generic arguments resolved + // For generic types with primary constructors that were resolved from class-level data sources, + // we need to ensure the class data sources contain the specific data for this instantiation + var testMetadata = await BuildTestMetadata(concreteType, resolvedMethod, dataRow).ConfigureAwait(false); + + successfulTests ??= []; + successfulTests.Add(testMetadata); } - successfulTests.Add(testMetadata); } } } @@ -859,8 +914,8 @@ private static async IAsyncEnumerable DiscoverGenericTestsStreamin $"Failed to create concrete type for {genericTypeDefinition.FullName ?? genericTypeDefinition.Name}. " + $"Error: {ex.Message}. " + $"Generic parameter count: {genericTypeDefinition.GetGenericArguments().Length}, " + - $"Type arguments: {string.Join(", ", typeArguments?.Select(static t => t.Name) ?? [ - ])}", ex), + $"Type arguments: {string.Join(", ", typeArguments?.Select(static t => t.Name) ?? [])}", + ex), $"[GENERIC TYPE CREATION FAILED] {genericTypeDefinition.Name}") { TestName = $"[GENERIC TYPE CREATION FAILED] {genericTypeDefinition.Name}", @@ -1037,7 +1092,16 @@ private static string GenerateTestName(Type testClass, MethodInfo testMethod) } } - // Default format - just method name to match source generation + // For constructed generic methods (e.g., from MakeGenericMethod), include type arguments + // This matches source-gen behavior where test names include type args like "MethodName" + if (testMethod.IsGenericMethod && !testMethod.IsGenericMethodDefinition) + { + var typeArgs = testMethod.GetGenericArguments(); + var typeArgNames = string.Join(", ", typeArgs.Select(static t => t.Name)); + return $"{testMethod.Name}<{typeArgNames}>"; + } + + // Default format - just method name return testMethod.Name; } @@ -1138,6 +1202,129 @@ private static bool HasTestMethods([DynamicallyAccessedMembers(DynamicallyAccess } } + /// + /// Resolves method-level generic instantiations from [GenerateGenericTest] attributes. + /// For non-generic methods, yields the method as-is. + /// For generic methods with [GenerateGenericTest], yields concrete method instantiations. + /// + private static IEnumerable<(MethodInfo Method, TestMetadata? FailedMetadata)> ResolveMethodInstantiations( + Type concreteClassType, + MethodInfo method) + { + // If method is not generic, yield it directly + if (!method.IsGenericMethodDefinition) + { + yield return (method, null); + yield break; + } + + // Check for [GenerateGenericTest] attributes on the method + var generateGenericTestAttributes = method.GetCustomAttributes(inherit: false).ToArray(); + + if (generateGenericTestAttributes.Length == 0) + { + // No [GenerateGenericTest] attributes - yield the generic method definition as-is + // TestBuilder will attempt to infer types from data sources later + yield return (method, null); + yield break; + } + + var methodGenericParams = method.GetGenericArguments(); + + // Process each [GenerateGenericTest] attribute + foreach (var genAttr in generateGenericTestAttributes) + { + var typeArguments = genAttr.TypeArguments; + + // Validate type argument count + if (typeArguments.Length == 0) + { + var failedMetadata = CreateFailedMethodGenericMetadata( + concreteClassType, + method, + $"[GenerateGenericTest] on method '{method.Name}' has no type arguments"); + yield return (method, failedMetadata); + continue; + } + + if (typeArguments.Length != methodGenericParams.Length) + { + var failedMetadata = CreateFailedMethodGenericMetadata( + concreteClassType, + method, + $"[GenerateGenericTest] on method '{method.Name}' provides {typeArguments.Length} type argument(s) " + + $"but method requires {methodGenericParams.Length}. " + + $"Provided: [{string.Join(", ", typeArguments.Select(static t => t.Name))}]"); + yield return (method, failedMetadata); + continue; + } + + // Try to create concrete method - capture result outside try-catch for yield + MethodInfo? resolvedMethod = null; + TestMetadata? errorMetadata = null; + + try + { + // Create concrete method + resolvedMethod = method.MakeGenericMethod(typeArguments); + } + catch (ArgumentException ex) + { + // Constraint violation + errorMetadata = CreateFailedMethodGenericMetadata( + concreteClassType, + method, + $"[GenerateGenericTest] constraint violation on method '{method.Name}': {ex.Message}. " + + $"Type arguments: [{string.Join(", ", typeArguments.Select(static t => t.Name))}]"); + } + catch (Exception ex) + { + errorMetadata = CreateFailedMethodGenericMetadata( + concreteClassType, + method, + $"Failed to create concrete method for '{method.Name}': {ex.Message}"); + } + + // Yield result outside of try-catch + if (errorMetadata != null) + { + yield return (method, errorMetadata); + } + else if (resolvedMethod != null) + { + yield return (resolvedMethod, null); + } + } + } + + /// + /// Creates a failed test metadata for method-level generic resolution errors. + /// + private static TestMetadata CreateFailedMethodGenericMetadata( + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.NonPublicConstructors | DynamicallyAccessedMemberTypes.PublicMethods | DynamicallyAccessedMemberTypes.NonPublicMethods | DynamicallyAccessedMemberTypes.PublicProperties)] + Type type, + MethodInfo method, + string errorMessage) + { + var testName = $"[GENERIC METHOD RESOLUTION FAILED] {type.FullName}.{method.Name}"; + var displayName = $"{testName} - {errorMessage}"; + var exception = new InvalidOperationException(errorMessage); + + return new FailedTestMetadata(exception, displayName) + { + TestName = testName, + TestClassType = type, + TestMethodName = method.Name, + FilePath = ExtractFilePath(method) ?? "Unknown", + LineNumber = ExtractLineNumber(method) ?? 0, + MethodMetadata = ReflectionMetadataBuilder.CreateMethodMetadata(type, method), + AttributeFactory = () => method.GetCustomAttributes().ToArray(), + DataSources = [], + ClassDataSources = [], + PropertyDataSources = [] + }; + } + private static string? ExtractFilePath(MethodInfo method) { return method.GetCustomAttribute()?.File; diff --git a/TUnit.Engine/Services/TestGenericTypeResolver.cs b/TUnit.Engine/Services/TestGenericTypeResolver.cs index aa2e886466..76c0a5a74c 100644 --- a/TUnit.Engine/Services/TestGenericTypeResolver.cs +++ b/TUnit.Engine/Services/TestGenericTypeResolver.cs @@ -35,22 +35,20 @@ public static TestGenericTypeResolution Resolve(TestMetadata metadata, TestBuild testData.ClassData); } - // Resolve method generic arguments if the test method is generic - if (metadata.GenericMethodInfo != null) + // First check if generic method type arguments are already resolved + // This handles constructed generic methods (created via MakeGenericMethod from [GenerateGenericTest]) + if (metadata.GenericMethodTypeArguments is { Length: > 0 }) { - // Check if generic method type arguments are already resolved - if (metadata.GenericMethodTypeArguments is { Length: > 0 }) - { - result.ResolvedMethodGenericArguments = metadata.GenericMethodTypeArguments; - } - else - { - result.ResolvedMethodGenericArguments = ResolveMethodGenericArguments( - metadata.MethodMetadata, - metadata.GenericMethodInfo, - testData.MethodData, - metadata.MethodMetadata.Parameters.Select(p => p.Type).ToArray()); - } + result.ResolvedMethodGenericArguments = metadata.GenericMethodTypeArguments; + } + // Otherwise resolve from GenericMethodInfo if the test method is a generic definition + else if (metadata.GenericMethodInfo != null) + { + result.ResolvedMethodGenericArguments = ResolveMethodGenericArguments( + metadata.MethodMetadata, + metadata.GenericMethodInfo, + testData.MethodData, + metadata.MethodMetadata.Parameters.Select(p => p.Type).ToArray()); } return result; diff --git a/TUnit.Engine/Services/TestIdentifierService.cs b/TUnit.Engine/Services/TestIdentifierService.cs index 5c6cb5eb46..b340154a84 100644 --- a/TUnit.Engine/Services/TestIdentifierService.cs +++ b/TUnit.Engine/Services/TestIdentifierService.cs @@ -29,6 +29,23 @@ public static string GenerateTestId(TestMetadata metadata, TestBuilder.TestData vsb.Append(combination.ClassDataLoopIndex); vsb.Append('.'); vsb.Append(metadata.TestMethodName); + + // Add method generic type arguments to ensure uniqueness for generic methods + // e.g., GenericMethod vs GenericMethod should have different IDs + if (combination.ResolvedMethodGenericArguments is { Length: > 0 }) + { + vsb.Append('<'); + for (var i = 0; i < combination.ResolvedMethodGenericArguments.Length; i++) + { + if (i > 0) + { + vsb.Append(','); + } + vsb.Append(combination.ResolvedMethodGenericArguments[i].FullName ?? combination.ResolvedMethodGenericArguments[i].Name); + } + vsb.Append('>'); + } + WriteTypeWithParameters(ref vsb, methodParameters); vsb.Append('.'); vsb.Append(combination.MethodDataSourceAttributeIndex); diff --git a/TUnit.TestProject/Bugs/4440/GenericMethodDiscoveryTests.cs b/TUnit.TestProject/Bugs/4440/GenericMethodDiscoveryTests.cs new file mode 100644 index 0000000000..460de9f8ed --- /dev/null +++ b/TUnit.TestProject/Bugs/4440/GenericMethodDiscoveryTests.cs @@ -0,0 +1,168 @@ +using TUnit.TestProject.Attributes; + +namespace TUnit.TestProject.Bugs._4440; + +/// +/// Test fixtures for issue #4440: Generic method with [GenerateGenericTest] + [MethodDataSource] +/// fails to be discovered at runtime in reflection mode. +/// + +#region Scenario 1: Non-generic class + generic method + data source (original bug) + +/// +/// The original bug scenario. This should produce 4 tests: +/// - GenericMethodWithDataSource<int>("hello") +/// - GenericMethodWithDataSource<int>("world") +/// - GenericMethodWithDataSource<double>("hello") +/// - GenericMethodWithDataSource<double>("world") +/// +public class Bug4440_NonGenericClassWithGenericMethodAndDataSource +{ + public static IEnumerable GetStrings() + { + yield return "hello"; + yield return "world"; + } + + [Test] + [EngineTest(ExpectedResult.Pass)] + [GenerateGenericTest(typeof(int))] + [GenerateGenericTest(typeof(double))] + [MethodDataSource(nameof(GetStrings))] + public async Task GenericMethodWithDataSource(string input) + { + await Assert.That(input).IsNotNullOrEmpty(); + await Assert.That(typeof(T).IsValueType).IsTrue(); + } +} + +#endregion + +#region Scenario 2: Non-generic class + generic method (no data source) + +/// +/// Tests generic method without data source. Should produce 3 tests. +/// +public class Bug4440_NonGenericClassWithGenericMethod +{ + [Test] + [EngineTest(ExpectedResult.Pass)] + [GenerateGenericTest(typeof(int))] + [GenerateGenericTest(typeof(string))] + [GenerateGenericTest(typeof(object))] + public async Task GenericMethod_Should_Work() + { + await Assert.That(typeof(T)).IsNotNull(); + } +} + +#endregion + +#region Scenario 3: Generic class + non-generic method + data source + +/// +/// Generic class with [GenerateGenericTest] and a non-generic method with data source. +/// Expected: 2 class types × 3 data items = 6 tests. +/// +[GenerateGenericTest(typeof(string))] +[GenerateGenericTest(typeof(object))] +public class Bug4440_GenericClassWithMethodDataSource where T : class +{ + public static IEnumerable GetNumbers() + { + yield return 1; + yield return 2; + yield return 3; + } + + [Test] + [EngineTest(ExpectedResult.Pass)] + [MethodDataSource(nameof(GetNumbers))] + public async Task TestWithDataSource(int number) + { + await Assert.That(typeof(T).IsClass).IsTrue(); + await Assert.That(number).IsGreaterThan(0); + } +} + +#endregion + +#region Scenario 4: Generic class + generic method + data source (cartesian product) + +/// +/// Cartesian product scenario: 2 class types × 2 method types × 2 data items = 8 tests. +/// +[GenerateGenericTest(typeof(string))] +[GenerateGenericTest(typeof(object))] +public class Bug4440_GenericClassGenericMethodWithDataSources where TClass : class +{ + public static IEnumerable GetBools() + { + yield return true; + yield return false; + } + + [Test] + [EngineTest(ExpectedResult.Pass)] + [GenerateGenericTest(typeof(int))] + [GenerateGenericTest(typeof(double))] + [MethodDataSource(nameof(GetBools))] + public async Task CartesianProduct(bool flag) where TMethod : struct + { + await Assert.That(typeof(TClass).IsClass).IsTrue(); + await Assert.That(typeof(TMethod).IsValueType).IsTrue(); + // Just verify we got a boolean value + await Assert.That(flag == true || flag == false).IsTrue(); + } +} + +#endregion + +#region Additional Scenarios + +/// +/// Tests method with multiple type parameters. Should produce 2 tests. +/// +public class GenericMethodMultipleTypeParams +{ + public static IEnumerable GetValues() + { + yield return "value"; + } + + [Test] + [EngineTest(ExpectedResult.Pass)] + [GenerateGenericTest(typeof(int), typeof(string))] + [GenerateGenericTest(typeof(double), typeof(object))] + [MethodDataSource(nameof(GetValues))] + public async Task MultiTypeParamMethod(string value) + { + await Assert.That(typeof(T1).IsValueType).IsTrue(); + await Assert.That(typeof(T2).IsClass).IsTrue(); + await Assert.That(value).IsNotNull(); + } +} + +/// +/// Tests generic method with constraints. Should produce 2 tests. +/// +public class GenericMethodWithConstraints +{ + public static IEnumerable GetData() + { + yield return 42; + } + + [Test] + [EngineTest(ExpectedResult.Pass)] + [GenerateGenericTest(typeof(int))] + [GenerateGenericTest(typeof(double))] + [MethodDataSource(nameof(GetData))] + public async Task ConstrainedGenericMethod(int value) where T : struct + { + await Assert.That(typeof(T).IsValueType).IsTrue(); + await Assert.That(value).IsEqualTo(42); + } +} + +#endregion diff --git a/docs/plans/2026-01-16-generic-method-discovery-design.md b/docs/plans/2026-01-16-generic-method-discovery-design.md new file mode 100644 index 0000000000..41a1c2bb92 --- /dev/null +++ b/docs/plans/2026-01-16-generic-method-discovery-design.md @@ -0,0 +1,237 @@ +# Generic Method Discovery with [GenerateGenericTest] + [MethodDataSource] + +**Issue:** [#4440](https://github.com/thomhurst/TUnit/issues/4440) +**Date:** 2026-01-16 +**Status:** Design Complete + +## Problem Statement + +When a generic **method** (not class) has both `[GenerateGenericTest]` and `[MethodDataSource]` attributes, tests fail to be discovered at runtime in reflection mode, though the source generator produces correct metadata. + +```csharp +public class NonGenericClassWithGenericMethodAndDataSource +{ + [Test] + [GenerateGenericTest(typeof(int))] + [GenerateGenericTest(typeof(double))] + [MethodDataSource(nameof(GetStrings))] + public async Task GenericMethod_With_DataSource(string input) { } +} +``` + +**Expected:** 4 tests (int×"hello", int×"world", double×"hello", double×"world") +**Actual:** 0 tests discovered in reflection mode + +## Root Cause Analysis + +| Mode | Class-level `[GenerateGenericTest]` | Method-level `[GenerateGenericTest]` | +|------|-------------------------------------|--------------------------------------| +| Source Generator | Handled (lines 3680-3733) | Handled (lines 3735-3751) | +| Reflection | Handled (lines 588-625) | **NOT handled** | + +In `ReflectionTestDataCollector.cs`, lines 588 and 716 only check for `[GenerateGenericTest]` on classes: +```csharp +var generateGenericTestAttributes = genericTypeDefinition.GetCustomAttributes(inherit: false).ToArray(); +``` + +No equivalent code exists for method-level `[GenerateGenericTest]`. + +## Solution Design + +### Architecture Overview + +Refactor discovery to separate concerns into three distinct responsibilities: + +``` +DiscoverTestsAsync(assembly) + └─> for each type: + └─> DiscoverTestsFromTypeAsync(type) + └─> for each (concreteClass, classData) in ResolveClassInstantiations(type): + └─> for each method in GetTestMethods(concreteClass): + └─> for each concreteMethod in ResolveMethodInstantiations(method): + └─> BuildTestMetadata(concreteClass, concreteMethod, classData) +``` + +| Method | Responsibility | +|--------|----------------| +| `ResolveClassInstantiationsAsync` | Yields `(Type, object[]?)` for each concrete class variant | +| `ResolveMethodInstantiations` | Yields `MethodInfo` for each concrete method variant | +| `BuildTestMetadata` | Creates metadata from concrete class + concrete method (unchanged) | + +### Method 1: `ResolveClassInstantiationsAsync` + +**Signature:** +```csharp +private static async IAsyncEnumerable<(Type ConcreteType, object?[]? ClassData)> ResolveClassInstantiationsAsync( + [DynamicallyAccessedMembers(...)] Type type, + [EnumeratorCancellation] CancellationToken cancellationToken) +``` + +**Logic:** +1. If type is NOT a generic type definition: + - Yield `(type, null)` once + +2. If type IS a generic type definition: + - Check for `[GenerateGenericTest]` attributes on class + - For each attribute: extract type args, validate constraints, yield `(concreteType, null)` + - Check for class-level data sources + - For each data row: infer type args, yield `(concreteType, dataRow)` + - If neither found: yield nothing (can't resolve open generic) + +### Method 2: `ResolveMethodInstantiations` + +**Signature:** +```csharp +private static IEnumerable ResolveMethodInstantiations( + Type concreteClassType, + MethodInfo method) +``` + +**Note:** Synchronous because `[GenerateGenericTest]` attributes are available immediately. + +**Logic:** +1. If method is NOT a generic method definition: + - Yield `method` once + +2. If method IS a generic method definition: + - Get `[GenerateGenericTest]` attributes from method + - If attributes found: + - For each: extract type args, validate constraints, yield `method.MakeGenericMethod(typeArgs)` + - If no attributes: + - Yield method as-is (TestBuilder will attempt inference from data sources) + +### Error Handling + +All errors become **visible failed tests** in the test explorer: + +```csharp +private static TestMetadata CreateFailedDiscoveryTest( + Type? testClass, + MethodInfo? testMethod, + string errorMessage, + Exception? exception = null) +{ + var fullMessage = exception != null + ? $"{errorMessage}\n\nException: {exception.GetType().Name}: {exception.Message}\n{exception.StackTrace}" + : errorMessage; + + return new FailedTestMetadata + { + TestName = testMethod?.Name ?? testClass?.Name ?? "Unknown", + TestClassType = testClass ?? typeof(object), + TestMethodName = testMethod?.Name ?? "DiscoveryError", + FailureReason = fullMessage, + DiscoveryException = exception ?? new TestDiscoveryException(errorMessage) + }; +} +``` + +**Error scenarios that create failed tests:** +- Constraint violations when calling `MakeGenericType`/`MakeGenericMethod` +- Type argument count mismatch +- Data source retrieval failures +- Reflection failures + +**What users see:** +``` +❌ GenericMethod_With_DataSource (Discovery Failed) + + [GenerateGenericTest] provides 1 type argument(s) but method + 'GenericMethod_With_DataSource' requires 2. + Provided: [Int32] +``` + +### TestBuilder Integration + +No changes needed to TestBuilder. By the time metadata reaches TestBuilder: +- Class type is concrete +- Method is concrete (with `GenericMethodTypeArguments` populated) +- `[MethodDataSource]` is preserved on the concrete method + +TestBuilder's existing data source handling works unchanged. + +## Implementation Plan + +### Step 1: Add Resolution Methods + +**File:** `TUnit.Engine/Discovery/ReflectionTestDataCollector.cs` + +Add two new methods: +- `ResolveClassInstantiationsAsync` - extracts and refactors existing logic from `DiscoverTestsFromGenericTypeAsync` +- `ResolveMethodInstantiations` - new logic for method-level `[GenerateGenericTest]` + +### Step 2: Add Error Handling Infrastructure + +**File:** `TUnit.Engine/Discovery/ReflectionTestDataCollector.cs` + +Add helper method: +- `CreateFailedDiscoveryTest` - creates visible failed test metadata for errors + +### Step 3: Refactor Main Discovery Loop + +**File:** `TUnit.Engine/Discovery/ReflectionTestDataCollector.cs` + +Refactor `DiscoverTestsFromTypeAsync` to use the new resolution methods: +```csharp +await foreach (var (concreteClass, classData) in ResolveClassInstantiationsAsync(type, cancellationToken)) +{ + foreach (var method in GetTestMethods(concreteClass)) + { + foreach (var concreteMethod in ResolveMethodInstantiations(concreteClass, method)) + { + yield return await BuildTestMetadata(concreteClass, concreteMethod, classData); + } + } +} +``` + +### Step 4: Add Test Fixtures + +**File:** `TUnit.TestProject/Bugs/4440/GenericMethodDiscoveryTests.cs` + +Create test fixtures covering: +- Non-generic class + generic method + data source (original bug) +- Generic class + generic method + data source (cartesian product) +- Error cases (constraint violations, type arg mismatches) + +### Step 5: Add Unit Tests + +**File:** `TUnit.Engine.Tests/ResolveClassInstantiationsTests.cs` +**File:** `TUnit.Engine.Tests/ResolveMethodInstantiationsTests.cs` + +Unit tests for the new resolution methods in isolation. + +### Step 6: Expand Integration Tests + +**File:** `TUnit.Engine.Tests/GenericMethodWithDataSourceTests.cs` + +Expand existing tests to run in both modes and verify parity. + +## Test Matrix + +| Scenario | Expected Tests | Source Gen | Reflection | +|----------|---------------|------------|------------| +| Non-generic class, generic method, 2 types, 2 data | 4 | Must pass | Must pass | +| Generic class (2 types), non-generic method, 3 data | 6 | Must pass | Must pass | +| Generic class (2 types), generic method (2 types), 2 data | 8 | Must pass | Must pass | +| Constraint violation | 1 failed | Must show error | Must show error | +| Type arg count mismatch | 1 failed | Must show error | Must show error | + +## Files Changed + +| File | Change | +|------|--------| +| `TUnit.Engine/Discovery/ReflectionTestDataCollector.cs` | Add resolution methods, refactor discovery loop | +| `TUnit.TestProject/Bugs/4440/GenericMethodDiscoveryTests.cs` | New test fixtures | +| `TUnit.Engine.Tests/GenericMethodWithDataSourceTests.cs` | Expand integration tests | +| `TUnit.Engine.Tests/ResolveClassInstantiationsTests.cs` | New unit tests | +| `TUnit.Engine.Tests/ResolveMethodInstantiationsTests.cs` | New unit tests | + +## Success Criteria + +1. Issue #4440 scenario discovers 4 tests in both modes +2. Cartesian product (class + method generics) works correctly +3. All errors visible as failed tests in test explorer +4. Source generator and reflection modes produce identical test counts +5. No regression in existing generic class tests +6. All existing tests continue to pass