add runsetting for empty data in DDT

microsoft · Apr 26, 2024 · 28a9559 · 28a9559
1 parent 4aa1cd3
commit 28a9559
Show file tree

Hide file tree

Showing 4 changed files with 66 additions and 14 deletions.
diff --git a/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs
@@ -301,20 +301,31 @@ private static bool TryProcessTestDataSourceTests(UnitTestElement test, TestMeth
     {
         foreach (var dataSource in testDataSources)
         {
-            var data = dataSource.GetData(methodInfo);
+            IEnumerable<object?[]>? data = null;
+            try
+            {
+                data = dataSource.GetData(methodInfo);
+            }
+            catch (Exception ex)
+            {
+                // if dataSource is empty without setting the `MarkTestsWithMissingDynamicDataAsInconclusive` it would throw inside GetData function.
+                if (ex is ArgumentException && MSTestSettings.CurrentSettings.MarkTestsWithMissingDynamicDataAsInconclusive)
+                {
+                    var discoveredTest = test.Clone();
+                    discoveredTest.DisplayName = dataSource.GetDisplayName(methodInfo, null) ?? discoveredTest.DisplayName;
+                    tests.Add(discoveredTest);
+                    continue;
+                }
+                else
+                {
+                    throw;
+                }
+            }
+
             var testDisplayNameFirstSeen = new Dictionary<string, int>();
             var discoveredTests = new List<UnitTestElement>();
             var index = 0;
 
-            // trying to handle the test that comes without data by ignoring it but still doesn't work geting another issue
-            if (!data.Any())
-            {
-                var discoveredTest = test.Clone();
-                discoveredTest.DisplayName = dataSource.GetDisplayName(methodInfo, null) ?? discoveredTest.DisplayName;
-                discoveredTest.Ignored = true;
-                tests.Add(discoveredTest);
-            }
-
             foreach (var d in data)
             {
                 var discoveredTest = test.Clone();

diff --git a/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs
@@ -217,7 +217,6 @@ internal UnitTestResult[] RunTestMethod()
         }
 
         // Set a result in case no result is present.
-        // check here the error with the test doesn't run even skipping that gives an error
         if (results.Count == 0)
         {
             TestResult emptyResult = new()
@@ -289,13 +288,39 @@ private bool ExecuteDataSourceBasedTests(List<TestResult> results)
         else
         {
             var testDataSources = _testMethodInfo.GetAttributes<Attribute>(false)?.OfType<UTF.ITestDataSource>();
+            Stopwatch watch = new();
 
             if (testDataSources != null)
             {
                 foreach (var testDataSource in testDataSources)
                 {
                     isDataDriven = true;
-                    foreach (var data in testDataSource.GetData(_testMethodInfo.MethodInfo))
+                    watch.Start();
+                    IEnumerable<object?[]>? dataSource = null;
+                    try
+                    {
+                        dataSource = testDataSource.GetData(_testMethodInfo.MethodInfo);
+                    }
+                    catch (Exception ex)
+                    {
+                        // if dataSource is empty with setting the `MarkTestsWithMissingDynamicDataAsInconclusive` it would throw inside GetData function.
+                        if (ex is ArgumentException && MSTestSettings.CurrentSettings.MarkTestsWithMissingDynamicDataAsInconclusive)
+                        {
+                            var inconclusiveResult = new TestResult
+                            {
+                                Outcome = UTF.UnitTestOutcome.Inconclusive,
+                                Duration = watch.Elapsed,
+                            };
+                            results.Add(inconclusiveResult);
+                            continue;
+                        }
+                        else
+                        {
+                            throw;
+                        }
+                    }
+
+                    foreach (var data in dataSource)
                     {
                         try
                         {

diff --git a/src/Adapter/MSTest.TestAdapter/MSTestSettings.cs b/src/Adapter/MSTest.TestAdapter/MSTestSettings.cs
@@ -56,6 +56,7 @@ public MSTestSettings()
         ForcedLegacyMode = false;
         TestSettingsFile = null;
         DisableParallelization = false;
+        MarkTestsWithMissingDynamicDataAsInconclusive = false;
         TestTimeout = 0;
         AssemblyInitializeTimeout = 0;
         ClassInitializeTimeout = 0;
@@ -160,6 +161,11 @@ public static RunConfigurationSettings RunConfigurationSettings
     /// </summary>
     internal int AssemblyCleanupTimeout { get; private set; }
 
+    /// <summary>
+    ///  Gets a value indicating whether to enable marking tests with missing dynamic data as Inconclusive.
+    /// </summary>
+    internal bool MarkTestsWithMissingDynamicDataAsInconclusive { get; private set; }
+
     /// <summary>
     ///  Gets specified global ClassInitializeTimeout timeout.
     /// </summary>
@@ -211,6 +217,7 @@ public static void PopulateSettings(MSTestSettings settings)
         CurrentSettings.TreatClassAndAssemblyCleanupWarningsAsErrors = settings.TreatClassAndAssemblyCleanupWarningsAsErrors;
         CurrentSettings.AssemblyInitializeTimeout = settings.AssemblyInitializeTimeout;
         CurrentSettings.AssemblyCleanupTimeout = settings.AssemblyCleanupTimeout;
+        CurrentSettings.MarkTestsWithMissingDynamicDataAsInconclusive = settings.MarkTestsWithMissingDynamicDataAsInconclusive;
         CurrentSettings.ClassInitializeTimeout = settings.ClassInitializeTimeout;
         CurrentSettings.ClassCleanupTimeout = settings.ClassCleanupTimeout;
         CurrentSettings.TestInitializeTimeout = settings.TestInitializeTimeout;
@@ -474,6 +481,16 @@ private static MSTestSettings ToSettings(XmlReader reader)
                             break;
                         }
 
+                    case "MARKTESTSWITHMISSINGDYNAMICDATAASINCONCLUSIVE":
+                        {
+                            if (bool.TryParse(reader.ReadInnerXml(), out bool markTestsWithMissingDynamicDataAsInconclusive) && markTestsWithMissingDynamicDataAsInconclusive)
+                            {
+                                settings.MarkTestsWithMissingDynamicDataAsInconclusive = markTestsWithMissingDynamicDataAsInconclusive;
+                            }
+
+                            break;
+                        }
+
                     case "ASSEMBLYINITIALIZETIMEOUT":
                         {
                             if (int.TryParse(reader.ReadInnerXml(), out int assemblyInitializeTimeout) && assemblyInitializeTimeout > 0)

diff --git a/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs b/src/TestFramework/TestFramework/Attributes/DataSource/DynamicDataAttribute.cs
@@ -148,8 +148,7 @@ public IEnumerable<object[]> GetData(MethodInfo methodInfo)
                     _dynamicDataDeclaringType.FullName));
         }
 
-        // adding check if env var is set with returning back the !
-        if (data.Any())
+        if (!data.Any())
         {
             throw new ArgumentException(
                 string.Format(