Version 1.1.0-beta2 release

Version 1.1.0-beta2 release

RELEASE_NOTES.md

@@ -1,3 +1,23 @@
+#### 1.1.0-beta2 December 23 2019 ####
+- [Add support for Xunit TestFrameworkAttribute attribute](https://github.com/akkadotnet/Akka.MultiNodeTestRunner/pull/116)
+
+In this release we added `MultiNodeTestFramework` to simplify non-parallel test setup. This test 
+framework is a simple override of the built-in `XunitTestFramework` that disables/ignores the
+Xunit `CollectionBehaviorAttribute`, put all test classes from a single assembly into a single test 
+collection, and disables the test collection parallelization.
+
+To use this test framework, you will need to add an assembly level attribute that tells Xunit to
+use this custom test framework instead:
+
+```c#
+[assembly: TestFramework("Akka.MultiNode.TestAdapter.MultiNodeTestFramework", "Akka.MultiNode.TestAdapter")]
+```
+
+Note that you can also use this assembly level attribute to achieve more or less the same effect:
+```c#
+[assembly: CollectionBehavior(DisableTestParallelization = true)]
+```
+
 #### 1.1.0-beta1 October 20 2019 ####
 
 - [Switch to pure Xunit implementation](https://github.com/akkadotnet/Akka.MultiNodeTestRunner/pull/105)

global.json

@@ -1,5 +1,5 @@
 {
   "msbuild-sdks": {
-    "MSBuild.Sdk.Extras": "3.0.38"
+    "MSBuild.Sdk.Extras": "3.0.44"
   }
 }

src/Akka.MultiNode.SampleMultiNodeTests/Metadata/SampleTestsMetadata.cs

@@ -1,6 +1,6 @@
 using Xunit;
 
-[assembly: CollectionBehavior(DisableTestParallelization = true)]
+[assembly: TestFramework("Akka.MultiNode.TestAdapter.MultiNodeTestFramework", "Akka.MultiNode.TestAdapter")]
 namespace Akka.MultiNode.TestAdapter.SampleTests.Metadata
 {
     /// <summary>

src/Akka.MultiNode.TestAdapter.Tests/Internal/MultiNodeTestRunnerDiscovery/DiscoveryCases.cs

@@ -5,6 +5,7 @@
 // </copyright>
 //-----------------------------------------------------------------------
 
+using System.Linq;
 using Akka.Remote.TestKit;
 
 namespace Akka.MultiNode.TestAdapter.Tests.Internal.MultiNodeTestRunnerDiscovery
@@ -139,6 +140,31 @@ public FloodyChildSpec3(FloodyConfig config) : base(config)
             }
         }
 
+        public class NoReflectionConfig : MultiNodeConfig
+        {
+            public NoReflectionConfig()
+            {
+                foreach(var i in Enumerable.Range(1, 10))
+                {
+                    Role("node-" + i);
+                }
+            }
+        }
+
+        public class NoReflectionSpec : MultiNodeSpec
+        {
+            public NoReflectionSpec(NoReflectionConfig config): base(config, typeof(NoReflectionSpec))
+            {
+            }
+
+            [MultiNodeFact(Skip = "Only for discovery tests")]
+            public void Dummy()
+            {
+            }
+
+            protected override int InitialParticipantsValueFactory { get; }
+        }
+
         public class DiverseConfig : MultiNodeConfig
         {
             public RoleName RoleProp { get; set; }

src/Akka.MultiNode.TestAdapter.Tests/Internal/MultiNodeTestRunnerDiscovery/DiscoverySpec.cs

@@ -21,7 +21,7 @@ public class DiscoverySpec
         public void No_abstract_classes()
         {
             var discoveredSpecs = DiscoverSpecs();
-            Assert.DoesNotContain(discoveredSpecs, s => s.TypeName == nameof(DiscoveryCases.NoAbstractClassesSpec));
+            Assert.False(discoveredSpecs.ContainsKey(KeyFromSpecName(nameof(DiscoveryCases.NoAbstractClassesSpec))));
         }
 
         [Fact(DisplayName = "Deeply inherited classes are discoverable")]
@@ -30,9 +30,7 @@ public void Deeply_inherited_are_ok()
             var discoveredSpecs = DiscoverSpecs();
             Assert.Equal(
                 "DeeplyInheritedChildRole", 
-                discoveredSpecs
-                    .FirstOrDefault(s => s.TypeName == KeyFromSpecName(nameof(DiscoveryCases.DeeplyInheritedChildSpec)))?.Nodes
-                    .First().Role);
+                discoveredSpecs[KeyFromSpecName(nameof(DiscoveryCases.DeeplyInheritedChildSpec))].First().Role);
         }
 
         [Fact(DisplayName = "Child test class with default constructors are ok")]
@@ -52,25 +50,19 @@ public void Child_class_with_default_constructor_are_ok()
         public void Discovered_count_equals_number_of_roles_mult_specs()
         {
             var discoveredSpecs = DiscoverSpecs();
-            Assert.Equal(5, discoveredSpecs
-                .FirstOrDefault(s => s.TypeName == KeyFromSpecName(nameof(DiscoveryCases.FloodyChildSpec1)))?.Nodes.Count);
-            Assert.Equal(5, discoveredSpecs
-                .FirstOrDefault(s => s.TypeName == KeyFromSpecName(nameof(DiscoveryCases.FloodyChildSpec2)))?.Nodes.Count);
-            Assert.Equal(5, discoveredSpecs
-                .FirstOrDefault(s => s.TypeName == KeyFromSpecName(nameof(DiscoveryCases.FloodyChildSpec3)))?.Nodes.Count);
+            Assert.Equal(5, discoveredSpecs[KeyFromSpecName(nameof(DiscoveryCases.FloodyChildSpec1))].Count);
+            Assert.Equal(5, discoveredSpecs[KeyFromSpecName(nameof(DiscoveryCases.FloodyChildSpec2))].Count);
+            Assert.Equal(5, discoveredSpecs[KeyFromSpecName(nameof(DiscoveryCases.FloodyChildSpec3))].Count);
         }
 
-        [Fact(DisplayName = "Only public props and fields are considered when looking for RoleNames")]
-        public void Public_props_and_fields_are_considered()
+        [Fact(DisplayName = "Only the MultiNodeConfig.Roles property is used to compute the number of Roles in MultiNodeFact")]
+        public void Only_MultiNodeConfig_role_count_used()
         {
             var discoveredSpecs = DiscoverSpecs();
-            Assert.Equal(
-                discoveredSpecs
-                    .FirstOrDefault(test => test.TypeName == KeyFromSpecName(nameof(DiscoveryCases.DiverseSpec)))?.Nodes
-                    .Select(n => n.Role), new[] {"RoleProp", "RoleField"});
+            Assert.Equal(10, discoveredSpecs[KeyFromSpecName(nameof(DiscoveryCases.NoReflectionSpec))].Select(c => c.Role).Count());
         }
-
-        private static List<MultiNodeTestCase> DiscoverSpecs()
+        
+        private static Dictionary<string, List<NodeTest>> DiscoverSpecs()
         {
             var assemblyPath = new Uri(typeof(DiscoveryCases).GetTypeInfo().Assembly.Location).LocalPath; 
 
@@ -80,7 +72,9 @@ private static List<MultiNodeTestCase> DiscoverSpecs()
                 {
                     controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery());
                     discovery.Finished.WaitOne();
-                    return discovery.TestCases;
+                    return discovery
+                        .TestCases
+                        .ToDictionary(t => t.TypeName, t => t.Nodes);
                 }
             }
         }

src/Akka.MultiNode.TestAdapter/Internal/CollectionPerSessionTestCollectionFactory.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Akka.MultiNode.TestAdapter.Internal
+{
+    internal class CollectionPerSessionTestCollectionFactory : IXunitTestCollectionFactory
+    {
+        private readonly Dictionary<IAssemblyInfo, TestCollection> _collectionCache =
+            new Dictionary<IAssemblyInfo, TestCollection>();
+
+        public ITestCollection Get(ITypeInfo testClass)
+        {
+            if (_collectionCache.TryGetValue(testClass.Assembly, out var collection))
+                return collection;
+
+            collection = new TestCollection(
+                new TestAssembly(testClass.Assembly),
+                null,
+                $"MultiNode test collection for {Path.GetFileName(testClass.Assembly.AssemblyPath)}");
+            _collectionCache[testClass.Assembly] = collection;
+            return collection;
+        }
+
+        public string DisplayName => "collection-per-session";
+    }
+}

src/Akka.MultiNode.TestAdapter/Internal/MultiNodeTestAssemblyRunner.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Akka.MultiNode.TestAdapter.Internal
+{
+    internal class MultiNodeTestAssemblyRunner : XunitTestAssemblyRunner
+    {
+        public MultiNodeTestAssemblyRunner(
+            ITestAssembly testAssembly,
+            IEnumerable<IXunitTestCase> testCases,
+            IMessageSink diagnosticMessageSink,
+            IMessageSink executionMessageSink,
+            ITestFrameworkExecutionOptions executionOptions) 
+            : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions)
+        {
+        }
+
+        protected override async Task<RunSummary> RunTestCollectionsAsync(IMessageBus messageBus, CancellationTokenSource cancellationTokenSource)
+        {
+            var summary = new RunSummary();
+
+            foreach (var (testCollection, testCases) in OrderTestCollections())
+            {
+                summary.Aggregate(await RunTestCollectionAsync(messageBus, testCollection, testCases, cancellationTokenSource));
+                if (cancellationTokenSource.IsCancellationRequested)
+                    break;
+            }
+
+            return summary;
+        }
+    }
+}

src/Akka.MultiNode.TestAdapter/Internal/MultiNodeTestCase.cs

@@ -143,16 +143,8 @@ private IEnumerable<RoleName> RoleNames(Type specType)
             {
                 var configType = ctorWithConfig.GetParameters().First().ParameterType;
                 var args = ConfigConstructorParamValues(configType);
-                var configInstance = Activator.CreateInstance(configType, args);
-                var roleType = typeof(RoleName);
-                var configProps = configType.GetProperties(BindingFlags.Instance | BindingFlags.Public);
-                var roleProps = configProps.Where(p => p.PropertyType == roleType && p.Name != "Myself")
-                    .Select(p => (RoleName) p.GetValue(configInstance));
-                var configFields = configType.GetFields(BindingFlags.Instance | BindingFlags.Public);
-                var roleFields = configFields.Where(f => f.FieldType == roleType && f.Name != "Myself")
-                    .Select(f => (RoleName) f.GetValue(configInstance));
-                var roles = roleProps.Concat(roleFields).Distinct();
-                return roles;
+                var configInstance = (MultiNodeConfig) Activator.CreateInstance(configType, args);
+                return configInstance.Roles;
             }
             catch (Exception e)
             {

src/Akka.MultiNode.TestAdapter/Internal/MultiNodeTestCaseRunner.cs

@@ -225,16 +225,12 @@ protected override async Task<RunSummary> RunTestAsync()
 
         private async Task DumpAggregatedSpecLogs(RunSummary summary, IActorRef timelineCollector)
         {
-            var dumpPath = Path.GetFullPath(Path.Combine(Path.Combine(Options.OutputDirectory, TestCase.DisplayName), "aggregated.txt"));
-            var failedSpecPath = Path.GetFullPath(Path.Combine(Options.OutputDirectory, Options.FailedSpecsDirectory, $"{TestCase.DisplayName}.txt"));
-
-            if (!Options.AppendLogOutput)
-            {
-                if(File.Exists(dumpPath))
-                    File.Delete(dumpPath);
-                if(File.Exists(failedSpecPath))
-                    File.Delete(failedSpecPath);
-            }
+            var dumpFolder = Path.GetFullPath(Path.Combine(Options.OutputDirectory, TestCase.DisplayName)); 
+            var dumpPath = Path.Combine(dumpFolder, "aggregated.txt");
+
+            Directory.CreateDirectory(dumpFolder);                
+            if (!Options.AppendLogOutput && File.Exists(dumpPath))
+                File.Delete(dumpPath);
 
             var logLines = await timelineCollector.Ask<string[]>(new TimelineLogCollectorActor.GetLog());
 
@@ -243,8 +239,10 @@ private async Task DumpAggregatedSpecLogs(RunSummary summary, IActorRef timeline
 
             if (summary.Failed > 0)
             {
-                Directory.CreateDirectory(Path.GetDirectoryName(failedSpecPath));
+                var failedSpecFolder = Path.GetFullPath(Path.Combine(Options.OutputDirectory, Options.FailedSpecsDirectory));
+                var failedSpecPath = Path.Combine(failedSpecFolder, $"{TestCase.DisplayName}.txt");
 
+                Directory.CreateDirectory(failedSpecFolder);
                 if(!Options.AppendLogOutput && File.Exists(failedSpecPath))
                     File.Delete(failedSpecPath);
 

src/Akka.MultiNode.TestAdapter/Internal/MultiNodeTestFrameworkExecutor.cs

@@ -0,0 +1,25 @@
+using System.Collections.Generic;
+using System.Reflection;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Akka.MultiNode.TestAdapter.Internal
+{
+    internal class MultiNodeTestFrameworkExecutor : XunitTestFrameworkExecutor
+    {
+        public MultiNodeTestFrameworkExecutor(
+            AssemblyName assemblyName,
+            ISourceInformationProvider sourceInformationProvider,
+            IMessageSink diagnosticMessageSink) 
+            : base(assemblyName, sourceInformationProvider, diagnosticMessageSink)
+        {
+        }
+
+        protected override async void RunTestCases(IEnumerable<IXunitTestCase> testCases, IMessageSink executionMessageSink,
+            ITestFrameworkExecutionOptions executionOptions)
+        {
+            using (var assemblyRunner = new MultiNodeTestAssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, executionMessageSink, executionOptions))
+                await assemblyRunner.RunAsync();
+        }
+    }
+}

src/Akka.MultiNode.TestAdapter/Internal/Reporting/TestRunCoordinator.cs

@@ -111,11 +111,11 @@ private void SetReceive()
                 if (_currentSpecRunActor == null) return;
                 _currentSpecRunActor.Forward(message);
             });
-            Receive<BeginNewSpec>(spec => ReceiveBeginSpecRun(spec));
-            ReceiveAsync<EndSpec>(spec => ReceiveEndSpecRun(spec));
+            Receive<BeginNewSpec>(ReceiveBeginSpecRun);
+            ReceiveAsync<EndSpec>(spec => _currentSpecRunActor != null ? ReceiveEndSpecRun(spec) : Task.CompletedTask);
             Receive<RequestTestRunState>(state => Sender.Tell(TestRunData.Copy(TestRunPassed(TestRunData))));
-            Receive<SubscribeFactCompletionMessages>(messages => AddSubscriber(messages));
-            Receive<UnsubscribeFactCompletionMessages>(messages => RemoveSubscriber(messages));
+            Receive<SubscribeFactCompletionMessages>(AddSubscriber);
+            Receive<UnsubscribeFactCompletionMessages>(RemoveSubscriber);
             ReceiveAsync<EndTestRun>(async run =>
             {
                 //clean up the current spec, if it hasn't been done already

src/Akka.MultiNode.TestAdapter/MultiNodeTestFramework.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Reflection;
+using Akka.MultiNode.TestAdapter.Internal;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Akka.MultiNode.TestAdapter
+{
+    public class MultiNodeTestFramework : TestFramework
+    {
+        public MultiNodeTestFramework(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink)
+        {
+        }
+
+        protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo)
+        {
+            return new XunitTestFrameworkDiscoverer(
+                assemblyInfo: assemblyInfo,
+                sourceProvider: SourceInformationProvider,
+                diagnosticMessageSink: DiagnosticMessageSink,
+                collectionFactory: new CollectionPerSessionTestCollectionFactory());
+        }
+
+        protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName)
+        {
+            return new MultiNodeTestFrameworkExecutor(assemblyName, SourceInformationProvider, DiagnosticMessageSink);
+        }
+    }
+}

src/common.props

@@ -24,7 +24,7 @@
     <NetStandardLibVersion>netstandard2.0</NetStandardLibVersion>
     <FluentAssertionsVersion>6.2.0</FluentAssertionsVersion>
     <FsCheckVersion>2.9.0</FsCheckVersion>
-    <AkkaVersion>1.4.27</AkkaVersion>
+    <AkkaVersion>1.4.31</AkkaVersion>
     <!--
     <AkkaVersion>1.4.28-beta637717616449070703</AkkaVersion>
     -->
@@ -38,7 +38,7 @@
   </PropertyGroup>
   <!-- SourceLink support for all Akka.NET projects -->
   <ItemGroup>
-    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
   </ItemGroup>
   <PropertyGroup>
     <PublishRepositoryUrl>true</PublishRepositoryUrl>