diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/MemoryDiagnoserTests.cs old mode 100755 new mode 100644 similarity index 94% rename from tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs rename to tests/BenchmarkDotNet.IntegrationTests.ManualRunning/MemoryDiagnoserTests.cs index cf573f9985..fbe989fa88 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/MemoryDiagnoserTests.cs @@ -24,8 +24,11 @@ using Xunit.Abstractions; using BenchmarkDotNet.Toolchains.Mono; -namespace BenchmarkDotNet.IntegrationTests +namespace BenchmarkDotNet.IntegrationTests.ManualRunning { + // #1925 + // These tests were moved to ManualRunning to stabilize the CI. + // Even after disabling tiered jit and blocking the finalizer thread, the tests are still flaky on CI. public class MemoryDiagnoserTests { private readonly ITestOutputHelper output; @@ -51,7 +54,6 @@ public class AccurateAllocations } [Theory, MemberData(nameof(GetToolchains))] - [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurate(IToolchain toolchain) { long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word @@ -109,7 +111,6 @@ private void AllocateUntilGcWakesUp() } [Theory, MemberData(nameof(GetToolchains))] - [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserDoesNotIncludeAllocationsFromSetupAndCleanup(IToolchain toolchain) { AssertAllocations(toolchain, typeof(AllocatingGlobalSetupAndCleanup), new Dictionary @@ -135,7 +136,6 @@ public ulong TimeConsuming() } [Theory, MemberData(nameof(GetToolchains))] - [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void EngineShouldNotInterfereAllocationResults(IToolchain toolchain) { AssertAllocations(toolchain, typeof(NoAllocationsAtAll), new Dictionary @@ -146,7 +146,6 @@ public void EngineShouldNotInterfereAllocationResults(IToolchain toolchain) // #1542 [Theory, MemberData(nameof(GetToolchains))] - [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void TieredJitShouldNotInterfereAllocationResults(IToolchain toolchain) { AssertAllocations(toolchain, typeof(NoAllocationsAtAll), new Dictionary @@ -162,7 +161,6 @@ public class NoBoxing } [Theory, MemberData(nameof(GetToolchains))] - [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void EngineShouldNotIntroduceBoxing(IToolchain toolchain) { AssertAllocations(toolchain, typeof(NoBoxing), new Dictionary @@ -183,7 +181,6 @@ public class NonAllocatingAsynchronousBenchmarks } [Theory, MemberData(nameof(GetToolchains))] - [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AwaitingTasksShouldNotInterfereAllocationResults(IToolchain toolchain) { AssertAllocations(toolchain, typeof(NonAllocatingAsynchronousBenchmarks), new Dictionary @@ -208,7 +205,6 @@ public void WithOperationsPerInvoke() } [Theory, MemberData(nameof(GetToolchains))] - [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocatedMemoryShouldBeScaledForOperationsPerInvoke(IToolchain toolchain) { long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word @@ -234,7 +230,6 @@ public byte[] SixtyFourBytesArray() } [TheoryEnvSpecific("Full Framework cannot measure precisely enough for low invocation counts.", EnvRequirement.DotNetCoreOnly), MemberData(nameof(GetToolchains))] - [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void AllocationQuantumIsNotAnIssueForNetCore21Plus(IToolchain toolchain) { long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word @@ -300,7 +295,6 @@ public void Allocate() [TheoryEnvSpecific("Full Framework cannot measure precisely enough", EnvRequirement.DotNetCoreOnly)] [MemberData(nameof(GetToolchains))] - [Trait(Constants.Category, Constants.BackwardCompatibilityCategory)] public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolchain) { long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word @@ -325,12 +319,8 @@ private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Diction } catch (MisconfiguredEnvironmentException e) { - if (ContinuousIntegration.IsLocalRun()) - { - output.WriteLine(e.SkipMessage); - return; - } - throw; + output.WriteLine(e.SkipMessage); + return; } foreach (var benchmarkAllocationsValidator in benchmarksAllocationsValidators)