diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/MicrosoftTestPlatformRunnerPoolTests.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/MicrosoftTestPlatformRunnerPoolTests.cs index 7789a7e084..41f1c2fd24 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/MicrosoftTestPlatformRunnerPoolTests.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/MicrosoftTestPlatformRunnerPoolTests.cs @@ -5,7 +5,9 @@ using Shouldly; using Stryker.Abstractions; using Stryker.Abstractions.Options; +using Stryker.Abstractions.Testing; using Stryker.TestRunner.MicrosoftTestPlatform.Models; +using Stryker.TestRunner.Results; using Stryker.TestRunner.Tests; namespace Stryker.TestRunner.MicrosoftTestPlatform.UnitTest; @@ -314,6 +316,280 @@ public void Constructor_ShouldUseDefaultLogger_WhenLoggerIsNull() // Assert pool.ShouldNotBeNull(); } + + [TestMethod] + public void CaptureCoverage_ShouldUsePerTestCapture_WhenCoverageBasedTestEnabled() + { + // Arrange + var options = new Mock(); + options.Setup(x => x.Concurrency).Returns(1); + options.Setup(x => x.OptimizationMode).Returns(OptimizationModes.CoverageBasedTest); + + var testsByAssembly = new Dictionary>(); + var testDescriptions = new Dictionary(); + var testSet = new TestSet(); + + var testNode1 = new TestNode("test-1", "Test1", "test", "discovered"); + var testNode2 = new TestNode("test-2", "Test2", "test", "discovered"); + testsByAssembly["assembly.dll"] = new List { testNode1, testNode2 }; + + var desc1 = new MtpTestDescription(testNode1); + var desc2 = new MtpTestDescription(testNode2); + testDescriptions["test-1"] = desc1; + testDescriptions["test-2"] = desc2; + testSet.RegisterTest(desc1.Description); + testSet.RegisterTest(desc2.Description); + + var capturedTests = new System.Collections.Concurrent.ConcurrentBag(); + + var runnerFactory = new Mock(); + runnerFactory.Setup(x => x.CreateRunner( + It.IsAny(), + It.IsAny>>(), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns>, Dictionary, TestSet, object, ILogger, IStrykerOptions>( + (id, tba, td, ts, dl, logger, opts) => + { + // Populate pool's dictionaries via references + if (tba.Count == 0) + { + foreach (var kvp in testsByAssembly) + tba[kvp.Key] = kvp.Value; + foreach (var kvp in testDescriptions) + td[kvp.Key] = kvp.Value; + } + return new TestableRunner(id, tba, td, ts, dl, + () => { }, + coverageHandler: (assembly, test, testId, confidence) => + { + capturedTests.Add(testId); + var covered = testId == desc1.Id + ? new[] { 1, 2 } + : new[] { 3 }; + return Task.FromResult( + CoverageRunResult.Create(testId, confidence, covered, Array.Empty(), Array.Empty())); + }); + }); + + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns(new[] { "assembly.dll" }); + + using var pool = new MicrosoftTestPlatformRunnerPool(options.Object, NullLogger.Instance, runnerFactory.Object); + + // Act + var coverage = pool.CaptureCoverage(project.Object).ToList(); + + // Assert + capturedTests.Count.ShouldBe(2, "Both tests should have been captured individually"); + coverage.Count.ShouldBe(2, "Should return one coverage result per test"); + + var cov1 = coverage.First(c => c.TestId == desc1.Id); + cov1.MutationsCovered.ShouldContain(1); + cov1.MutationsCovered.ShouldContain(2); + cov1.MutationsCovered.ShouldNotContain(3); + cov1.Confidence.ShouldBe(CoverageConfidence.Normal); + + var cov2 = coverage.First(c => c.TestId == desc2.Id); + cov2.MutationsCovered.ShouldContain(3); + cov2.MutationsCovered.ShouldNotContain(1); + cov2.Confidence.ShouldBe(CoverageConfidence.Normal); + } + + [TestMethod] + public void CaptureCoverage_ShouldUseExactConfidence_WhenPerTestInIsolationEnabled() + { + // Arrange + var options = new Mock(); + options.Setup(x => x.Concurrency).Returns(1); + options.Setup(x => x.OptimizationMode).Returns( + OptimizationModes.CoverageBasedTest | OptimizationModes.CaptureCoveragePerTest); + + var testsByAssembly = new Dictionary>(); + var testDescriptions = new Dictionary(); + var testSet = new TestSet(); + + var testNode = new TestNode("test-1", "Test1", "test", "discovered"); + testsByAssembly["assembly.dll"] = new List { testNode }; + var desc = new MtpTestDescription(testNode); + testDescriptions["test-1"] = desc; + testSet.RegisterTest(desc.Description); + + CoverageConfidence? capturedConfidence = null; + + var runnerFactory = new Mock(); + runnerFactory.Setup(x => x.CreateRunner( + It.IsAny(), + It.IsAny>>(), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns>, Dictionary, TestSet, object, ILogger, IStrykerOptions>( + (id, tba, td, ts, dl, logger, opts) => + { + if (tba.Count == 0) + { + foreach (var kvp in testsByAssembly) + tba[kvp.Key] = kvp.Value; + foreach (var kvp in testDescriptions) + td[kvp.Key] = kvp.Value; + } + return new TestableRunner(id, tba, td, ts, dl, + () => { }, + coverageHandler: (assembly, test, testId, confidence) => + { + capturedConfidence = confidence; + return Task.FromResult( + CoverageRunResult.Create(testId, confidence, new[] { 1 }, Array.Empty(), Array.Empty())); + }); + }); + + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns(new[] { "assembly.dll" }); + + using var pool = new MicrosoftTestPlatformRunnerPool(options.Object, NullLogger.Instance, runnerFactory.Object); + + // Act + var coverage = pool.CaptureCoverage(project.Object).ToList(); + + // Assert + capturedConfidence.ShouldBe(CoverageConfidence.Exact, + "perTestInIsolation should use Exact confidence"); + coverage.Single().Confidence.ShouldBe(CoverageConfidence.Exact); + } + + [TestMethod] + public void CaptureCoverage_ShouldFallbackToAggregate_WhenOnlySkipUncoveredEnabled() + { + // Arrange: "all" mode = SkipUncoveredMutants only (no CoverageBasedTest) + var options = new Mock(); + options.Setup(x => x.Concurrency).Returns(1); + options.Setup(x => x.OptimizationMode).Returns(OptimizationModes.SkipUncoveredMutants); + using var pool = new MicrosoftTestPlatformRunnerPool(options.Object, NullLogger.Instance); + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns(Array.Empty()); + + // Act + var coverage = pool.CaptureCoverage(project.Object).ToList(); + + // Assert: aggregate mode returns empty when no tests discovered + coverage.ShouldBeEmpty(); + } + + [TestMethod] + public void CaptureCoverage_ShouldHandleEmptyTestSet_InPerTestMode() + { + // Arrange + var options = new Mock(); + options.Setup(x => x.Concurrency).Returns(1); + options.Setup(x => x.OptimizationMode).Returns(OptimizationModes.CoverageBasedTest); + + var runnerFactory = new Mock(); + runnerFactory.Setup(x => x.CreateRunner( + It.IsAny(), + It.IsAny>>(), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns>, Dictionary, TestSet, object, ILogger, IStrykerOptions>( + (id, tba, td, ts, dl, logger, opts) => + new TestableRunner(id, () => { })); + + using var pool = new MicrosoftTestPlatformRunnerPool(options.Object, NullLogger.Instance, runnerFactory.Object); + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns(Array.Empty()); + + // Act + var coverage = pool.CaptureCoverage(project.Object).ToList(); + + // Assert: no tests = no coverage results + coverage.ShouldBeEmpty(); + } + + [TestMethod] + public void CaptureCoverage_ShouldHandleMultipleAssemblies_InPerTestMode() + { + // Arrange + var options = new Mock(); + options.Setup(x => x.Concurrency).Returns(2); + options.Setup(x => x.OptimizationMode).Returns(OptimizationModes.CoverageBasedTest); + + var testsByAssembly = new Dictionary>(); + var testDescriptions = new Dictionary(); + var testSet = new TestSet(); + + // Assembly 1: 2 tests + var testA1 = new TestNode("a1-test-1", "A1Test1", "test", "discovered"); + var testA2 = new TestNode("a1-test-2", "A1Test2", "test", "discovered"); + testsByAssembly["assembly1.dll"] = new List { testA1, testA2 }; + + // Assembly 2: 1 test + var testB1 = new TestNode("a2-test-1", "A2Test1", "test", "discovered"); + testsByAssembly["assembly2.dll"] = new List { testB1 }; + + foreach (var (_, tests) in testsByAssembly) + { + foreach (var t in tests) + { + var desc = new MtpTestDescription(t); + testDescriptions[t.Uid] = desc; + testSet.RegisterTest(desc.Description); + } + } + + var capturedAssemblies = new System.Collections.Concurrent.ConcurrentBag(); + + var runnerFactory = new Mock(); + runnerFactory.Setup(x => x.CreateRunner( + It.IsAny(), + It.IsAny>>(), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns>, Dictionary, TestSet, object, ILogger, IStrykerOptions>( + (id, tba, td, ts, dl, logger, opts) => + { + // Populate pool's dictionaries via references + if (tba.Count == 0) + { + foreach (var kvp in testsByAssembly) + tba[kvp.Key] = kvp.Value; + foreach (var kvp in testDescriptions) + td[kvp.Key] = kvp.Value; + } + return new TestableRunner(id, tba, td, ts, dl, + () => { }, + coverageHandler: (assembly, test, testId, confidence) => + { + capturedAssemblies.Add(assembly); + return Task.FromResult( + CoverageRunResult.Create(testId, confidence, new[] { 1 }, Array.Empty(), Array.Empty())); + }); + }); + + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns(new[] { "assembly1.dll", "assembly2.dll" }); + + using var pool = new MicrosoftTestPlatformRunnerPool(options.Object, NullLogger.Instance, runnerFactory.Object); + + // Act + var coverage = pool.CaptureCoverage(project.Object).ToList(); + + // Assert + coverage.Count.ShouldBe(3, "Should capture coverage for all 3 tests across both assemblies"); + capturedAssemblies.Count.ShouldBe(3); + capturedAssemblies.Count(a => a == "assembly1.dll").ShouldBe(2); + capturedAssemblies.Count(a => a == "assembly2.dll").ShouldBe(1); + } } diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs index cc9e11ff5f..95b08ca732 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs @@ -1,6 +1,12 @@ +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Moq; using Shouldly; +using Stryker.Abstractions; +using Stryker.Abstractions.Options; +using Stryker.Abstractions.Testing; using Stryker.TestRunner.MicrosoftTestPlatform.Models; +using Stryker.TestRunner.Results; using Stryker.TestRunner.Tests; namespace Stryker.TestRunner.MicrosoftTestPlatform.UnitTest; @@ -57,10 +63,10 @@ public async Task SetCoverageMode_ShouldEnableCoverageMode() var result = await runner.DiscoverTestsAsync(testAssembly); result.ShouldBeTrue("Server should be recreated successfully after enabling coverage mode"); - // Trying to enable again should be a no-op + // Enabling again should still delete any stale coverage file (defensive cleanup) await File.WriteAllTextAsync(coverageFilePath, "test"); runner.SetCoverageMode(true); - File.Exists(coverageFilePath).ShouldBeTrue("Should not delete file when mode is already enabled"); + File.Exists(coverageFilePath).ShouldBeFalse("Should delete stale coverage file even when mode is already enabled"); } finally { @@ -107,10 +113,10 @@ public async Task SetCoverageMode_ShouldDisableCoverageMode() var result = await runner.DiscoverTestsAsync(testAssembly); result.ShouldBeTrue("Server should be recreated successfully after disabling coverage mode"); - // Trying to disable again should be a no-op (no servers disposed, no file deletion) + // Disabling again should still delete any stale coverage file (defensive cleanup) await File.WriteAllTextAsync(coverageFilePath, "test"); runner.SetCoverageMode(false); - File.Exists(coverageFilePath).ShouldBeTrue("Should not delete file when mode is already disabled"); + File.Exists(coverageFilePath).ShouldBeFalse("Should delete stale coverage file even when mode is already disabled"); } finally { @@ -144,13 +150,12 @@ public async Task SetCoverageMode_ShouldNoOp_WhenModeIsAlreadySet() runner.SetCoverageMode(true); File.Exists(coverageFilePath).ShouldBeFalse("Coverage file should be deleted on first enable"); - // Create a coverage file to verify no-op doesn't delete it + // Create a coverage file to verify defensive cleanup still happens await File.WriteAllTextAsync(coverageFilePath, "test-data"); - - // Try to enable again - should do nothing (no server disposal, no file deletion) + + // Try to enable again - servers should NOT be disposed, but stale coverage file should be deleted runner.SetCoverageMode(true); - File.Exists(coverageFilePath).ShouldBeTrue("Coverage file should NOT be deleted when mode already enabled"); - (await File.ReadAllTextAsync(coverageFilePath)).ShouldBe("test-data", "File content should be unchanged"); + File.Exists(coverageFilePath).ShouldBeFalse("Stale coverage file should be deleted even when mode already enabled"); // Verify servers are still functional (not disposed) var result = await runner.DiscoverTestsAsync(testAssembly); @@ -485,4 +490,197 @@ public async Task ResetServerAsync_ShouldDisposeAndClearAllServers() var serversAfter = (Dictionary)serversField.GetValue(runner)!; serversAfter.ShouldBeEmpty("all servers should be disposed and removed after reset"); } + + [TestMethod] + public async Task StopAndRemoveServerAsync_ShouldRemoveServerFromDictionary() + { + var runnerId = 610; + using var runner = new SingleMicrosoftTestPlatformRunner( + runnerId, + _testsByAssembly, + _testDescriptions, + _testSet, + _discoveryLock, + NullLogger.Instance); + + var testAssembly = typeof(SingleMicrosoftTestPlatformRunnerCoverageTests).Assembly.Location; + await runner.DiscoverTestsAsync(testAssembly); + + var serversField = typeof(SingleMicrosoftTestPlatformRunner) + .GetField("_assemblyServers", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!; + + var serversBefore = (Dictionary)serversField.GetValue(runner)!; + serversBefore.ShouldNotBeEmpty("servers should exist after discovery"); + + var method = typeof(SingleMicrosoftTestPlatformRunner) + .GetMethod("StopAndRemoveServerAsync", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!; + await (Task)method.Invoke(runner, new object[] { testAssembly })!; + + var serversAfter = (Dictionary)serversField.GetValue(runner)!; + serversAfter.ContainsKey(testAssembly).ShouldBeFalse("server should be removed after stop"); + } + + [TestMethod] + public void ReadCoverageData_ShouldReturnCoveredAndStaticMutants_FromFile() + { + var runnerId = 620; + var coverageFilePath = Path.Combine(Path.GetTempPath(), $"stryker-coverage-{runnerId}.txt"); + + try + { + using var runner = new SingleMicrosoftTestPlatformRunner( + runnerId, + _testsByAssembly, + _testDescriptions, + _testSet, + _discoveryLock, + NullLogger.Instance); + + File.WriteAllText(coverageFilePath, "1,2,3;10"); + + var result = runner.ReadCoverageData(); + + result.CoveredMutants.Count.ShouldBe(3); + result.CoveredMutants.ShouldContain(1); + result.CoveredMutants.ShouldContain(2); + result.CoveredMutants.ShouldContain(3); + result.StaticMutants.Count.ShouldBe(1); + result.StaticMutants.ShouldContain(10); + } + finally + { + if (File.Exists(coverageFilePath)) + { + File.Delete(coverageFilePath); + } + } + } + + [TestMethod] + public void ReadCoverageData_ShouldReturnEmpty_WhenNoCoverageFile() + { + var runnerId = 621; + using var runner = new SingleMicrosoftTestPlatformRunner( + runnerId, + _testsByAssembly, + _testDescriptions, + _testSet, + _discoveryLock, + NullLogger.Instance); + + var result = runner.ReadCoverageData(); + + result.CoveredMutants.ShouldBeEmpty(); + result.StaticMutants.ShouldBeEmpty(); + } + + [TestMethod] + public void CaptureCoverageTestByTest_ShouldReturnDubious_WhenHandlerThrows() + { + var options = new Mock(); + options.Setup(x => x.Concurrency).Returns(1); + options.Setup(x => x.OptimizationMode).Returns(OptimizationModes.CoverageBasedTest); + + var testNode = new TestNode("test-1", "ThrowingTest", "test", "discovered"); + var testsByAssembly = new Dictionary> + { + ["assembly.dll"] = [testNode] + }; + var testDescriptions = new Dictionary + { + ["test-1"] = new(testNode) + }; + + var runnerFactory = new Mock(); + runnerFactory.Setup(x => x.CreateRunner( + It.IsAny(), + It.IsAny>>(), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns>, Dictionary, TestSet, object, ILogger, IStrykerOptions>( + (id, tba, td, ts, dl, logger, opts) => + { + if (tba.Count == 0) + { + foreach (var kvp in testsByAssembly) + tba[kvp.Key] = kvp.Value; + foreach (var kvp in testDescriptions) + td[kvp.Key] = kvp.Value; + } + return new TestableRunner(id, tba, td, ts, dl, + () => { }, + coverageHandler: (_, _, _, _) => + throw new InvalidOperationException("Server startup failed")); + }); + + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns(new[] { "assembly.dll" }); + + using var pool = new MicrosoftTestPlatformRunnerPool(options.Object, NullLogger.Instance, runnerFactory.Object); + + var coverage = pool.CaptureCoverage(project.Object).ToList(); + + coverage.Count.ShouldBe(1); + coverage[0].Confidence.ShouldBe(CoverageConfidence.Dubious); + coverage[0].MutationsCovered.ShouldBeEmpty(); + } + + [TestMethod] + public void CaptureCoverageTestByTest_ShouldReturnDubious_WhenCoverageIsEmpty() + { + var options = new Mock(); + options.Setup(x => x.Concurrency).Returns(1); + options.Setup(x => x.OptimizationMode).Returns(OptimizationModes.CoverageBasedTest); + + var testNode = new TestNode("test-1", "NoCoverageTest", "test", "discovered"); + var testsByAssembly = new Dictionary> + { + ["assembly.dll"] = [testNode] + }; + var testDescriptions = new Dictionary + { + ["test-1"] = new(testNode) + }; + + var runnerFactory = new Mock(); + runnerFactory.Setup(x => x.CreateRunner( + It.IsAny(), + It.IsAny>>(), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns>, Dictionary, TestSet, object, ILogger, IStrykerOptions>( + (id, tba, td, ts, dl, logger, opts) => + { + if (tba.Count == 0) + { + foreach (var kvp in testsByAssembly) + tba[kvp.Key] = kvp.Value; + foreach (var kvp in testDescriptions) + td[kvp.Key] = kvp.Value; + } + return new TestableRunner(id, tba, td, ts, dl, + () => { }, + coverageHandler: (_, _, testId, _) => + Task.FromResult( + CoverageRunResult.Create(testId, CoverageConfidence.Dubious, + Array.Empty(), Array.Empty(), Array.Empty()))); + }); + + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns(new[] { "assembly.dll" }); + + using var pool = new MicrosoftTestPlatformRunnerPool(options.Object, NullLogger.Instance, runnerFactory.Object); + + var coverage = pool.CaptureCoverage(project.Object).ToList(); + + coverage.Count.ShouldBe(1); + coverage[0].Confidence.ShouldBe(CoverageConfidence.Dubious); + coverage[0].MutationsCovered.ShouldBeEmpty(); + } } diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/TestableRunner.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/TestableRunner.cs index 5f425decd4..b98b9d857a 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/TestableRunner.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/TestableRunner.cs @@ -1,5 +1,7 @@ using Microsoft.Extensions.Logging.Abstractions; +using Stryker.Abstractions.Testing; using Stryker.TestRunner.MicrosoftTestPlatform.Models; +using Stryker.TestRunner.Results; using Stryker.TestRunner.Tests; namespace Stryker.TestRunner.MicrosoftTestPlatform.UnitTest; @@ -7,17 +9,51 @@ namespace Stryker.TestRunner.MicrosoftTestPlatform.UnitTest; internal class TestableRunner : SingleMicrosoftTestPlatformRunner { private readonly Action _onDispose; + private readonly Func>? _coverageHandler; - public TestableRunner(int id, Action onDispose) - : base(id, new Dictionary>(), - new Dictionary(), - new TestSet(), - new object(), + public TestableRunner(int id, Action onDispose) + : base(id, new Dictionary>(), + new Dictionary(), + new TestSet(), + new object(), NullLogger.Instance) { _onDispose = onDispose; } + public TestableRunner( + int id, + Dictionary> testsByAssembly, + Dictionary testDescriptions, + TestSet testSet, + object discoveryLock, + Action onDispose, + Func>? coverageHandler = null) + : base(id, testsByAssembly, testDescriptions, testSet, discoveryLock, NullLogger.Instance) + { + _onDispose = onDispose; + _coverageHandler = coverageHandler; + } + + internal override async Task RunSingleTestForCoverageAsync( + string assembly, TestNode test, string testId, CoverageConfidence confidence) + { + if (_coverageHandler is not null) + { + try + { + return await _coverageHandler(assembly, test, testId, confidence).ConfigureAwait(false); + } + catch + { + return CoverageRunResult.Create(testId, CoverageConfidence.Dubious, + Array.Empty(), Array.Empty(), Array.Empty()); + } + } + + return CoverageRunResult.Create(testId, confidence, Array.Empty(), Array.Empty(), Array.Empty()); + } + public override void Dispose(bool disposing) { _onDispose?.Invoke(); diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs index 12c44f48ea..c70a0becf5 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs @@ -17,7 +17,7 @@ namespace Stryker.TestRunner.MicrosoftTestPlatform; /// public sealed class MicrosoftTestPlatformRunnerPool : ITestRunner { - private readonly AutoResetEvent _runnerAvailableHandler = new(false); + private readonly SemaphoreSlim _runnerAvailable; private readonly ConcurrentBag _availableRunners = new(); private readonly ILogger _logger; private readonly int _countOfRunners; @@ -36,6 +36,7 @@ public MicrosoftTestPlatformRunnerPool(IStrykerOptions options, ILogger? logger _options = options; _countOfRunners = Math.Max(1, options.Concurrency); _runnerFactory = runnerFactory ?? new DefaultRunnerFactory(); + _runnerAvailable = new SemaphoreSlim(0, _countOfRunners); _logger.LogWarning("The Microsoft Test Platform testrunner is currently in preview. Results should be verified since this feature is still being tested."); Initialize(); @@ -63,7 +64,7 @@ private void Initialize() _logger, _options); _availableRunners.Add(runner); - _runnerAvailableHandler.Set(); + _runnerAvailable.Release(); }); } @@ -82,7 +83,8 @@ public async Task DiscoverTestsAsync(string assembly) public async Task InitialTestAsync(IProjectAndTests project) { var assemblies = project.GetTestAssemblies(); - if (!assemblies.Any()) + ArgumentNullException.ThrowIfNull(assemblies); + if (assemblies.Count == 0) { return new TestRunResult(false, "No test assemblies found"); } @@ -97,9 +99,21 @@ public async Task InitialTestAsync(IProjectAndTests project) public IEnumerable CaptureCoverage(IProjectAndTests project) { - _logger.LogInformation("Starting coverage capture for MTP runner"); + if (_options.OptimizationMode.HasFlag(OptimizationModes.CoverageBasedTest)) + { + var confidence = _options.OptimizationMode.HasFlag(OptimizationModes.CaptureCoveragePerTest) + ? CoverageConfidence.Exact + : CoverageConfidence.Normal; + return CaptureCoverageTestByTest(confidence); + } + + return CaptureCoverageInOneGo(project); + } + + private IEnumerable CaptureCoverageInOneGo(IProjectAndTests project) + { + _logger.LogInformation("Starting aggregate coverage capture for MTP runner"); - // Enable coverage mode on all runners foreach (var runner in _availableRunners) { runner.SetCoverageMode(true); @@ -107,7 +121,6 @@ public IEnumerable CaptureCoverage(IProjectAndTests project) try { - // Run all tests with coverage tracking enabled var testResult = RunThisAsync(runner => runner.InitialTestAsync(project)).GetAwaiter().GetResult(); if (testResult.FailingTests.IsEveryTest) @@ -115,10 +128,8 @@ public IEnumerable CaptureCoverage(IProjectAndTests project) _logger.LogWarning("Coverage test run failed: {Message}", testResult.ResultMessage); } - // Reset test processes to trigger coverage file flush (process exit writes coverage) ResetTestProcesses(); - // Aggregate coverage data from all runners var allCoveredMutants = new HashSet(); var allStaticMutants = new HashSet(); @@ -135,12 +146,9 @@ public IEnumerable CaptureCoverage(IProjectAndTests project) } } - _logger.LogInformation("Coverage capture complete: {CoveredCount} mutations covered, {StaticCount} static mutations", + _logger.LogInformation("Aggregate coverage capture complete: {CoveredCount} mutations covered, {StaticCount} static mutations", allCoveredMutants.Count, allStaticMutants.Count); - // For cumulative coverage, we return a single coverage result that applies to all tests - // Each test is assumed to cover all the mutations that were covered during the full test run - // Static mutants are marked as such for proper handling during mutation testing return _testDescriptions.Values.Select(testDescription => CoverageRunResult.Create( testDescription.Id, @@ -151,7 +159,63 @@ public IEnumerable CaptureCoverage(IProjectAndTests project) } finally { - // Disable coverage mode on all runners for subsequent mutation testing + foreach (var runner in _availableRunners) + { + runner.SetCoverageMode(false); + } + } + } + + private IEnumerable CaptureCoverageTestByTest( + CoverageConfidence confidence) + { + _logger.LogInformation("Starting per-test coverage capture for MTP runner"); + + foreach (var runner in _availableRunners) + { + runner.SetCoverageMode(true); + } + + try + { + var allTests = new List<(string Assembly, TestNode Test, string TestId)>(); + foreach (var (assembly, tests) in _testsByAssembly) + { + foreach (var test in tests) + { + if (_testDescriptions.TryGetValue(test.Uid, out var desc)) + { + allTests.Add((assembly, test, desc.Id)); + } + } + } + + _logger.LogInformation("Capturing per-test coverage for {TestCount} tests across {AssemblyCount} assemblies", + allTests.Count, _testsByAssembly.Count); + + var results = new ConcurrentBag(); + + Parallel.ForEach(allTests, + new ParallelOptions { MaxDegreeOfParallelism = _countOfRunners }, + testInfo => + { + var result = RunThisAsync(async runner => + await runner.RunSingleTestForCoverageAsync( + testInfo.Assembly, testInfo.Test, testInfo.TestId, confidence) + .ConfigureAwait(false)) + .GetAwaiter().GetResult(); + + results.Add(result); + }); + + _logger.LogInformation( + "Per-test coverage capture complete: {TestCount} tests captured", + results.Count); + + return results; + } + finally + { foreach (var runner in _availableRunners) { runner.SetCoverageMode(false); @@ -166,7 +230,8 @@ public async Task TestMultipleMutantsAsync( TestUpdateHandler? update) { var assemblies = project.GetTestAssemblies(); - if (!assemblies.Any()) + ArgumentNullException.ThrowIfNull(assemblies); + if (assemblies.Count == 0) { return new TestRunResult(false, "No test assemblies found"); } @@ -176,40 +241,36 @@ public async Task TestMultipleMutantsAsync( private async Task RunThisAsync(Func> task) { - SingleMicrosoftTestPlatformRunner? runner; - - // Try to get a runner with a timeout to prevent indefinite blocking - var attempts = 0; - const int maxWaitTimeSeconds = 300; // 5 minutes max wait - const int waitIntervalMs = 1000; // Check every second - var maxAttempts = maxWaitTimeSeconds * 1000 / waitIntervalMs; + const int maxWaitTimeSeconds = 300; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(maxWaitTimeSeconds)); - while (!_availableRunners.TryTake(out runner)) + // Single CTS shared across retries so the timeout is a hard upper bound + while (true) { - if (!_runnerAvailableHandler.WaitOne(waitIntervalMs)) + try { - attempts++; - if (attempts >= maxAttempts) - { - throw new TimeoutException($"Timed out waiting for an available test runner after {maxWaitTimeSeconds} seconds. Available runners: {_availableRunners.Count}, Total runners: {_countOfRunners}"); - } + await _runnerAvailable.WaitAsync(cts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw new TimeoutException($"Timed out waiting for an available test runner after {maxWaitTimeSeconds} seconds. Available runners: {_availableRunners.Count}, Total runners: {_countOfRunners}"); + } - if (attempts % 30 == 0) // Log every 30 seconds - { - _logger.LogWarning("Waiting for available test runner... ({Attempts}s elapsed, {Available}/{Total} runners available)", - attempts, _availableRunners.Count, _countOfRunners); - } + if (!_availableRunners.TryTake(out var runner)) + { + _runnerAvailable.Release(); + continue; } - } - try - { - return await task(runner).ConfigureAwait(false); - } - finally - { - _availableRunners.Add(runner); - _runnerAvailableHandler.Set(); + try + { + return await task(runner).ConfigureAwait(false); + } + finally + { + _availableRunners.Add(runner); + _runnerAvailable.Release(); + } } } @@ -219,7 +280,7 @@ public void Dispose() { runner.Dispose(); } - _runnerAvailableHandler.Dispose(); + _runnerAvailable.Dispose(); } } diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs index 61edf42003..e93f5633a4 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs @@ -29,11 +29,11 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable private readonly IStrykerOptions? _options; private readonly Dictionary _assemblyServers = new(); - private readonly object _serverLock = new(); + private readonly SemaphoreSlim _serverLock = new(1, 1); private bool _disposed; private bool _coverageMode; - private string RunnerId => $"MtpRunner-{_id}"; + private readonly string _runnerId; public SingleMicrosoftTestPlatformRunner( int id, @@ -55,6 +55,7 @@ public SingleMicrosoftTestPlatformRunner( // Create unique file paths for this runner to communicate with the test process _mutantFilePath = Path.Combine(Path.GetTempPath(), $"stryker-mutant-{_id}.txt"); _coverageFilePath = Path.Combine(Path.GetTempPath(), $"stryker-coverage-{_id}.txt"); + _runnerId = $"MtpRunner-{_id}"; // Initialize with no active mutation WriteMutantIdToFile(-1); @@ -83,17 +84,21 @@ public Task TestMultipleMutantsAsync( // When testing a single mutant, activate it; otherwise use -1 (no mutation) var mutantId = mutants.Count == 1 ? mutants[0].Id : -1; - _logger.LogDebug("{RunnerId}: Testing mutant(s) [{Mutants}] with active mutation ID: {MutantId}", - RunnerId, string.Join(",", mutants.Select(m => m.Id)), mutantId); + if (_logger.IsEnabled(Microsoft.Extensions.Logging.LogLevel.Debug)) + { + _logger.LogDebug("{RunnerId}: Testing mutant(s) [{Mutants}] with active mutation ID: {MutantId}", + _runnerId, string.Join(",", mutants.Select(m => m.Id)), mutantId); + } return RunAllTestsAsync(assemblies, mutantId, mutants, update, timeoutCalc); } public async Task ResetServerAsync() { - _logger.LogDebug("{RunnerId}: Resetting test servers to reload assemblies", RunnerId); + _logger.LogDebug("{RunnerId}: Resetting test servers to reload assemblies", _runnerId); - lock (_serverLock) + await _serverLock.WaitAsync().ConfigureAwait(false); + try { foreach (var server in _assemblyServers.Values) { @@ -101,9 +106,98 @@ public async Task ResetServerAsync() } _assemblyServers.Clear(); } + finally + { + _serverLock.Release(); + } - _logger.LogDebug("{RunnerId}: Test servers reset complete", RunnerId); - await Task.CompletedTask; + _logger.LogDebug("{RunnerId}: Test servers reset complete", _runnerId); + } + + /// + /// Stops and removes the server for a specific assembly. This triggers ProcessExit + /// in the test process, causing MutantControl.FlushCoverageToFile() to be called. + /// The server is removed from the cache so a fresh one is created on next use. + /// + internal async Task StopAndRemoveServerAsync(string assembly) + { + AssemblyTestServer? server; + await _serverLock.WaitAsync().ConfigureAwait(false); + try + { + _assemblyServers.TryGetValue(assembly, out server); + _assemblyServers.Remove(assembly); + } + finally + { + _serverLock.Release(); + } + + if (server is not null) + { + await server.StopAsync().ConfigureAwait(false); + } + } + + /// + /// Runs a single test in isolation to capture its per-test coverage data. + /// The flow is: start server → run one test → stop server (triggers coverage flush) → read coverage file. + /// This is used by the pool's CaptureCoverageTestByTest method. + /// + internal virtual async Task RunSingleTestForCoverageAsync( + string assembly, TestNode test, string testId, CoverageConfidence confidence) + { + try + { + DeleteCoverageFile(); + + var server = await GetOrCreateServerAsync(assembly).ConfigureAwait(false); + await server.RunTestsAsync(new[] { test }).ConfigureAwait(false); + await StopAndRemoveServerAsync(assembly).ConfigureAwait(false); + + var (coveredMutants, staticMutants) = ReadCoverageData(); + + DeleteCoverageFile(); + + // Empty coverage likely means the process was force-killed before FlushCoverageToFile ran + if (coveredMutants.Count == 0 && staticMutants.Count == 0) + { + _logger.LogWarning( + "{RunnerId}: No coverage data captured for test {TestId} — coverage file was empty or missing. Marking as Dubious.", + _runnerId, testId); + + return CoverageRunResult.Create( + testId, + CoverageConfidence.Dubious, + coveredMutants, + staticMutants, + Array.Empty()); + } + + _logger.LogDebug( + "{RunnerId}: Test {TestId} covers {CoveredCount} mutants ({StaticCount} static)", + _runnerId, testId, coveredMutants.Count, staticMutants.Count); + + return CoverageRunResult.Create( + testId, + confidence, + coveredMutants, + staticMutants, + Array.Empty()); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "{RunnerId}: Failed to capture coverage for test {TestId}", _runnerId, testId); + try { await StopAndRemoveServerAsync(assembly).ConfigureAwait(false); } + catch { /* best-effort cleanup to prevent server leak */ } + DeleteCoverageFile(); + return CoverageRunResult.Create( + testId, + CoverageConfidence.Dubious, + Array.Empty(), + Array.Empty(), + Array.Empty()); + } } private void WriteMutantIdToFile(int mutantId) @@ -112,12 +206,12 @@ private void WriteMutantIdToFile(int mutantId) { File.WriteAllText(_mutantFilePath, mutantId.ToString()); _logger.LogDebug("{RunnerId}: Wrote mutant ID {MutantId} to file {FilePath}", - RunnerId, mutantId, _mutantFilePath); + _runnerId, mutantId, _mutantFilePath); } catch (Exception ex) { _logger.LogWarning(ex, "{RunnerId}: Failed to write mutant ID to file {FilePath}", - RunnerId, _mutantFilePath); + _runnerId, _mutantFilePath); } } @@ -143,26 +237,28 @@ private void WriteMutantIdToFile(int mutantId) /// public void SetCoverageMode(bool enabled) { - lock (_serverLock) + _serverLock.Wait(); + try { - if (_coverageMode == enabled) + if (_coverageMode != enabled) { - // Already in the desired state; no action needed - return; - } - - _coverageMode = enabled; - _logger.LogDebug("{RunnerId}: Coverage mode {Status}", RunnerId, enabled ? "enabled" : "disabled"); + _coverageMode = enabled; + _logger.LogDebug("{RunnerId}: Coverage mode {Status}", _runnerId, enabled ? "enabled" : "disabled"); - // Reset servers to apply the new environment variables - foreach (var server in _assemblyServers.Values) - { - server.Dispose(); + foreach (var server in _assemblyServers.Values) + { + server.Dispose(); + } + _assemblyServers.Clear(); } - _assemblyServers.Clear(); + } + finally + { + _serverLock.Release(); } - // Clean up any existing coverage file, even when enabling, to ensure we start fresh + // Always clean up any existing coverage file to prevent stale data, + // even when the mode hasn't changed (e.g. retry/re-run paths) DeleteCoverageFile(); } @@ -174,14 +270,14 @@ public void SetCoverageMode(bool enabled) { if (!File.Exists(_coverageFilePath)) { - _logger.LogDebug("{RunnerId}: Coverage file not found at {Path}", RunnerId, _coverageFilePath); + _logger.LogDebug("{RunnerId}: Coverage file not found at {Path}", _runnerId, _coverageFilePath); return (Array.Empty(), Array.Empty()); } try { var content = File.ReadAllText(_coverageFilePath).Trim(); - _logger.LogDebug("{RunnerId}: Read coverage data: {Content}", RunnerId, content); + _logger.LogDebug("{RunnerId}: Read coverage data: {Content}", _runnerId, content); if (string.IsNullOrEmpty(content)) { @@ -196,7 +292,7 @@ public void SetCoverageMode(bool enabled) } catch (Exception ex) { - _logger.LogWarning(ex, "{RunnerId}: Failed to read coverage file at {Path}", RunnerId, _coverageFilePath); + _logger.LogWarning(ex, "{RunnerId}: Failed to read coverage file at {Path}", _runnerId, _coverageFilePath); return (Array.Empty(), Array.Empty()); } } @@ -208,12 +304,16 @@ private static IReadOnlyList ParseMutantIds(string idString) return Array.Empty(); } - return idString - .Split(',', StringSplitOptions.RemoveEmptyEntries) - .Select(s => int.TryParse(s.Trim(), out var id) ? id : (int?)null) - .Where(id => id.HasValue) - .Select(id => id.Value) - .ToList(); + var parts = idString.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var result = new List(parts.Length); + foreach (var part in parts) + { + if (int.TryParse(part, out var id)) + { + result.Add(id); + } + } + return result; } private void DeleteCoverageFile() @@ -227,36 +327,36 @@ private void DeleteCoverageFile() } catch (Exception ex) { - _logger.LogWarning(ex, "{RunnerId}: Failed to delete coverage file at {Path}", RunnerId, _coverageFilePath); + _logger.LogWarning(ex, "{RunnerId}: Failed to delete coverage file at {Path}", _runnerId, _coverageFilePath); } } private async Task GetOrCreateServerAsync(string assembly) { - AssemblyTestServer? server; - lock (_serverLock) + await _serverLock.WaitAsync().ConfigureAwait(false); + try { - if (_assemblyServers.TryGetValue(assembly, out server) && server.IsInitialized) + if (_assemblyServers.TryGetValue(assembly, out var existing) && existing.IsInitialized) { - return server; + return existing; } - } - var environmentVariables = BuildEnvironmentVariables(); - server = new AssemblyTestServer(assembly, environmentVariables, _logger, RunnerId, _options); + var environmentVariables = BuildEnvironmentVariables(); + var server = new AssemblyTestServer(assembly, environmentVariables, _logger, _runnerId, _options); - var started = await server.StartAsync().ConfigureAwait(false); - if (!started) - { - throw new InvalidOperationException($"Failed to start test server for {assembly}"); - } + var started = await server.StartAsync().ConfigureAwait(false); + if (!started) + { + throw new InvalidOperationException($"Failed to start test server for {assembly}"); + } - lock (_serverLock) - { _assemblyServers[assembly] = server; + return server; + } + finally + { + _serverLock.Release(); } - - return server; } private async Task DiscoverTestsInternalAsync(string assembly) @@ -278,12 +378,12 @@ private async Task DiscoverTestsInternalAsync(string assembly) } } - _logger.LogDebug("{RunnerId}: Discovered {TestCount} tests in {Assembly}", RunnerId, tests.Count, assembly); + _logger.LogDebug("{RunnerId}: Discovered {TestCount} tests in {Assembly}", _runnerId, tests.Count, assembly); return tests.Count > 0; } catch (Exception ex) { - _logger.LogDebug(ex, "{RunnerId}: Failed to discover tests in {Assembly}", RunnerId, assembly); + _logger.LogDebug(ex, "{RunnerId}: Failed to discover tests in {Assembly}", _runnerId, assembly); return false; } } @@ -298,40 +398,42 @@ private async Task DiscoverTestsInternalAsync(string assembly) internal TimeSpan? CalculateAssemblyTimeout(List discoveredTests, ITimeoutValueCalculator timeoutCalc, string assembly) { - var estimatedTimeMs = (int)discoveredTests - .Where(t => _testDescriptions.TryGetValue(t.Uid, out _)) - .Sum(t => - { - lock (_discoveryLock) - { - return _testDescriptions.TryGetValue(t.Uid, out var desc) - ? desc.InitialRunTime.TotalMilliseconds - : 0; - } - }); - + int estimatedTimeMs; + lock (_discoveryLock) + { + estimatedTimeMs = (int)discoveredTests + .Sum(t => _testDescriptions.TryGetValue(t.Uid, out var desc) + ? desc.InitialRunTime.TotalMilliseconds + : 0); + } + var timeoutMs = timeoutCalc.CalculateTimeoutValue(estimatedTimeMs); _logger.LogDebug("{RunnerId}: Using {TimeoutMs} ms as test run timeout for {Assembly}", - RunnerId, timeoutMs, Path.GetFileName(assembly)); - + _runnerId, timeoutMs, Path.GetFileName(assembly)); + return TimeSpan.FromMilliseconds(timeoutMs); } internal async Task HandleAssemblyTimeoutAsync(string assembly, List discoveredTests, List allTimedOutTests) { - _logger.LogDebug("{RunnerId}: Test run timed out for {Assembly}", RunnerId, Path.GetFileName(assembly)); + _logger.LogDebug("{RunnerId}: Test run timed out for {Assembly}", _runnerId, Path.GetFileName(assembly)); allTimedOutTests.AddRange(discoveredTests.Select(t => t.Uid)); AssemblyTestServer? server; - lock (_serverLock) + await _serverLock.WaitAsync().ConfigureAwait(false); + try { _assemblyServers.TryGetValue(assembly, out server); } + finally + { + _serverLock.Release(); + } if (server is not null) { - _logger.LogDebug("{RunnerId}: Restarting test server for {Assembly} after timeout", RunnerId, Path.GetFileName(assembly)); + _logger.LogDebug("{RunnerId}: Restarting test server for {Assembly} after timeout", _runnerId, Path.GetFileName(assembly)); await server.RestartAsync().ConfigureAwait(false); } } @@ -356,9 +458,9 @@ public void Aggregate(TestRunResult result, List? discoveredTests) } else { - var executedIds = result.ExecutedTests.GetIdentifiers().ToList(); - _executedTests.AddRange(executedIds); - _totalExecutedTests += executedIds.Count; + var before = _executedTests.Count; + _executedTests.AddRange(result.ExecutedTests.GetIdentifiers()); + _totalExecutedTests += _executedTests.Count - before; } _failedTests.AddRange(result.FailingTests.GetIdentifiers()); @@ -389,7 +491,8 @@ public ITestIdentifiers BuildExecutedTests() => internal async Task<(TestRunResult? Result, bool TimedOut, List? DiscoveredTests)> ProcessSingleAssemblyAsync( string assembly, - ITimeoutValueCalculator? timeoutCalc) + ITimeoutValueCalculator? timeoutCalc, + bool registerInitialResults = false) { if (!File.Exists(assembly)) { @@ -397,15 +500,15 @@ public ITestIdentifiers BuildExecutedTests() => } var discoveredTests = GetDiscoveredTests(assembly); - + TimeSpan? timeout = null; if (timeoutCalc is not null && discoveredTests is not null) { timeout = CalculateAssemblyTimeout(discoveredTests, timeoutCalc, assembly); } - var (testResults, timedOut) = await RunTestsInternalAsync(assembly, null, timeout).ConfigureAwait(false); - + var (testResults, timedOut) = await RunTestsInternalAsync(assembly, null, timeout, registerInitialResults).ConfigureAwait(false); + return (testResults as TestRunResult, timedOut, discoveredTests); } @@ -424,7 +527,7 @@ private async Task RunAllTestsAsync( foreach (var assembly in assemblies) { - var (result, timedOut, discoveredTests) = await ProcessSingleAssemblyAsync(assembly, timeoutCalc).ConfigureAwait(false); + var (result, timedOut, discoveredTests) = await ProcessSingleAssemblyAsync(assembly, timeoutCalc, mutantId == -1).ConfigureAwait(false); if (discoveredTests is not null) { @@ -446,29 +549,26 @@ private async Task RunAllTestsAsync( var failedTestIds = accumulator.BuildFailedTests(); var timedOutTestIds = accumulator.BuildTimedOutTests(); - IEnumerable testDescriptionValues; - lock (_discoveryLock) - { - testDescriptionValues = _testDescriptions.Values.ToList(); - } - if (update is not null && mutants is not null) { update.Invoke(mutants, failedTestIds, executedTests, timedOutTestIds); } - return new TestRunResult( - testDescriptionValues, - executedTests, - failedTestIds, - timedOutTestIds, - accumulator.BuildErrorMessage(), - accumulator.Messages, - accumulator.TotalDuration); + lock (_discoveryLock) + { + return new TestRunResult( + _testDescriptions.Values, + executedTests, + failedTestIds, + timedOutTestIds, + accumulator.BuildErrorMessage(), + accumulator.Messages, + accumulator.TotalDuration); + } } catch (Exception ex) { - _logger.LogDebug(ex, "{RunnerId}: Failed to run tests for mutant ID {MutantId}", RunnerId, mutantId); + _logger.LogDebug(ex, "{RunnerId}: Failed to run tests for mutant ID {MutantId}", _runnerId, mutantId); return new TestRunResult(false, ex.Message); } } @@ -476,13 +576,13 @@ private async Task RunAllTestsAsync( internal async Task<(ITestRunResult Result, bool TimedOut)> RunTestsInternalAsync( string assembly, Func? testUidFilter, - TimeSpan? timeout = null) + TimeSpan? timeout = null, + bool registerInitialResults = false) { var startTime = DateTime.UtcNow; try { - // Get or create the server for this assembly (reuses existing server) var server = await GetOrCreateServerAsync(assembly).ConfigureAwait(false); List? tests = null; @@ -499,48 +599,61 @@ private async Task RunAllTestsAsync( var (testResults, timedOut) = await server.RunTestsAsync(testsToRun, timeout).ConfigureAwait(false); var duration = DateTime.UtcNow - startTime; - var finishedTests = testResults.Where(x => x.Node.ExecutionState is not "in-progress").ToList(); - var failedTests = finishedTests.Where(x => x.Node.ExecutionState is "failed").Select(x => x.Node.Uid).ToList(); - lock (_discoveryLock) + // Single pass: build finished/failed lists, error messages, and per-test messages together + var finishedTests = new List(testResults.Count); + var failedTestUids = new List(); + var errorParts = new List(); + var messageParts = new List(); + foreach (var tr in testResults) { - foreach (var testResult in finishedTests.Where(tr => _testDescriptions.ContainsKey(tr.Node.Uid))) + if (tr.Node.ExecutionState is "in-progress") { - var testDescription = _testDescriptions[testResult.Node.Uid]; - testDescription.RegisterInitialTestResult(new MtpTestResult(duration)); + continue; } - } - var errorMessagesStr = string.Join(Environment.NewLine, - finishedTests.Where(x => x.Node.ExecutionState is "failed") - .Select(x => $"{x.Node.DisplayName}{Environment.NewLine}{Environment.NewLine}Test failed")); + finishedTests.Add(tr); + messageParts.Add($"{tr.Node.DisplayName}{Environment.NewLine}{Environment.NewLine}State: {tr.Node.ExecutionState}"); - var messages = finishedTests.Select(x => - $"{x.Node.DisplayName}{Environment.NewLine}{Environment.NewLine}State: {x.Node.ExecutionState}"); + if (tr.Node.ExecutionState is "failed") + { + failedTestUids.Add(tr.Node.Uid); + errorParts.Add($"{tr.Node.DisplayName}{Environment.NewLine}{Environment.NewLine}Test failed"); + } + } + + var errorMessagesStr = string.Join(Environment.NewLine, errorParts); var totalDiscoveredTests = tests?.Count ?? 0; - var executedTestCount = finishedTests.Count; - var executedTests = totalDiscoveredTests > 0 && executedTestCount >= totalDiscoveredTests + var executedTests = totalDiscoveredTests > 0 && finishedTests.Count >= totalDiscoveredTests ? TestIdentifierList.EveryTest() : new TestIdentifierList(finishedTests.Select(x => x.Node.Uid)); - var failedTestIds = new TestIdentifierList(failedTests); - + var failedTestIds = new TestIdentifierList(failedTestUids); - IEnumerable testDescriptionValues; + TestRunResult result; lock (_discoveryLock) { - testDescriptionValues = _testDescriptions.Values.ToList(); - } + if (registerInitialResults) + { + foreach (var tr in finishedTests) + { + if (_testDescriptions.TryGetValue(tr.Node.Uid, out var desc)) + { + desc.RegisterInitialTestResult(new MtpTestResult(duration)); + } + } + } - var result = new TestRunResult( - testDescriptionValues, - executedTests, - failedTestIds, - TestIdentifierList.NoTest(), - errorMessagesStr, - messages, - duration); + result = new TestRunResult( + _testDescriptions.Values, + executedTests, + failedTestIds, + TestIdentifierList.NoTest(), + errorMessagesStr, + messageParts, + duration); + } return (result, timedOut); } @@ -565,7 +678,8 @@ public virtual void Dispose(bool disposing) if (disposing) { - lock (_serverLock) + _serverLock.Wait(); + try { foreach (var server in _assemblyServers.Values) { @@ -573,6 +687,10 @@ public virtual void Dispose(bool disposing) } _assemblyServers.Clear(); } + finally + { + _serverLock.Release(); + } // Clean up temp files try @@ -589,7 +707,7 @@ public virtual void Dispose(bool disposing) catch (Exception ex) { // Ignore cleanup errors - _logger.LogWarning(ex, "{RunnerId}: Failed to clean up temp files", RunnerId); + _logger.LogWarning(ex, "{RunnerId}: Failed to clean up temp files", _runnerId); } } _disposed = true;