From 01259f5576609d1e5d143c0dc68ba8993eb3087b Mon Sep 17 00:00:00 2001 From: Piotr Nawrot Date: Wed, 1 Apr 2026 17:24:21 +0200 Subject: [PATCH 1/9] refactor(MTP): extract CaptureCoverageInOneGo and add routing in CaptureCoverage Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../MicrosoftTestPlatformRunnerPool.cs | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs index 12c44f48ea..18c29e60d5 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs @@ -97,9 +97,21 @@ public async Task InitialTestAsync(IProjectAndTests project) public IEnumerable CaptureCoverage(IProjectAndTests project) { - _logger.LogInformation("Starting coverage capture for MTP runner"); + if (_options.OptimizationMode.HasFlag(OptimizationModes.CoverageBasedTest)) + { + var confidence = _options.OptimizationMode.HasFlag(OptimizationModes.CaptureCoveragePerTest) + ? CoverageConfidence.Exact + : CoverageConfidence.Normal; + return CaptureCoverageTestByTest(project, confidence); + } + + return CaptureCoverageInOneGo(project); + } + + private IEnumerable CaptureCoverageInOneGo(IProjectAndTests project) + { + _logger.LogInformation("Starting aggregate coverage capture for MTP runner"); - // Enable coverage mode on all runners foreach (var runner in _availableRunners) { runner.SetCoverageMode(true); @@ -107,7 +119,6 @@ public IEnumerable CaptureCoverage(IProjectAndTests project) try { - // Run all tests with coverage tracking enabled var testResult = RunThisAsync(runner => runner.InitialTestAsync(project)).GetAwaiter().GetResult(); if (testResult.FailingTests.IsEveryTest) @@ -115,10 +126,8 @@ public IEnumerable CaptureCoverage(IProjectAndTests project) _logger.LogWarning("Coverage test run failed: {Message}", testResult.ResultMessage); } - // Reset test processes to trigger coverage file flush (process exit writes coverage) ResetTestProcesses(); - // Aggregate coverage data from all runners var allCoveredMutants = new HashSet(); var allStaticMutants = new HashSet(); @@ -135,12 +144,9 @@ public IEnumerable CaptureCoverage(IProjectAndTests project) } } - _logger.LogInformation("Coverage capture complete: {CoveredCount} mutations covered, {StaticCount} static mutations", + _logger.LogInformation("Aggregate coverage capture complete: {CoveredCount} mutations covered, {StaticCount} static mutations", allCoveredMutants.Count, allStaticMutants.Count); - // For cumulative coverage, we return a single coverage result that applies to all tests - // Each test is assumed to cover all the mutations that were covered during the full test run - // Static mutants are marked as such for proper handling during mutation testing return _testDescriptions.Values.Select(testDescription => CoverageRunResult.Create( testDescription.Id, @@ -151,7 +157,6 @@ public IEnumerable CaptureCoverage(IProjectAndTests project) } finally { - // Disable coverage mode on all runners for subsequent mutation testing foreach (var runner in _availableRunners) { runner.SetCoverageMode(false); @@ -159,6 +164,13 @@ public IEnumerable CaptureCoverage(IProjectAndTests project) } } + // Stub: real implementation added in Task 6 + private IEnumerable CaptureCoverageTestByTest( + IProjectAndTests project, CoverageConfidence confidence) + { + throw new NotImplementedException("CaptureCoverageTestByTest will be implemented in Task 6"); + } + public async Task TestMultipleMutantsAsync( IProjectAndTests project, ITimeoutValueCalculator? timeoutCalc, From df69e009be1e2e75a591e66969325b5dc4c565fa Mon Sep 17 00:00:00 2001 From: Piotr Nawrot Date: Wed, 1 Apr 2026 17:24:41 +0200 Subject: [PATCH 2/9] test(MTP): extend TestableRunner to support per-test coverage handler Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../TestableRunner.cs | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/TestableRunner.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/TestableRunner.cs index 5f425decd4..032f566fcf 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/TestableRunner.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/TestableRunner.cs @@ -1,5 +1,7 @@ using Microsoft.Extensions.Logging.Abstractions; +using Stryker.Abstractions.Testing; using Stryker.TestRunner.MicrosoftTestPlatform.Models; +using Stryker.TestRunner.Results; using Stryker.TestRunner.Tests; namespace Stryker.TestRunner.MicrosoftTestPlatform.UnitTest; @@ -7,17 +9,43 @@ namespace Stryker.TestRunner.MicrosoftTestPlatform.UnitTest; internal class TestableRunner : SingleMicrosoftTestPlatformRunner { private readonly Action _onDispose; + private readonly Func>? _coverageHandler; - public TestableRunner(int id, Action onDispose) - : base(id, new Dictionary>(), - new Dictionary(), - new TestSet(), - new object(), + public TestableRunner(int id, Action onDispose) + : base(id, new Dictionary>(), + new Dictionary(), + new TestSet(), + new object(), NullLogger.Instance) { _onDispose = onDispose; } + public TestableRunner( + int id, + Dictionary> testsByAssembly, + Dictionary testDescriptions, + TestSet testSet, + object discoveryLock, + Action onDispose, + Func>? coverageHandler = null) + : base(id, testsByAssembly, testDescriptions, testSet, discoveryLock, NullLogger.Instance) + { + _onDispose = onDispose; + _coverageHandler = coverageHandler; + } + + internal override async Task RunSingleTestForCoverageAsync( + string assembly, TestNode test, string testId, CoverageConfidence confidence) + { + if (_coverageHandler is not null) + { + return await _coverageHandler(assembly, test, testId, confidence).ConfigureAwait(false); + } + + return CoverageRunResult.Create(testId, confidence, Array.Empty(), Array.Empty(), Array.Empty()); + } + public override void Dispose(bool disposing) { _onDispose?.Invoke(); From f55bb61e641e4ccef3b1dd0a3d949fda786d6a39 Mon Sep 17 00:00:00 2001 From: Piotr Nawrot Date: Wed, 1 Apr 2026 17:24:55 +0200 Subject: [PATCH 3/9] feat(MTP): add StopAndRemoveServerAsync helper for per-test coverage Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...icrosoftTestPlatformRunnerCoverageTests.cs | 83 +++++++++++++++++++ .../SingleMicrosoftTestPlatformRunner.cs | 65 +++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs index cc9e11ff5f..4a9343173d 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs @@ -485,4 +485,87 @@ public async Task ResetServerAsync_ShouldDisposeAndClearAllServers() var serversAfter = (Dictionary)serversField.GetValue(runner)!; serversAfter.ShouldBeEmpty("all servers should be disposed and removed after reset"); } + + [TestMethod] + public async Task StopAndRemoveServerAsync_ShouldRemoveServerFromDictionary() + { + var runnerId = 610; + using var runner = new SingleMicrosoftTestPlatformRunner( + runnerId, + _testsByAssembly, + _testDescriptions, + _testSet, + _discoveryLock, + NullLogger.Instance); + + var testAssembly = typeof(SingleMicrosoftTestPlatformRunnerCoverageTests).Assembly.Location; + await runner.DiscoverTestsAsync(testAssembly); + + var serversField = typeof(SingleMicrosoftTestPlatformRunner) + .GetField("_assemblyServers", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!; + + var serversBefore = (Dictionary)serversField.GetValue(runner)!; + serversBefore.ShouldNotBeEmpty("servers should exist after discovery"); + + var method = typeof(SingleMicrosoftTestPlatformRunner) + .GetMethod("StopAndRemoveServerAsync", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!; + await (Task)method.Invoke(runner, new object[] { testAssembly })!; + + var serversAfter = (Dictionary)serversField.GetValue(runner)!; + serversAfter.ContainsKey(testAssembly).ShouldBeFalse("server should be removed after stop"); + } + + [TestMethod] + public async Task RunSingleTestForCoverageAsync_ShouldReturnCoverageFromFile() + { + var runnerId = 620; + var coverageFilePath = Path.Combine(Path.GetTempPath(), $"stryker-coverage-{runnerId}.txt"); + + try + { + using var runner = new SingleMicrosoftTestPlatformRunner( + runnerId, + _testsByAssembly, + _testDescriptions, + _testSet, + _discoveryLock, + NullLogger.Instance); + + File.WriteAllText(coverageFilePath, "1,2,3;10"); + + var result = runner.ReadCoverageData(); + + result.CoveredMutants.Count.ShouldBe(3); + result.CoveredMutants.ShouldContain(1); + result.CoveredMutants.ShouldContain(2); + result.CoveredMutants.ShouldContain(3); + result.StaticMutants.Count.ShouldBe(1); + result.StaticMutants.ShouldContain(10); + } + finally + { + if (File.Exists(coverageFilePath)) + { + File.Delete(coverageFilePath); + } + } + } + + [TestMethod] + public void RunSingleTestForCoverageAsync_ShouldReturnDubious_WhenNoCoverageFile() + { + var runnerId = 621; + using var runner = new SingleMicrosoftTestPlatformRunner( + runnerId, + _testsByAssembly, + _testDescriptions, + _testSet, + _discoveryLock, + NullLogger.Instance); + + var result = runner.ReadCoverageData(); + + result.CoveredMutants.ShouldBeEmpty(); + result.StaticMutants.ShouldBeEmpty(); + } } diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs index 61edf42003..94462f8d39 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs @@ -106,6 +106,71 @@ public async Task ResetServerAsync() await Task.CompletedTask; } + /// + /// Stops and removes the server for a specific assembly. This triggers ProcessExit + /// in the test process, causing MutantControl.FlushCoverageToFile() to be called. + /// The server is removed from the cache so a fresh one is created on next use. + /// + internal async Task StopAndRemoveServerAsync(string assembly) + { + AssemblyTestServer? server; + lock (_serverLock) + { + _assemblyServers.TryGetValue(assembly, out server); + _assemblyServers.Remove(assembly); + } + + if (server is not null) + { + await server.StopAsync().ConfigureAwait(false); + } + } + + /// + /// Runs a single test in isolation to capture its per-test coverage data. + /// The flow is: start server → run one test → stop server (triggers coverage flush) → read coverage file. + /// This is used by the pool's CaptureCoverageTestByTest method. + /// + internal virtual async Task RunSingleTestForCoverageAsync( + string assembly, TestNode test, string testId, CoverageConfidence confidence) + { + try + { + DeleteCoverageFile(); + + var server = await GetOrCreateServerAsync(assembly).ConfigureAwait(false); + await server.RunTestsAsync(new[] { test }).ConfigureAwait(false); + await StopAndRemoveServerAsync(assembly).ConfigureAwait(false); + + var (coveredMutants, staticMutants) = ReadCoverageData(); + + _logger.LogDebug( + "{RunnerId}: Test {TestId} covers {CoveredCount} mutants ({StaticCount} static)", + RunnerId, testId, coveredMutants.Count, staticMutants.Count); + + DeleteCoverageFile(); + + return CoverageRunResult.Create( + testId, + confidence, + coveredMutants, + staticMutants, + Array.Empty()); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "{RunnerId}: Failed to capture coverage for test {TestId}", RunnerId, testId); + + return CoverageRunResult.Create( + testId, + CoverageConfidence.Dubious, + Array.Empty(), + Array.Empty(), + Array.Empty()); + } + } + private void WriteMutantIdToFile(int mutantId) { try From d275345513d884b9386252d0c33491bcee49fe2c Mon Sep 17 00:00:00 2001 From: Piotr Nawrot Date: Wed, 1 Apr 2026 17:34:42 +0200 Subject: [PATCH 4/9] feat(MTP): implement per-test coverage capture via isolated test execution MTP runner now captures per-test coverage by running each test in an isolated process. When coverage-analysis is set to perTest or perTestInIsolation, each test gets its own MTP server process. The server is stopped after each test, triggering MutantControl.FlushCoverageToFile() via ProcessExit, and the resulting coverage file is read to build per-test coverage results. This enables Stryker to determine which tests cover which mutants for MTP-based frameworks (xUnit v3, TUnit, MSTest with MTP, NUnit with MTP), unlocking coverage-based test optimization that was previously only available with VsTest. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../MicrosoftTestPlatformRunnerPoolTests.cs | 276 ++++++++++++++++++ .../MicrosoftTestPlatformRunnerPool.cs | 54 +++- 2 files changed, 328 insertions(+), 2 deletions(-) diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/MicrosoftTestPlatformRunnerPoolTests.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/MicrosoftTestPlatformRunnerPoolTests.cs index 7789a7e084..41f1c2fd24 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/MicrosoftTestPlatformRunnerPoolTests.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/MicrosoftTestPlatformRunnerPoolTests.cs @@ -5,7 +5,9 @@ using Shouldly; using Stryker.Abstractions; using Stryker.Abstractions.Options; +using Stryker.Abstractions.Testing; using Stryker.TestRunner.MicrosoftTestPlatform.Models; +using Stryker.TestRunner.Results; using Stryker.TestRunner.Tests; namespace Stryker.TestRunner.MicrosoftTestPlatform.UnitTest; @@ -314,6 +316,280 @@ public void Constructor_ShouldUseDefaultLogger_WhenLoggerIsNull() // Assert pool.ShouldNotBeNull(); } + + [TestMethod] + public void CaptureCoverage_ShouldUsePerTestCapture_WhenCoverageBasedTestEnabled() + { + // Arrange + var options = new Mock(); + options.Setup(x => x.Concurrency).Returns(1); + options.Setup(x => x.OptimizationMode).Returns(OptimizationModes.CoverageBasedTest); + + var testsByAssembly = new Dictionary>(); + var testDescriptions = new Dictionary(); + var testSet = new TestSet(); + + var testNode1 = new TestNode("test-1", "Test1", "test", "discovered"); + var testNode2 = new TestNode("test-2", "Test2", "test", "discovered"); + testsByAssembly["assembly.dll"] = new List { testNode1, testNode2 }; + + var desc1 = new MtpTestDescription(testNode1); + var desc2 = new MtpTestDescription(testNode2); + testDescriptions["test-1"] = desc1; + testDescriptions["test-2"] = desc2; + testSet.RegisterTest(desc1.Description); + testSet.RegisterTest(desc2.Description); + + var capturedTests = new System.Collections.Concurrent.ConcurrentBag(); + + var runnerFactory = new Mock(); + runnerFactory.Setup(x => x.CreateRunner( + It.IsAny(), + It.IsAny>>(), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns>, Dictionary, TestSet, object, ILogger, IStrykerOptions>( + (id, tba, td, ts, dl, logger, opts) => + { + // Populate pool's dictionaries via references + if (tba.Count == 0) + { + foreach (var kvp in testsByAssembly) + tba[kvp.Key] = kvp.Value; + foreach (var kvp in testDescriptions) + td[kvp.Key] = kvp.Value; + } + return new TestableRunner(id, tba, td, ts, dl, + () => { }, + coverageHandler: (assembly, test, testId, confidence) => + { + capturedTests.Add(testId); + var covered = testId == desc1.Id + ? new[] { 1, 2 } + : new[] { 3 }; + return Task.FromResult( + CoverageRunResult.Create(testId, confidence, covered, Array.Empty(), Array.Empty())); + }); + }); + + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns(new[] { "assembly.dll" }); + + using var pool = new MicrosoftTestPlatformRunnerPool(options.Object, NullLogger.Instance, runnerFactory.Object); + + // Act + var coverage = pool.CaptureCoverage(project.Object).ToList(); + + // Assert + capturedTests.Count.ShouldBe(2, "Both tests should have been captured individually"); + coverage.Count.ShouldBe(2, "Should return one coverage result per test"); + + var cov1 = coverage.First(c => c.TestId == desc1.Id); + cov1.MutationsCovered.ShouldContain(1); + cov1.MutationsCovered.ShouldContain(2); + cov1.MutationsCovered.ShouldNotContain(3); + cov1.Confidence.ShouldBe(CoverageConfidence.Normal); + + var cov2 = coverage.First(c => c.TestId == desc2.Id); + cov2.MutationsCovered.ShouldContain(3); + cov2.MutationsCovered.ShouldNotContain(1); + cov2.Confidence.ShouldBe(CoverageConfidence.Normal); + } + + [TestMethod] + public void CaptureCoverage_ShouldUseExactConfidence_WhenPerTestInIsolationEnabled() + { + // Arrange + var options = new Mock(); + options.Setup(x => x.Concurrency).Returns(1); + options.Setup(x => x.OptimizationMode).Returns( + OptimizationModes.CoverageBasedTest | OptimizationModes.CaptureCoveragePerTest); + + var testsByAssembly = new Dictionary>(); + var testDescriptions = new Dictionary(); + var testSet = new TestSet(); + + var testNode = new TestNode("test-1", "Test1", "test", "discovered"); + testsByAssembly["assembly.dll"] = new List { testNode }; + var desc = new MtpTestDescription(testNode); + testDescriptions["test-1"] = desc; + testSet.RegisterTest(desc.Description); + + CoverageConfidence? capturedConfidence = null; + + var runnerFactory = new Mock(); + runnerFactory.Setup(x => x.CreateRunner( + It.IsAny(), + It.IsAny>>(), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns>, Dictionary, TestSet, object, ILogger, IStrykerOptions>( + (id, tba, td, ts, dl, logger, opts) => + { + if (tba.Count == 0) + { + foreach (var kvp in testsByAssembly) + tba[kvp.Key] = kvp.Value; + foreach (var kvp in testDescriptions) + td[kvp.Key] = kvp.Value; + } + return new TestableRunner(id, tba, td, ts, dl, + () => { }, + coverageHandler: (assembly, test, testId, confidence) => + { + capturedConfidence = confidence; + return Task.FromResult( + CoverageRunResult.Create(testId, confidence, new[] { 1 }, Array.Empty(), Array.Empty())); + }); + }); + + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns(new[] { "assembly.dll" }); + + using var pool = new MicrosoftTestPlatformRunnerPool(options.Object, NullLogger.Instance, runnerFactory.Object); + + // Act + var coverage = pool.CaptureCoverage(project.Object).ToList(); + + // Assert + capturedConfidence.ShouldBe(CoverageConfidence.Exact, + "perTestInIsolation should use Exact confidence"); + coverage.Single().Confidence.ShouldBe(CoverageConfidence.Exact); + } + + [TestMethod] + public void CaptureCoverage_ShouldFallbackToAggregate_WhenOnlySkipUncoveredEnabled() + { + // Arrange: "all" mode = SkipUncoveredMutants only (no CoverageBasedTest) + var options = new Mock(); + options.Setup(x => x.Concurrency).Returns(1); + options.Setup(x => x.OptimizationMode).Returns(OptimizationModes.SkipUncoveredMutants); + using var pool = new MicrosoftTestPlatformRunnerPool(options.Object, NullLogger.Instance); + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns(Array.Empty()); + + // Act + var coverage = pool.CaptureCoverage(project.Object).ToList(); + + // Assert: aggregate mode returns empty when no tests discovered + coverage.ShouldBeEmpty(); + } + + [TestMethod] + public void CaptureCoverage_ShouldHandleEmptyTestSet_InPerTestMode() + { + // Arrange + var options = new Mock(); + options.Setup(x => x.Concurrency).Returns(1); + options.Setup(x => x.OptimizationMode).Returns(OptimizationModes.CoverageBasedTest); + + var runnerFactory = new Mock(); + runnerFactory.Setup(x => x.CreateRunner( + It.IsAny(), + It.IsAny>>(), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns>, Dictionary, TestSet, object, ILogger, IStrykerOptions>( + (id, tba, td, ts, dl, logger, opts) => + new TestableRunner(id, () => { })); + + using var pool = new MicrosoftTestPlatformRunnerPool(options.Object, NullLogger.Instance, runnerFactory.Object); + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns(Array.Empty()); + + // Act + var coverage = pool.CaptureCoverage(project.Object).ToList(); + + // Assert: no tests = no coverage results + coverage.ShouldBeEmpty(); + } + + [TestMethod] + public void CaptureCoverage_ShouldHandleMultipleAssemblies_InPerTestMode() + { + // Arrange + var options = new Mock(); + options.Setup(x => x.Concurrency).Returns(2); + options.Setup(x => x.OptimizationMode).Returns(OptimizationModes.CoverageBasedTest); + + var testsByAssembly = new Dictionary>(); + var testDescriptions = new Dictionary(); + var testSet = new TestSet(); + + // Assembly 1: 2 tests + var testA1 = new TestNode("a1-test-1", "A1Test1", "test", "discovered"); + var testA2 = new TestNode("a1-test-2", "A1Test2", "test", "discovered"); + testsByAssembly["assembly1.dll"] = new List { testA1, testA2 }; + + // Assembly 2: 1 test + var testB1 = new TestNode("a2-test-1", "A2Test1", "test", "discovered"); + testsByAssembly["assembly2.dll"] = new List { testB1 }; + + foreach (var (_, tests) in testsByAssembly) + { + foreach (var t in tests) + { + var desc = new MtpTestDescription(t); + testDescriptions[t.Uid] = desc; + testSet.RegisterTest(desc.Description); + } + } + + var capturedAssemblies = new System.Collections.Concurrent.ConcurrentBag(); + + var runnerFactory = new Mock(); + runnerFactory.Setup(x => x.CreateRunner( + It.IsAny(), + It.IsAny>>(), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns>, Dictionary, TestSet, object, ILogger, IStrykerOptions>( + (id, tba, td, ts, dl, logger, opts) => + { + // Populate pool's dictionaries via references + if (tba.Count == 0) + { + foreach (var kvp in testsByAssembly) + tba[kvp.Key] = kvp.Value; + foreach (var kvp in testDescriptions) + td[kvp.Key] = kvp.Value; + } + return new TestableRunner(id, tba, td, ts, dl, + () => { }, + coverageHandler: (assembly, test, testId, confidence) => + { + capturedAssemblies.Add(assembly); + return Task.FromResult( + CoverageRunResult.Create(testId, confidence, new[] { 1 }, Array.Empty(), Array.Empty())); + }); + }); + + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns(new[] { "assembly1.dll", "assembly2.dll" }); + + using var pool = new MicrosoftTestPlatformRunnerPool(options.Object, NullLogger.Instance, runnerFactory.Object); + + // Act + var coverage = pool.CaptureCoverage(project.Object).ToList(); + + // Assert + coverage.Count.ShouldBe(3, "Should capture coverage for all 3 tests across both assemblies"); + capturedAssemblies.Count.ShouldBe(3); + capturedAssemblies.Count(a => a == "assembly1.dll").ShouldBe(2); + capturedAssemblies.Count(a => a == "assembly2.dll").ShouldBe(1); + } } diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs index 18c29e60d5..e1fa99f36a 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs @@ -164,11 +164,61 @@ private IEnumerable CaptureCoverageInOneGo(IProjectAndTests } } - // Stub: real implementation added in Task 6 private IEnumerable CaptureCoverageTestByTest( IProjectAndTests project, CoverageConfidence confidence) { - throw new NotImplementedException("CaptureCoverageTestByTest will be implemented in Task 6"); + _logger.LogInformation("Starting per-test coverage capture for MTP runner"); + + foreach (var runner in _availableRunners) + { + runner.SetCoverageMode(true); + } + + try + { + var allTests = new List<(string Assembly, TestNode Test, string TestId)>(); + foreach (var (assembly, tests) in _testsByAssembly) + { + foreach (var test in tests) + { + if (_testDescriptions.TryGetValue(test.Uid, out var desc)) + { + allTests.Add((assembly, test, desc.Id)); + } + } + } + + _logger.LogInformation("Capturing per-test coverage for {TestCount} tests across {AssemblyCount} assemblies", + allTests.Count, _testsByAssembly.Count); + + var results = new ConcurrentBag(); + + Parallel.ForEach(allTests, + new ParallelOptions { MaxDegreeOfParallelism = _countOfRunners }, + testInfo => + { + var result = RunThisAsync(async runner => + await runner.RunSingleTestForCoverageAsync( + testInfo.Assembly, testInfo.Test, testInfo.TestId, confidence) + .ConfigureAwait(false)) + .GetAwaiter().GetResult(); + + results.Add(result); + }); + + _logger.LogInformation( + "Per-test coverage capture complete: {TestCount} tests captured", + results.Count); + + return results; + } + finally + { + foreach (var runner in _availableRunners) + { + runner.SetCoverageMode(false); + } + } } public async Task TestMultipleMutantsAsync( From 4610bfbecb8080f33190b5428d58a19eb330b4ba Mon Sep 17 00:00:00 2001 From: Piotr Nawrot Date: Wed, 1 Apr 2026 18:01:25 +0200 Subject: [PATCH 5/9] fix(MTP): downgrade to Dubious confidence when coverage capture is empty When RunSingleTestForCoverageAsync gets empty coverage data (e.g., server force-killed before FlushCoverageToFile ran), the result now correctly uses CoverageConfidence.Dubious instead of the requested confidence level. This prevents silently marking mutants as uncovered when coverage capture failed. Also fixes misleading test names (renamed RunSingleTestForCoverageAsync_* to ReadCoverageData_* where they only tested ReadCoverageData), and adds proper tests for the Dubious confidence paths through the pool. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...icrosoftTestPlatformRunnerCoverageTests.cs | 120 +++++++++++++++++- .../TestableRunner.cs | 10 +- .../SingleMicrosoftTestPlatformRunner.cs | 19 ++- 3 files changed, 144 insertions(+), 5 deletions(-) diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs index 4a9343173d..4b8454d7f2 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs @@ -1,6 +1,12 @@ +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Moq; using Shouldly; +using Stryker.Abstractions; +using Stryker.Abstractions.Options; +using Stryker.Abstractions.Testing; using Stryker.TestRunner.MicrosoftTestPlatform.Models; +using Stryker.TestRunner.Results; using Stryker.TestRunner.Tests; namespace Stryker.TestRunner.MicrosoftTestPlatform.UnitTest; @@ -516,7 +522,7 @@ public async Task StopAndRemoveServerAsync_ShouldRemoveServerFromDictionary() } [TestMethod] - public async Task RunSingleTestForCoverageAsync_ShouldReturnCoverageFromFile() + public void ReadCoverageData_ShouldReturnCoveredAndStaticMutants_FromFile() { var runnerId = 620; var coverageFilePath = Path.Combine(Path.GetTempPath(), $"stryker-coverage-{runnerId}.txt"); @@ -552,7 +558,7 @@ public async Task RunSingleTestForCoverageAsync_ShouldReturnCoverageFromFile() } [TestMethod] - public void RunSingleTestForCoverageAsync_ShouldReturnDubious_WhenNoCoverageFile() + public void ReadCoverageData_ShouldReturnEmpty_WhenNoCoverageFile() { var runnerId = 621; using var runner = new SingleMicrosoftTestPlatformRunner( @@ -568,4 +574,114 @@ public void RunSingleTestForCoverageAsync_ShouldReturnDubious_WhenNoCoverageFile result.CoveredMutants.ShouldBeEmpty(); result.StaticMutants.ShouldBeEmpty(); } + + [TestMethod] + public void CaptureCoverageTestByTest_ShouldReturnDubious_WhenHandlerThrows() + { + var options = new Mock(); + options.Setup(x => x.Concurrency).Returns(1); + options.Setup(x => x.OptimizationMode).Returns(OptimizationModes.CoverageBasedTest); + + var testNode = new TestNode("test-1", "ThrowingTest", "test", "discovered"); + var testsByAssembly = new Dictionary> + { + ["assembly.dll"] = [testNode] + }; + var testDescriptions = new Dictionary + { + ["test-1"] = new(testNode) + }; + + var runnerFactory = new Mock(); + runnerFactory.Setup(x => x.CreateRunner( + It.IsAny(), + It.IsAny>>(), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns>, Dictionary, TestSet, object, ILogger, IStrykerOptions>( + (id, tba, td, ts, dl, logger, opts) => + { + if (tba.Count == 0) + { + foreach (var kvp in testsByAssembly) + tba[kvp.Key] = kvp.Value; + foreach (var kvp in testDescriptions) + td[kvp.Key] = kvp.Value; + } + return new TestableRunner(id, tba, td, ts, dl, + () => { }, + coverageHandler: (_, _, _, _) => + throw new InvalidOperationException("Server startup failed")); + }); + + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns(new[] { "assembly.dll" }); + + using var pool = new MicrosoftTestPlatformRunnerPool(options.Object, NullLogger.Instance, runnerFactory.Object); + + var coverage = pool.CaptureCoverage(project.Object).ToList(); + + coverage.Count.ShouldBe(1); + coverage[0].Confidence.ShouldBe(CoverageConfidence.Dubious); + coverage[0].MutationsCovered.ShouldBeEmpty(); + } + + [TestMethod] + public void CaptureCoverageTestByTest_ShouldReturnDubious_WhenCoverageIsEmpty() + { + var options = new Mock(); + options.Setup(x => x.Concurrency).Returns(1); + options.Setup(x => x.OptimizationMode).Returns(OptimizationModes.CoverageBasedTest); + + var testNode = new TestNode("test-1", "NoCoverageTest", "test", "discovered"); + var testsByAssembly = new Dictionary> + { + ["assembly.dll"] = [testNode] + }; + var testDescriptions = new Dictionary + { + ["test-1"] = new(testNode) + }; + + var runnerFactory = new Mock(); + runnerFactory.Setup(x => x.CreateRunner( + It.IsAny(), + It.IsAny>>(), + It.IsAny>(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns>, Dictionary, TestSet, object, ILogger, IStrykerOptions>( + (id, tba, td, ts, dl, logger, opts) => + { + if (tba.Count == 0) + { + foreach (var kvp in testsByAssembly) + tba[kvp.Key] = kvp.Value; + foreach (var kvp in testDescriptions) + td[kvp.Key] = kvp.Value; + } + return new TestableRunner(id, tba, td, ts, dl, + () => { }, + coverageHandler: (_, _, testId, _) => + Task.FromResult( + CoverageRunResult.Create(testId, CoverageConfidence.Dubious, + Array.Empty(), Array.Empty(), Array.Empty()))); + }); + + var project = new Mock(); + project.Setup(x => x.GetTestAssemblies()).Returns(new[] { "assembly.dll" }); + + using var pool = new MicrosoftTestPlatformRunnerPool(options.Object, NullLogger.Instance, runnerFactory.Object); + + var coverage = pool.CaptureCoverage(project.Object).ToList(); + + coverage.Count.ShouldBe(1); + coverage[0].Confidence.ShouldBe(CoverageConfidence.Dubious); + coverage[0].MutationsCovered.ShouldBeEmpty(); + } } diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/TestableRunner.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/TestableRunner.cs index 032f566fcf..b98b9d857a 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/TestableRunner.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/TestableRunner.cs @@ -40,7 +40,15 @@ internal override async Task RunSingleTestForCoverageAsync( { if (_coverageHandler is not null) { - return await _coverageHandler(assembly, test, testId, confidence).ConfigureAwait(false); + try + { + return await _coverageHandler(assembly, test, testId, confidence).ConfigureAwait(false); + } + catch + { + return CoverageRunResult.Create(testId, CoverageConfidence.Dubious, + Array.Empty(), Array.Empty(), Array.Empty()); + } } return CoverageRunResult.Create(testId, confidence, Array.Empty(), Array.Empty(), Array.Empty()); diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs index 94462f8d39..62b3b75d53 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs @@ -144,12 +144,27 @@ internal virtual async Task RunSingleTestForCoverageAsync( var (coveredMutants, staticMutants) = ReadCoverageData(); + DeleteCoverageFile(); + + // Empty coverage likely means the process was force-killed before FlushCoverageToFile ran + if (coveredMutants.Count == 0 && staticMutants.Count == 0) + { + _logger.LogWarning( + "{RunnerId}: No coverage data captured for test {TestId} — coverage file was empty or missing. Marking as Dubious.", + RunnerId, testId); + + return CoverageRunResult.Create( + testId, + CoverageConfidence.Dubious, + coveredMutants, + staticMutants, + Array.Empty()); + } + _logger.LogDebug( "{RunnerId}: Test {TestId} covers {CoveredCount} mutants ({StaticCount} static)", RunnerId, testId, coveredMutants.Count, staticMutants.Count); - DeleteCoverageFile(); - return CoverageRunResult.Create( testId, confidence, From bdf3b049bfa881e7b61872587aac2d4431c615b3 Mon Sep 17 00:00:00 2001 From: Piotr Nawrot Date: Wed, 1 Apr 2026 19:11:57 +0200 Subject: [PATCH 6/9] refactor(test-runner): replace lock/AutoResetEvent with SemaphoreSlim for async-safe concurrency - Replace AutoResetEvent with SemaphoreSlim in RunnerPool.RunThisAsync to avoid blocking thread-pool threads during runner checkout - Replace object _serverLock with SemaphoreSlim(1,1) in SingleRunner to enable holding the lock across await in GetOrCreateServerAsync, eliminating the TOCTOU race in the check-create-start-store pattern - Fix CalculateAssemblyTimeout to snapshot _testDescriptions once instead of acquiring _discoveryLock per LINQ element Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../MicrosoftTestPlatformRunnerPool.cs | 44 ++++----- .../SingleMicrosoftTestPlatformRunner.cs | 93 ++++++++++++------- 2 files changed, 77 insertions(+), 60 deletions(-) diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs index e1fa99f36a..3401e0d475 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs @@ -17,7 +17,7 @@ namespace Stryker.TestRunner.MicrosoftTestPlatform; /// public sealed class MicrosoftTestPlatformRunnerPool : ITestRunner { - private readonly AutoResetEvent _runnerAvailableHandler = new(false); + private readonly SemaphoreSlim _runnerAvailable; private readonly ConcurrentBag _availableRunners = new(); private readonly ILogger _logger; private readonly int _countOfRunners; @@ -36,6 +36,7 @@ public MicrosoftTestPlatformRunnerPool(IStrykerOptions options, ILogger? logger _options = options; _countOfRunners = Math.Max(1, options.Concurrency); _runnerFactory = runnerFactory ?? new DefaultRunnerFactory(); + _runnerAvailable = new SemaphoreSlim(0, _countOfRunners); _logger.LogWarning("The Microsoft Test Platform testrunner is currently in preview. Results should be verified since this feature is still being tested."); Initialize(); @@ -63,7 +64,7 @@ private void Initialize() _logger, _options); _availableRunners.Add(runner); - _runnerAvailableHandler.Set(); + _runnerAvailable.Release(); }); } @@ -238,30 +239,23 @@ public async Task TestMultipleMutantsAsync( private async Task RunThisAsync(Func> task) { - SingleMicrosoftTestPlatformRunner? runner; + const int maxWaitTimeSeconds = 300; + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(maxWaitTimeSeconds)); - // Try to get a runner with a timeout to prevent indefinite blocking - var attempts = 0; - const int maxWaitTimeSeconds = 300; // 5 minutes max wait - const int waitIntervalMs = 1000; // Check every second - var maxAttempts = maxWaitTimeSeconds * 1000 / waitIntervalMs; - - while (!_availableRunners.TryTake(out runner)) + try { - if (!_runnerAvailableHandler.WaitOne(waitIntervalMs)) - { - attempts++; - if (attempts >= maxAttempts) - { - throw new TimeoutException($"Timed out waiting for an available test runner after {maxWaitTimeSeconds} seconds. Available runners: {_availableRunners.Count}, Total runners: {_countOfRunners}"); - } + await _runnerAvailable.WaitAsync(cts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw new TimeoutException($"Timed out waiting for an available test runner after {maxWaitTimeSeconds} seconds. Available runners: {_availableRunners.Count}, Total runners: {_countOfRunners}"); + } - if (attempts % 30 == 0) // Log every 30 seconds - { - _logger.LogWarning("Waiting for available test runner... ({Attempts}s elapsed, {Available}/{Total} runners available)", - attempts, _availableRunners.Count, _countOfRunners); - } - } + if (!_availableRunners.TryTake(out var runner)) + { + // Another thread grabbed the runner between the semaphore release and our TryTake; re-wait + _runnerAvailable.Release(); + return await RunThisAsync(task).ConfigureAwait(false); } try @@ -271,7 +265,7 @@ private async Task RunThisAsync(Func _assemblyServers = new(); - private readonly object _serverLock = new(); + private readonly SemaphoreSlim _serverLock = new(1, 1); private bool _disposed; private bool _coverageMode; @@ -93,7 +93,8 @@ public async Task ResetServerAsync() { _logger.LogDebug("{RunnerId}: Resetting test servers to reload assemblies", RunnerId); - lock (_serverLock) + await _serverLock.WaitAsync().ConfigureAwait(false); + try { foreach (var server in _assemblyServers.Values) { @@ -101,9 +102,12 @@ public async Task ResetServerAsync() } _assemblyServers.Clear(); } + finally + { + _serverLock.Release(); + } _logger.LogDebug("{RunnerId}: Test servers reset complete", RunnerId); - await Task.CompletedTask; } /// @@ -114,11 +118,16 @@ public async Task ResetServerAsync() internal async Task StopAndRemoveServerAsync(string assembly) { AssemblyTestServer? server; - lock (_serverLock) + await _serverLock.WaitAsync().ConfigureAwait(false); + try { _assemblyServers.TryGetValue(assembly, out server); _assemblyServers.Remove(assembly); } + finally + { + _serverLock.Release(); + } if (server is not null) { @@ -223,24 +232,27 @@ private void WriteMutantIdToFile(int mutantId) /// public void SetCoverageMode(bool enabled) { - lock (_serverLock) + _serverLock.Wait(); + try { if (_coverageMode == enabled) { - // Already in the desired state; no action needed return; } _coverageMode = enabled; _logger.LogDebug("{RunnerId}: Coverage mode {Status}", RunnerId, enabled ? "enabled" : "disabled"); - // Reset servers to apply the new environment variables foreach (var server in _assemblyServers.Values) { server.Dispose(); } _assemblyServers.Clear(); } + finally + { + _serverLock.Release(); + } // Clean up any existing coverage file, even when enabling, to ensure we start fresh DeleteCoverageFile(); @@ -313,30 +325,30 @@ private void DeleteCoverageFile() private async Task GetOrCreateServerAsync(string assembly) { - AssemblyTestServer? server; - lock (_serverLock) + await _serverLock.WaitAsync().ConfigureAwait(false); + try { - if (_assemblyServers.TryGetValue(assembly, out server) && server.IsInitialized) + if (_assemblyServers.TryGetValue(assembly, out var existing) && existing.IsInitialized) { - return server; + return existing; } - } - var environmentVariables = BuildEnvironmentVariables(); - server = new AssemblyTestServer(assembly, environmentVariables, _logger, RunnerId, _options); + var environmentVariables = BuildEnvironmentVariables(); + var server = new AssemblyTestServer(assembly, environmentVariables, _logger, RunnerId, _options); - var started = await server.StartAsync().ConfigureAwait(false); - if (!started) - { - throw new InvalidOperationException($"Failed to start test server for {assembly}"); - } + var started = await server.StartAsync().ConfigureAwait(false); + if (!started) + { + throw new InvalidOperationException($"Failed to start test server for {assembly}"); + } - lock (_serverLock) - { _assemblyServers[assembly] = server; + return server; + } + finally + { + _serverLock.Release(); } - - return server; } private async Task DiscoverTestsInternalAsync(string assembly) @@ -378,17 +390,17 @@ private async Task DiscoverTestsInternalAsync(string assembly) internal TimeSpan? CalculateAssemblyTimeout(List discoveredTests, ITimeoutValueCalculator timeoutCalc, string assembly) { + Dictionary descriptionsSnapshot; + lock (_discoveryLock) + { + descriptionsSnapshot = new Dictionary(_testDescriptions); + } + var estimatedTimeMs = (int)discoveredTests - .Where(t => _testDescriptions.TryGetValue(t.Uid, out _)) - .Sum(t => - { - lock (_discoveryLock) - { - return _testDescriptions.TryGetValue(t.Uid, out var desc) - ? desc.InitialRunTime.TotalMilliseconds - : 0; - } - }); + .Where(t => descriptionsSnapshot.ContainsKey(t.Uid)) + .Sum(t => descriptionsSnapshot.TryGetValue(t.Uid, out var desc) + ? desc.InitialRunTime.TotalMilliseconds + : 0); var timeoutMs = timeoutCalc.CalculateTimeoutValue(estimatedTimeMs); _logger.LogDebug("{RunnerId}: Using {TimeoutMs} ms as test run timeout for {Assembly}", @@ -404,10 +416,15 @@ internal async Task HandleAssemblyTimeoutAsync(string assembly, List d allTimedOutTests.AddRange(discoveredTests.Select(t => t.Uid)); AssemblyTestServer? server; - lock (_serverLock) + await _serverLock.WaitAsync().ConfigureAwait(false); + try { _assemblyServers.TryGetValue(assembly, out server); } + finally + { + _serverLock.Release(); + } if (server is not null) { @@ -645,7 +662,8 @@ public virtual void Dispose(bool disposing) if (disposing) { - lock (_serverLock) + _serverLock.Wait(); + try { foreach (var server in _assemblyServers.Values) { @@ -653,6 +671,11 @@ public virtual void Dispose(bool disposing) } _assemblyServers.Clear(); } + finally + { + _serverLock.Release(); + _serverLock.Dispose(); + } // Clean up temp files try From 35e56f5b940381147f25f6c43e491cb7f37f9866 Mon Sep 17 00:00:00 2001 From: Piotr Nawrot Date: Wed, 1 Apr 2026 20:48:02 +0200 Subject: [PATCH 7/9] fix(test-runner): replace recursive retry with bounded while loop in RunThisAsync Replace recursive self-call with a while loop sharing a single CancellationTokenSource so the 300-second timeout acts as a hard upper bound across all retries, preventing potential infinite loops if the semaphore/bag invariant is ever broken. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../MicrosoftTestPlatformRunnerPool.cs | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs index 3401e0d475..5ebcb7e0ed 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs @@ -242,30 +242,33 @@ private async Task RunThisAsync(Func Date: Wed, 1 Apr 2026 21:37:18 +0200 Subject: [PATCH 8/9] perf(test-runner): fix coverage leak, dispose race, and allocation hot spots - Cache _runnerId as readonly field (was allocating per access at 28 call sites) - Add StopAndRemoveServerAsync in RunSingleTestForCoverageAsync catch to prevent coverage data leaking between tests - Remove _serverLock.Dispose() after Release() to prevent ObjectDisposedException race - Single-pass foreach in RunTestsInternalAsync replacing 2x ToList + multiple re-iterations - Only register initial test results during initial run (mutantId == -1), not per-mutation - Move TestRunResult construction inside _discoveryLock to avoid ToList() copy of _testDescriptions.Values - Hold lock for CalculateAssemblyTimeout sum instead of copying entire dictionary - Fix ParseMutantIds: foreach + TrimEntries instead of LINQ chain with nullable boxing - Fix TestRunAccumulator: avoid ToList() just for Count - Guard debug log string allocation with IsEnabled check - Replace .Any() with .Count == 0 on IReadOnlyList in pool Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../MicrosoftTestPlatformRunnerPool.cs | 6 +- .../SingleMicrosoftTestPlatformRunner.cs | 207 ++++++++++-------- 2 files changed, 115 insertions(+), 98 deletions(-) diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs index 5ebcb7e0ed..451a7c3470 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs @@ -83,7 +83,8 @@ public async Task DiscoverTestsAsync(string assembly) public async Task InitialTestAsync(IProjectAndTests project) { var assemblies = project.GetTestAssemblies(); - if (!assemblies.Any()) + ArgumentNullException.ThrowIfNull(assemblies); + if (assemblies.Count == 0) { return new TestRunResult(false, "No test assemblies found"); } @@ -229,7 +230,8 @@ public async Task TestMultipleMutantsAsync( TestUpdateHandler? update) { var assemblies = project.GetTestAssemblies(); - if (!assemblies.Any()) + ArgumentNullException.ThrowIfNull(assemblies); + if (assemblies.Count == 0) { return new TestRunResult(false, "No test assemblies found"); } diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs index 6f3c0be868..6cb7ef227b 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs @@ -33,7 +33,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable private bool _disposed; private bool _coverageMode; - private string RunnerId => $"MtpRunner-{_id}"; + private readonly string _runnerId; public SingleMicrosoftTestPlatformRunner( int id, @@ -55,6 +55,7 @@ public SingleMicrosoftTestPlatformRunner( // Create unique file paths for this runner to communicate with the test process _mutantFilePath = Path.Combine(Path.GetTempPath(), $"stryker-mutant-{_id}.txt"); _coverageFilePath = Path.Combine(Path.GetTempPath(), $"stryker-coverage-{_id}.txt"); + _runnerId = $"MtpRunner-{_id}"; // Initialize with no active mutation WriteMutantIdToFile(-1); @@ -83,15 +84,18 @@ public Task TestMultipleMutantsAsync( // When testing a single mutant, activate it; otherwise use -1 (no mutation) var mutantId = mutants.Count == 1 ? mutants[0].Id : -1; - _logger.LogDebug("{RunnerId}: Testing mutant(s) [{Mutants}] with active mutation ID: {MutantId}", - RunnerId, string.Join(",", mutants.Select(m => m.Id)), mutantId); + if (_logger.IsEnabled(Microsoft.Extensions.Logging.LogLevel.Debug)) + { + _logger.LogDebug("{RunnerId}: Testing mutant(s) [{Mutants}] with active mutation ID: {MutantId}", + _runnerId, string.Join(",", mutants.Select(m => m.Id)), mutantId); + } return RunAllTestsAsync(assemblies, mutantId, mutants, update, timeoutCalc); } public async Task ResetServerAsync() { - _logger.LogDebug("{RunnerId}: Resetting test servers to reload assemblies", RunnerId); + _logger.LogDebug("{_runnerId}: Resetting test servers to reload assemblies", _runnerId); await _serverLock.WaitAsync().ConfigureAwait(false); try @@ -107,7 +111,7 @@ public async Task ResetServerAsync() _serverLock.Release(); } - _logger.LogDebug("{RunnerId}: Test servers reset complete", RunnerId); + _logger.LogDebug("{_runnerId}: Test servers reset complete", _runnerId); } /// @@ -159,8 +163,8 @@ internal virtual async Task RunSingleTestForCoverageAsync( if (coveredMutants.Count == 0 && staticMutants.Count == 0) { _logger.LogWarning( - "{RunnerId}: No coverage data captured for test {TestId} — coverage file was empty or missing. Marking as Dubious.", - RunnerId, testId); + "{_runnerId}: No coverage data captured for test {TestId} — coverage file was empty or missing. Marking as Dubious.", + _runnerId, testId); return CoverageRunResult.Create( testId, @@ -171,8 +175,8 @@ internal virtual async Task RunSingleTestForCoverageAsync( } _logger.LogDebug( - "{RunnerId}: Test {TestId} covers {CoveredCount} mutants ({StaticCount} static)", - RunnerId, testId, coveredMutants.Count, staticMutants.Count); + "{_runnerId}: Test {TestId} covers {CoveredCount} mutants ({StaticCount} static)", + _runnerId, testId, coveredMutants.Count, staticMutants.Count); return CoverageRunResult.Create( testId, @@ -183,9 +187,9 @@ internal virtual async Task RunSingleTestForCoverageAsync( } catch (Exception ex) { - _logger.LogWarning(ex, - "{RunnerId}: Failed to capture coverage for test {TestId}", RunnerId, testId); - + _logger.LogWarning(ex, "{_runnerId}: Failed to capture coverage for test {TestId}", _runnerId, testId); + try { await StopAndRemoveServerAsync(assembly).ConfigureAwait(false); } + catch { /* best-effort cleanup to prevent coverage data from leaking to the next test */ } return CoverageRunResult.Create( testId, CoverageConfidence.Dubious, @@ -200,13 +204,13 @@ private void WriteMutantIdToFile(int mutantId) try { File.WriteAllText(_mutantFilePath, mutantId.ToString()); - _logger.LogDebug("{RunnerId}: Wrote mutant ID {MutantId} to file {FilePath}", - RunnerId, mutantId, _mutantFilePath); + _logger.LogDebug("{_runnerId}: Wrote mutant ID {MutantId} to file {FilePath}", + _runnerId, mutantId, _mutantFilePath); } catch (Exception ex) { - _logger.LogWarning(ex, "{RunnerId}: Failed to write mutant ID to file {FilePath}", - RunnerId, _mutantFilePath); + _logger.LogWarning(ex, "{_runnerId}: Failed to write mutant ID to file {FilePath}", + _runnerId, _mutantFilePath); } } @@ -241,7 +245,7 @@ public void SetCoverageMode(bool enabled) } _coverageMode = enabled; - _logger.LogDebug("{RunnerId}: Coverage mode {Status}", RunnerId, enabled ? "enabled" : "disabled"); + _logger.LogDebug("{_runnerId}: Coverage mode {Status}", _runnerId, enabled ? "enabled" : "disabled"); foreach (var server in _assemblyServers.Values) { @@ -266,14 +270,14 @@ public void SetCoverageMode(bool enabled) { if (!File.Exists(_coverageFilePath)) { - _logger.LogDebug("{RunnerId}: Coverage file not found at {Path}", RunnerId, _coverageFilePath); + _logger.LogDebug("{_runnerId}: Coverage file not found at {Path}", _runnerId, _coverageFilePath); return (Array.Empty(), Array.Empty()); } try { var content = File.ReadAllText(_coverageFilePath).Trim(); - _logger.LogDebug("{RunnerId}: Read coverage data: {Content}", RunnerId, content); + _logger.LogDebug("{_runnerId}: Read coverage data: {Content}", _runnerId, content); if (string.IsNullOrEmpty(content)) { @@ -288,7 +292,7 @@ public void SetCoverageMode(bool enabled) } catch (Exception ex) { - _logger.LogWarning(ex, "{RunnerId}: Failed to read coverage file at {Path}", RunnerId, _coverageFilePath); + _logger.LogWarning(ex, "{_runnerId}: Failed to read coverage file at {Path}", _runnerId, _coverageFilePath); return (Array.Empty(), Array.Empty()); } } @@ -300,12 +304,16 @@ private static IReadOnlyList ParseMutantIds(string idString) return Array.Empty(); } - return idString - .Split(',', StringSplitOptions.RemoveEmptyEntries) - .Select(s => int.TryParse(s.Trim(), out var id) ? id : (int?)null) - .Where(id => id.HasValue) - .Select(id => id.Value) - .ToList(); + var parts = idString.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var result = new List(parts.Length); + foreach (var part in parts) + { + if (int.TryParse(part, out var id)) + { + result.Add(id); + } + } + return result; } private void DeleteCoverageFile() @@ -319,7 +327,7 @@ private void DeleteCoverageFile() } catch (Exception ex) { - _logger.LogWarning(ex, "{RunnerId}: Failed to delete coverage file at {Path}", RunnerId, _coverageFilePath); + _logger.LogWarning(ex, "{_runnerId}: Failed to delete coverage file at {Path}", _runnerId, _coverageFilePath); } } @@ -334,7 +342,7 @@ private async Task GetOrCreateServerAsync(string assembly) } var environmentVariables = BuildEnvironmentVariables(); - var server = new AssemblyTestServer(assembly, environmentVariables, _logger, RunnerId, _options); + var server = new AssemblyTestServer(assembly, environmentVariables, _logger, _runnerId, _options); var started = await server.StartAsync().ConfigureAwait(false); if (!started) @@ -370,12 +378,12 @@ private async Task DiscoverTestsInternalAsync(string assembly) } } - _logger.LogDebug("{RunnerId}: Discovered {TestCount} tests in {Assembly}", RunnerId, tests.Count, assembly); + _logger.LogDebug("{_runnerId}: Discovered {TestCount} tests in {Assembly}", _runnerId, tests.Count, assembly); return tests.Count > 0; } catch (Exception ex) { - _logger.LogDebug(ex, "{RunnerId}: Failed to discover tests in {Assembly}", RunnerId, assembly); + _logger.LogDebug(ex, "{_runnerId}: Failed to discover tests in {Assembly}", _runnerId, assembly); return false; } } @@ -390,28 +398,25 @@ private async Task DiscoverTestsInternalAsync(string assembly) internal TimeSpan? CalculateAssemblyTimeout(List discoveredTests, ITimeoutValueCalculator timeoutCalc, string assembly) { - Dictionary descriptionsSnapshot; + int estimatedTimeMs; lock (_discoveryLock) { - descriptionsSnapshot = new Dictionary(_testDescriptions); + estimatedTimeMs = (int)discoveredTests + .Sum(t => _testDescriptions.TryGetValue(t.Uid, out var desc) + ? desc.InitialRunTime.TotalMilliseconds + : 0); } - var estimatedTimeMs = (int)discoveredTests - .Where(t => descriptionsSnapshot.ContainsKey(t.Uid)) - .Sum(t => descriptionsSnapshot.TryGetValue(t.Uid, out var desc) - ? desc.InitialRunTime.TotalMilliseconds - : 0); - var timeoutMs = timeoutCalc.CalculateTimeoutValue(estimatedTimeMs); _logger.LogDebug("{RunnerId}: Using {TimeoutMs} ms as test run timeout for {Assembly}", - RunnerId, timeoutMs, Path.GetFileName(assembly)); - + _runnerId, timeoutMs, Path.GetFileName(assembly)); + return TimeSpan.FromMilliseconds(timeoutMs); } internal async Task HandleAssemblyTimeoutAsync(string assembly, List discoveredTests, List allTimedOutTests) { - _logger.LogDebug("{RunnerId}: Test run timed out for {Assembly}", RunnerId, Path.GetFileName(assembly)); + _logger.LogDebug("{_runnerId}: Test run timed out for {Assembly}", _runnerId, Path.GetFileName(assembly)); allTimedOutTests.AddRange(discoveredTests.Select(t => t.Uid)); @@ -428,7 +433,7 @@ internal async Task HandleAssemblyTimeoutAsync(string assembly, List d if (server is not null) { - _logger.LogDebug("{RunnerId}: Restarting test server for {Assembly} after timeout", RunnerId, Path.GetFileName(assembly)); + _logger.LogDebug("{_runnerId}: Restarting test server for {Assembly} after timeout", _runnerId, Path.GetFileName(assembly)); await server.RestartAsync().ConfigureAwait(false); } } @@ -453,9 +458,9 @@ public void Aggregate(TestRunResult result, List? discoveredTests) } else { - var executedIds = result.ExecutedTests.GetIdentifiers().ToList(); - _executedTests.AddRange(executedIds); - _totalExecutedTests += executedIds.Count; + var before = _executedTests.Count; + _executedTests.AddRange(result.ExecutedTests.GetIdentifiers()); + _totalExecutedTests += _executedTests.Count - before; } _failedTests.AddRange(result.FailingTests.GetIdentifiers()); @@ -486,7 +491,8 @@ public ITestIdentifiers BuildExecutedTests() => internal async Task<(TestRunResult? Result, bool TimedOut, List? DiscoveredTests)> ProcessSingleAssemblyAsync( string assembly, - ITimeoutValueCalculator? timeoutCalc) + ITimeoutValueCalculator? timeoutCalc, + bool registerInitialResults = false) { if (!File.Exists(assembly)) { @@ -494,15 +500,15 @@ public ITestIdentifiers BuildExecutedTests() => } var discoveredTests = GetDiscoveredTests(assembly); - + TimeSpan? timeout = null; if (timeoutCalc is not null && discoveredTests is not null) { timeout = CalculateAssemblyTimeout(discoveredTests, timeoutCalc, assembly); } - var (testResults, timedOut) = await RunTestsInternalAsync(assembly, null, timeout).ConfigureAwait(false); - + var (testResults, timedOut) = await RunTestsInternalAsync(assembly, null, timeout, registerInitialResults).ConfigureAwait(false); + return (testResults as TestRunResult, timedOut, discoveredTests); } @@ -521,7 +527,7 @@ private async Task RunAllTestsAsync( foreach (var assembly in assemblies) { - var (result, timedOut, discoveredTests) = await ProcessSingleAssemblyAsync(assembly, timeoutCalc).ConfigureAwait(false); + var (result, timedOut, discoveredTests) = await ProcessSingleAssemblyAsync(assembly, timeoutCalc, mutantId == -1).ConfigureAwait(false); if (discoveredTests is not null) { @@ -543,29 +549,26 @@ private async Task RunAllTestsAsync( var failedTestIds = accumulator.BuildFailedTests(); var timedOutTestIds = accumulator.BuildTimedOutTests(); - IEnumerable testDescriptionValues; - lock (_discoveryLock) - { - testDescriptionValues = _testDescriptions.Values.ToList(); - } - if (update is not null && mutants is not null) { update.Invoke(mutants, failedTestIds, executedTests, timedOutTestIds); } - return new TestRunResult( - testDescriptionValues, - executedTests, - failedTestIds, - timedOutTestIds, - accumulator.BuildErrorMessage(), - accumulator.Messages, - accumulator.TotalDuration); + lock (_discoveryLock) + { + return new TestRunResult( + _testDescriptions.Values, + executedTests, + failedTestIds, + timedOutTestIds, + accumulator.BuildErrorMessage(), + accumulator.Messages, + accumulator.TotalDuration); + } } catch (Exception ex) { - _logger.LogDebug(ex, "{RunnerId}: Failed to run tests for mutant ID {MutantId}", RunnerId, mutantId); + _logger.LogDebug(ex, "{_runnerId}: Failed to run tests for mutant ID {MutantId}", _runnerId, mutantId); return new TestRunResult(false, ex.Message); } } @@ -573,13 +576,13 @@ private async Task RunAllTestsAsync( internal async Task<(ITestRunResult Result, bool TimedOut)> RunTestsInternalAsync( string assembly, Func? testUidFilter, - TimeSpan? timeout = null) + TimeSpan? timeout = null, + bool registerInitialResults = false) { var startTime = DateTime.UtcNow; try { - // Get or create the server for this assembly (reuses existing server) var server = await GetOrCreateServerAsync(assembly).ConfigureAwait(false); List? tests = null; @@ -596,48 +599,61 @@ private async Task RunAllTestsAsync( var (testResults, timedOut) = await server.RunTestsAsync(testsToRun, timeout).ConfigureAwait(false); var duration = DateTime.UtcNow - startTime; - var finishedTests = testResults.Where(x => x.Node.ExecutionState is not "in-progress").ToList(); - var failedTests = finishedTests.Where(x => x.Node.ExecutionState is "failed").Select(x => x.Node.Uid).ToList(); - lock (_discoveryLock) + // Single pass: build finished/failed lists, error messages, and per-test messages together + var finishedTests = new List(testResults.Count); + var failedTestUids = new List(); + var errorParts = new List(); + var messageParts = new List(); + foreach (var tr in testResults) { - foreach (var testResult in finishedTests.Where(tr => _testDescriptions.ContainsKey(tr.Node.Uid))) + if (tr.Node.ExecutionState is "in-progress") { - var testDescription = _testDescriptions[testResult.Node.Uid]; - testDescription.RegisterInitialTestResult(new MtpTestResult(duration)); + continue; } - } - var errorMessagesStr = string.Join(Environment.NewLine, - finishedTests.Where(x => x.Node.ExecutionState is "failed") - .Select(x => $"{x.Node.DisplayName}{Environment.NewLine}{Environment.NewLine}Test failed")); + finishedTests.Add(tr); + messageParts.Add($"{tr.Node.DisplayName}{Environment.NewLine}{Environment.NewLine}State: {tr.Node.ExecutionState}"); - var messages = finishedTests.Select(x => - $"{x.Node.DisplayName}{Environment.NewLine}{Environment.NewLine}State: {x.Node.ExecutionState}"); + if (tr.Node.ExecutionState is "failed") + { + failedTestUids.Add(tr.Node.Uid); + errorParts.Add($"{tr.Node.DisplayName}{Environment.NewLine}{Environment.NewLine}Test failed"); + } + } + + var errorMessagesStr = string.Join(Environment.NewLine, errorParts); var totalDiscoveredTests = tests?.Count ?? 0; - var executedTestCount = finishedTests.Count; - var executedTests = totalDiscoveredTests > 0 && executedTestCount >= totalDiscoveredTests + var executedTests = totalDiscoveredTests > 0 && finishedTests.Count >= totalDiscoveredTests ? TestIdentifierList.EveryTest() : new TestIdentifierList(finishedTests.Select(x => x.Node.Uid)); - var failedTestIds = new TestIdentifierList(failedTests); - + var failedTestIds = new TestIdentifierList(failedTestUids); - IEnumerable testDescriptionValues; + TestRunResult result; lock (_discoveryLock) { - testDescriptionValues = _testDescriptions.Values.ToList(); - } + if (registerInitialResults) + { + foreach (var tr in finishedTests) + { + if (_testDescriptions.TryGetValue(tr.Node.Uid, out var desc)) + { + desc.RegisterInitialTestResult(new MtpTestResult(duration)); + } + } + } - var result = new TestRunResult( - testDescriptionValues, - executedTests, - failedTestIds, - TestIdentifierList.NoTest(), - errorMessagesStr, - messages, - duration); + result = new TestRunResult( + _testDescriptions.Values, + executedTests, + failedTestIds, + TestIdentifierList.NoTest(), + errorMessagesStr, + messageParts, + duration); + } return (result, timedOut); } @@ -674,7 +690,6 @@ public virtual void Dispose(bool disposing) finally { _serverLock.Release(); - _serverLock.Dispose(); } // Clean up temp files @@ -692,7 +707,7 @@ public virtual void Dispose(bool disposing) catch (Exception ex) { // Ignore cleanup errors - _logger.LogWarning(ex, "{RunnerId}: Failed to clean up temp files", RunnerId); + _logger.LogWarning(ex, "{_runnerId}: Failed to clean up temp files", _runnerId); } } _disposed = true; From 4fabe49d859befb76faef398f957281d52056b74 Mon Sep 17 00:00:00 2001 From: Piotr Nawrot Date: Sat, 11 Apr 2026 22:48:03 +0200 Subject: [PATCH 9/9] =?UTF-8?q?fix(MTP):=20address=20review=20findings=20?= =?UTF-8?q?=E2=80=94=20logging=20consistency,=20stale=20coverage,=20unused?= =?UTF-8?q?=20param?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Normalize all structured log templates from {_runnerId} to {RunnerId} - SetCoverageMode now always deletes coverage file to prevent stale data on re-entry - Add DeleteCoverageFile() to RunSingleTestForCoverageAsync error path - Remove unused project parameter from CaptureCoverageTestByTest Co-Authored-By: Claude Opus 4.6 (1M context) --- ...icrosoftTestPlatformRunnerCoverageTests.cs | 17 +++--- .../MicrosoftTestPlatformRunnerPool.cs | 4 +- .../SingleMicrosoftTestPlatformRunner.cs | 58 +++++++++---------- 3 files changed, 39 insertions(+), 40 deletions(-) diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs index 4b8454d7f2..95b08ca732 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform.UnitTest/SingleMicrosoftTestPlatformRunnerCoverageTests.cs @@ -63,10 +63,10 @@ public async Task SetCoverageMode_ShouldEnableCoverageMode() var result = await runner.DiscoverTestsAsync(testAssembly); result.ShouldBeTrue("Server should be recreated successfully after enabling coverage mode"); - // Trying to enable again should be a no-op + // Enabling again should still delete any stale coverage file (defensive cleanup) await File.WriteAllTextAsync(coverageFilePath, "test"); runner.SetCoverageMode(true); - File.Exists(coverageFilePath).ShouldBeTrue("Should not delete file when mode is already enabled"); + File.Exists(coverageFilePath).ShouldBeFalse("Should delete stale coverage file even when mode is already enabled"); } finally { @@ -113,10 +113,10 @@ public async Task SetCoverageMode_ShouldDisableCoverageMode() var result = await runner.DiscoverTestsAsync(testAssembly); result.ShouldBeTrue("Server should be recreated successfully after disabling coverage mode"); - // Trying to disable again should be a no-op (no servers disposed, no file deletion) + // Disabling again should still delete any stale coverage file (defensive cleanup) await File.WriteAllTextAsync(coverageFilePath, "test"); runner.SetCoverageMode(false); - File.Exists(coverageFilePath).ShouldBeTrue("Should not delete file when mode is already disabled"); + File.Exists(coverageFilePath).ShouldBeFalse("Should delete stale coverage file even when mode is already disabled"); } finally { @@ -150,13 +150,12 @@ public async Task SetCoverageMode_ShouldNoOp_WhenModeIsAlreadySet() runner.SetCoverageMode(true); File.Exists(coverageFilePath).ShouldBeFalse("Coverage file should be deleted on first enable"); - // Create a coverage file to verify no-op doesn't delete it + // Create a coverage file to verify defensive cleanup still happens await File.WriteAllTextAsync(coverageFilePath, "test-data"); - - // Try to enable again - should do nothing (no server disposal, no file deletion) + + // Try to enable again - servers should NOT be disposed, but stale coverage file should be deleted runner.SetCoverageMode(true); - File.Exists(coverageFilePath).ShouldBeTrue("Coverage file should NOT be deleted when mode already enabled"); - (await File.ReadAllTextAsync(coverageFilePath)).ShouldBe("test-data", "File content should be unchanged"); + File.Exists(coverageFilePath).ShouldBeFalse("Stale coverage file should be deleted even when mode already enabled"); // Verify servers are still functional (not disposed) var result = await runner.DiscoverTestsAsync(testAssembly); diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs index 451a7c3470..c70a0becf5 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/MicrosoftTestPlatformRunnerPool.cs @@ -104,7 +104,7 @@ public IEnumerable CaptureCoverage(IProjectAndTests project) var confidence = _options.OptimizationMode.HasFlag(OptimizationModes.CaptureCoveragePerTest) ? CoverageConfidence.Exact : CoverageConfidence.Normal; - return CaptureCoverageTestByTest(project, confidence); + return CaptureCoverageTestByTest(confidence); } return CaptureCoverageInOneGo(project); @@ -167,7 +167,7 @@ private IEnumerable CaptureCoverageInOneGo(IProjectAndTests } private IEnumerable CaptureCoverageTestByTest( - IProjectAndTests project, CoverageConfidence confidence) + CoverageConfidence confidence) { _logger.LogInformation("Starting per-test coverage capture for MTP runner"); diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs index 6cb7ef227b..e93f5633a4 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs @@ -95,7 +95,7 @@ public Task TestMultipleMutantsAsync( public async Task ResetServerAsync() { - _logger.LogDebug("{_runnerId}: Resetting test servers to reload assemblies", _runnerId); + _logger.LogDebug("{RunnerId}: Resetting test servers to reload assemblies", _runnerId); await _serverLock.WaitAsync().ConfigureAwait(false); try @@ -111,7 +111,7 @@ public async Task ResetServerAsync() _serverLock.Release(); } - _logger.LogDebug("{_runnerId}: Test servers reset complete", _runnerId); + _logger.LogDebug("{RunnerId}: Test servers reset complete", _runnerId); } /// @@ -163,7 +163,7 @@ internal virtual async Task RunSingleTestForCoverageAsync( if (coveredMutants.Count == 0 && staticMutants.Count == 0) { _logger.LogWarning( - "{_runnerId}: No coverage data captured for test {TestId} — coverage file was empty or missing. Marking as Dubious.", + "{RunnerId}: No coverage data captured for test {TestId} — coverage file was empty or missing. Marking as Dubious.", _runnerId, testId); return CoverageRunResult.Create( @@ -175,7 +175,7 @@ internal virtual async Task RunSingleTestForCoverageAsync( } _logger.LogDebug( - "{_runnerId}: Test {TestId} covers {CoveredCount} mutants ({StaticCount} static)", + "{RunnerId}: Test {TestId} covers {CoveredCount} mutants ({StaticCount} static)", _runnerId, testId, coveredMutants.Count, staticMutants.Count); return CoverageRunResult.Create( @@ -187,9 +187,10 @@ internal virtual async Task RunSingleTestForCoverageAsync( } catch (Exception ex) { - _logger.LogWarning(ex, "{_runnerId}: Failed to capture coverage for test {TestId}", _runnerId, testId); + _logger.LogWarning(ex, "{RunnerId}: Failed to capture coverage for test {TestId}", _runnerId, testId); try { await StopAndRemoveServerAsync(assembly).ConfigureAwait(false); } - catch { /* best-effort cleanup to prevent coverage data from leaking to the next test */ } + catch { /* best-effort cleanup to prevent server leak */ } + DeleteCoverageFile(); return CoverageRunResult.Create( testId, CoverageConfidence.Dubious, @@ -204,12 +205,12 @@ private void WriteMutantIdToFile(int mutantId) try { File.WriteAllText(_mutantFilePath, mutantId.ToString()); - _logger.LogDebug("{_runnerId}: Wrote mutant ID {MutantId} to file {FilePath}", + _logger.LogDebug("{RunnerId}: Wrote mutant ID {MutantId} to file {FilePath}", _runnerId, mutantId, _mutantFilePath); } catch (Exception ex) { - _logger.LogWarning(ex, "{_runnerId}: Failed to write mutant ID to file {FilePath}", + _logger.LogWarning(ex, "{RunnerId}: Failed to write mutant ID to file {FilePath}", _runnerId, _mutantFilePath); } } @@ -239,26 +240,25 @@ public void SetCoverageMode(bool enabled) _serverLock.Wait(); try { - if (_coverageMode == enabled) + if (_coverageMode != enabled) { - return; - } - - _coverageMode = enabled; - _logger.LogDebug("{_runnerId}: Coverage mode {Status}", _runnerId, enabled ? "enabled" : "disabled"); + _coverageMode = enabled; + _logger.LogDebug("{RunnerId}: Coverage mode {Status}", _runnerId, enabled ? "enabled" : "disabled"); - foreach (var server in _assemblyServers.Values) - { - server.Dispose(); + foreach (var server in _assemblyServers.Values) + { + server.Dispose(); + } + _assemblyServers.Clear(); } - _assemblyServers.Clear(); } finally { _serverLock.Release(); } - // Clean up any existing coverage file, even when enabling, to ensure we start fresh + // Always clean up any existing coverage file to prevent stale data, + // even when the mode hasn't changed (e.g. retry/re-run paths) DeleteCoverageFile(); } @@ -270,14 +270,14 @@ public void SetCoverageMode(bool enabled) { if (!File.Exists(_coverageFilePath)) { - _logger.LogDebug("{_runnerId}: Coverage file not found at {Path}", _runnerId, _coverageFilePath); + _logger.LogDebug("{RunnerId}: Coverage file not found at {Path}", _runnerId, _coverageFilePath); return (Array.Empty(), Array.Empty()); } try { var content = File.ReadAllText(_coverageFilePath).Trim(); - _logger.LogDebug("{_runnerId}: Read coverage data: {Content}", _runnerId, content); + _logger.LogDebug("{RunnerId}: Read coverage data: {Content}", _runnerId, content); if (string.IsNullOrEmpty(content)) { @@ -292,7 +292,7 @@ public void SetCoverageMode(bool enabled) } catch (Exception ex) { - _logger.LogWarning(ex, "{_runnerId}: Failed to read coverage file at {Path}", _runnerId, _coverageFilePath); + _logger.LogWarning(ex, "{RunnerId}: Failed to read coverage file at {Path}", _runnerId, _coverageFilePath); return (Array.Empty(), Array.Empty()); } } @@ -327,7 +327,7 @@ private void DeleteCoverageFile() } catch (Exception ex) { - _logger.LogWarning(ex, "{_runnerId}: Failed to delete coverage file at {Path}", _runnerId, _coverageFilePath); + _logger.LogWarning(ex, "{RunnerId}: Failed to delete coverage file at {Path}", _runnerId, _coverageFilePath); } } @@ -378,12 +378,12 @@ private async Task DiscoverTestsInternalAsync(string assembly) } } - _logger.LogDebug("{_runnerId}: Discovered {TestCount} tests in {Assembly}", _runnerId, tests.Count, assembly); + _logger.LogDebug("{RunnerId}: Discovered {TestCount} tests in {Assembly}", _runnerId, tests.Count, assembly); return tests.Count > 0; } catch (Exception ex) { - _logger.LogDebug(ex, "{_runnerId}: Failed to discover tests in {Assembly}", _runnerId, assembly); + _logger.LogDebug(ex, "{RunnerId}: Failed to discover tests in {Assembly}", _runnerId, assembly); return false; } } @@ -416,7 +416,7 @@ private async Task DiscoverTestsInternalAsync(string assembly) internal async Task HandleAssemblyTimeoutAsync(string assembly, List discoveredTests, List allTimedOutTests) { - _logger.LogDebug("{_runnerId}: Test run timed out for {Assembly}", _runnerId, Path.GetFileName(assembly)); + _logger.LogDebug("{RunnerId}: Test run timed out for {Assembly}", _runnerId, Path.GetFileName(assembly)); allTimedOutTests.AddRange(discoveredTests.Select(t => t.Uid)); @@ -433,7 +433,7 @@ internal async Task HandleAssemblyTimeoutAsync(string assembly, List d if (server is not null) { - _logger.LogDebug("{_runnerId}: Restarting test server for {Assembly} after timeout", _runnerId, Path.GetFileName(assembly)); + _logger.LogDebug("{RunnerId}: Restarting test server for {Assembly} after timeout", _runnerId, Path.GetFileName(assembly)); await server.RestartAsync().ConfigureAwait(false); } } @@ -568,7 +568,7 @@ private async Task RunAllTestsAsync( } catch (Exception ex) { - _logger.LogDebug(ex, "{_runnerId}: Failed to run tests for mutant ID {MutantId}", _runnerId, mutantId); + _logger.LogDebug(ex, "{RunnerId}: Failed to run tests for mutant ID {MutantId}", _runnerId, mutantId); return new TestRunResult(false, ex.Message); } } @@ -707,7 +707,7 @@ public virtual void Dispose(bool disposing) catch (Exception ex) { // Ignore cleanup errors - _logger.LogWarning(ex, "{_runnerId}: Failed to clean up temp files", _runnerId); + _logger.LogWarning(ex, "{RunnerId}: Failed to clean up temp files", _runnerId); } } _disposed = true;