diff --git a/Directory.Packages.props b/Directory.Packages.props index 2b873f95..c7ff8cae 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -7,11 +7,11 @@ - + - + @@ -45,8 +45,8 @@ - - + + diff --git a/source/CakeBuild/CreativeCoders.CakeBuild/Tasks/Templates/Settings/ITestTaskSettings.cs b/source/CakeBuild/CreativeCoders.CakeBuild/Tasks/Templates/Settings/ITestTaskSettings.cs index 2c05c885..d713cc2f 100644 --- a/source/CakeBuild/CreativeCoders.CakeBuild/Tasks/Templates/Settings/ITestTaskSettings.cs +++ b/source/CakeBuild/CreativeCoders.CakeBuild/Tasks/Templates/Settings/ITestTaskSettings.cs @@ -25,5 +25,26 @@ IEnumerable TestProjects bool GenerateCoverageReport => true; + /// + /// Gets a value that indicates whether dotnet test is run in Microsoft.Testing.Platform (MTP) mode + /// instead of VSTest mode. + /// + /// + /// to pass Microsoft.Testing.Platform arguments to dotnet test; otherwise, + /// to pass VSTest arguments. The default is . + /// + /// + /// This setting only selects the arguments passed to dotnet test; it does not switch the test mode of the + /// .NET SDK. The repository must enable the Microsoft.Testing.Platform mode in its global.json + /// ("test": { "runner": "Microsoft.Testing.Platform" }). The test projects must reference + /// Microsoft.Testing.Extensions.TrxReport and, if is + /// , Microsoft.Testing.Extensions.CodeCoverage. In this mode, the TRX file + /// (<Project>.trx) is written to instead of + /// , and the coverage report is written in Cobertura format to + /// <Project>.cobertura.xml in . Because these files are + /// named after the test project file, the file names of all test projects must be unique. + /// + bool UseMicrosoftTestingPlatform => false; + DirectoryPath TestSourceDir => Context.RootDir.Combine("tests"); } diff --git a/source/CakeBuild/CreativeCoders.CakeBuild/Tasks/Templates/TestTask.cs b/source/CakeBuild/CreativeCoders.CakeBuild/Tasks/Templates/TestTask.cs index 43407291..cb18ea74 100755 --- a/source/CakeBuild/CreativeCoders.CakeBuild/Tasks/Templates/TestTask.cs +++ b/source/CakeBuild/CreativeCoders.CakeBuild/Tasks/Templates/TestTask.cs @@ -1,6 +1,7 @@ using Cake.Common.Diagnostics; using Cake.Common.Tools.DotNet; using Cake.Common.Tools.DotNet.Test; +using Cake.Core; using Cake.Core.IO; using CreativeCoders.CakeBuild.Tasks.Templates.Settings; using JetBrains.Annotations; @@ -16,6 +17,10 @@ protected override Task RunAsyncCore(T context, ITestTaskSettings taskSettings) context.Information($"Found {testProjects.Length} test project(s)"); + context.Information(taskSettings.UseMicrosoftTestingPlatform + ? "Test platform: Microsoft Testing Platform" + : "Test platform: VSTest"); + foreach (var testProject in testProjects) { context.Information($"Test project found: {testProject.GetFilename()}"); @@ -27,25 +32,71 @@ protected override Task RunAsyncCore(T context, ITestTaskSettings taskSettings) return Task.CompletedTask; } + /// + /// Customizes the dotnet test settings for a single test project before the test run starts. + /// + /// The build context. + /// The dotnet test settings to customize. + /// + /// The hook is called after the base settings (configuration, build, results directory and, in VSTest mode, + /// loggers and collectors) have been applied. If is + /// , the Microsoft.Testing.Platform arguments are appended after the hook has run; an + /// set by the hook is kept and runs before + /// these arguments. In this mode, do not set or + /// , because dotnet test does not support them. + /// protected virtual void ApplyDotNetTestSettings(T context, DotNetTestSettings dotNetBuildSettings) { } private DotNetTestSettings CreateDotNetBuildSettings(T context, FilePath testProject, ITestTaskSettings testSettings) { - var testResultFile = - context.TestResultsDir.CombineWithFilePath($"{testProject.GetFilenameWithoutExtension()}.trx"); - var dotNetTestSettings = new DotNetTestSettings { Configuration = context.BuildConfiguration, NoBuild = context.HasExecutedTask(typeof(BuildTask)), - Loggers = [$"trx;LogFileName={testResultFile}"], - Collectors = testSettings.GenerateCoverageReport ? ["XPlat Code Coverage"] : [], ResultsDirectory = context.CodeCoverageDir }; + if (testSettings.UseMicrosoftTestingPlatform) + { + dotNetTestSettings.PathType = DotNetTestPathType.Project; + } + else + { + var testResultFile = + context.TestResultsDir.CombineWithFilePath($"{testProject.GetFilenameWithoutExtension()}.trx"); + + dotNetTestSettings.Loggers = [$"trx;LogFileName={testResultFile}"]; + dotNetTestSettings.Collectors = testSettings.GenerateCoverageReport ? ["XPlat Code Coverage"] : []; + } + ApplyDotNetTestSettings(context, dotNetTestSettings); + if (testSettings.UseMicrosoftTestingPlatform) + { + var customArgumentCustomization = dotNetTestSettings.ArgumentCustomization; + + dotNetTestSettings.ArgumentCustomization = args => + { + args = customArgumentCustomization?.Invoke(args) ?? args; + + args.Append("--report-trx"); + args.Append("--report-trx-filename"); + args.AppendQuoted($"{testProject.GetFilenameWithoutExtension()}.trx"); + + if (testSettings.GenerateCoverageReport) + { + args.Append("--coverage"); + args.Append("--coverage-output-format"); + args.Append("cobertura"); + args.Append("--coverage-output"); + args.AppendQuoted($"{testProject.GetFilenameWithoutExtension()}.cobertura.xml"); + } + + return args; + }; + } + if (dotNetTestSettings.NoBuild) { context.Information("Skip build"); diff --git a/tests/CreativeCoders.CakeBuild.Tests/Tasks/Templates/TestTaskTests.cs b/tests/CreativeCoders.CakeBuild.Tests/Tasks/Templates/TestTaskTests.cs index 1f42be55..a8fb02fe 100644 --- a/tests/CreativeCoders.CakeBuild.Tests/Tasks/Templates/TestTaskTests.cs +++ b/tests/CreativeCoders.CakeBuild.Tests/Tasks/Templates/TestTaskTests.cs @@ -1,4 +1,6 @@ using AwesomeAssertions; +using Cake.Common.Tools.DotNet.Test; +using Cake.Core; using Cake.Core.IO; using CreativeCoders.CakeBuild.Tasks.Templates; using CreativeCoders.CakeBuild.Tasks.Templates.Settings; @@ -9,11 +11,57 @@ namespace CreativeCoders.CakeBuild.Tests.Tasks.Templates; public class TestTaskTests { - private class TestTestBuildContext(Cake.Core.ICakeContext context) + private class TestTestBuildContext(ICakeContext context, bool generateCoverageReport = true) : CakeBuildContext(context), ITestTaskSettings { public IEnumerable TestProjects => [new FilePath("/repo/tests/Project.Tests.csproj")]; + + public bool GenerateCoverageReport { get; } = generateCoverageReport; + } + + private class MtpTestBuildContext( + ICakeContext context, + bool generateCoverageReport = true, + IEnumerable? testProjects = null) + : CakeBuildContext(context), ITestTaskSettings + { + public IEnumerable TestProjects { get; } = + testProjects ?? [new FilePath("/repo/tests/Project.Tests.csproj")]; + + public bool GenerateCoverageReport { get; } = generateCoverageReport; + + public bool UseMicrosoftTestingPlatform => true; + } + + private class FrameworkTestTask : TestTask + { + protected override void ApplyDotNetTestSettings(MtpTestBuildContext context, + DotNetTestSettings dotNetBuildSettings) + { + dotNetBuildSettings.Framework = "net10.0"; + } + } + + private class ArgumentCustomizationTestTask : TestTask + { + protected override void ApplyDotNetTestSettings(MtpTestBuildContext context, + DotNetTestSettings dotNetBuildSettings) + { + dotNetBuildSettings.ArgumentCustomization = args => args.Append("--custom-argument"); + } + } + + private static ICakeContext CreateCakeContext(List capturedCalls) + { + var cakeContext = CakeTestHelper.CreateCakeContext("dotnet-test"); + CakeTestHelper.SetupFileSystem(cakeContext, "/repo", "/repo/test.sln"); + + A.CallTo(() => cakeContext.ProcessRunner.Start("dotnet-test", A._)) + .Invokes((FilePath _, ProcessSettings settings) => + capturedCalls.Add(settings.Arguments.Select(arg => arg.Render()).ToArray())); + + return cakeContext; } [Fact] @@ -41,4 +89,223 @@ await act .Contains("\"/repo/tests/Project.Tests.csproj\"")))) .MustHaveHappenedOnceExactly(); } + + [Fact] + public void UseMicrosoftTestingPlatform_NotOverridden_IsFalse() + { + // Arrange + var cakeContext = CakeTestHelper.CreateCakeContext("dotnet-test"); + CakeTestHelper.SetupFileSystem(cakeContext, "/repo", "/repo/test.sln"); + + ITestTaskSettings settings = new TestTestBuildContext(cakeContext); + + // Act + var useMicrosoftTestingPlatform = settings.UseMicrosoftTestingPlatform; + + // Assert + useMicrosoftTestingPlatform + .Should() + .BeFalse(); + } + + [Fact] + public async Task RunAsync_DefaultSettings_PassesVsTestArguments() + { + // Arrange + var calls = new List(); + var context = new TestTestBuildContext(CreateCakeContext(calls)); + var task = new TestTask(); + + var expectedLogger = $"\"trx;LogFileName={context.TestResultsDir.FullPath}/Project.Tests.trx\""; + + // Act + await task.RunAsync(context); + + // Assert + calls + .Should() + .ContainSingle() + .Which + .Should() + .HaveElementAt(0, "test") + .And.HaveElementAt(1, "\"/repo/tests/Project.Tests.csproj\"") + .And.ContainInConsecutiveOrder("--logger", expectedLogger) + .And.ContainInConsecutiveOrder("--collect", "\"XPlat Code Coverage\"") + .And.NotContain("--project") + .And.NotContain("--report-trx") + .And.NotContain("--coverage"); + } + + [Fact] + public async Task RunAsync_VsTestWithoutCoverage_OmitsCollector() + { + // Arrange + var calls = new List(); + var context = new TestTestBuildContext(CreateCakeContext(calls), false); + var task = new TestTask(); + + // Act + await task.RunAsync(context); + + // Assert + calls + .Should() + .ContainSingle() + .Which + .Should() + .Contain("--logger") + .And.NotContain("--collect") + .And.NotContain("\"XPlat Code Coverage\""); + } + + [Fact] + public async Task RunAsync_UseMicrosoftTestingPlatformWithCoverage_PassesMtpArguments() + { + // Arrange + var calls = new List(); + var context = new MtpTestBuildContext(CreateCakeContext(calls)); + var task = new TestTask(); + + // Act + await task.RunAsync(context); + + // Assert + calls + .Should() + .ContainSingle() + .Which + .Should() + .HaveElementAt(0, "test") + .And.ContainInConsecutiveOrder("--project", "\"/repo/tests/Project.Tests.csproj\"") + .And.ContainInConsecutiveOrder("--results-directory", $"\"{context.CodeCoverageDir.FullPath}\"") + .And.ContainInConsecutiveOrder("--report-trx", "--report-trx-filename", "\"Project.Tests.trx\"") + .And.ContainInConsecutiveOrder("--coverage", "--coverage-output-format", "cobertura") + .And.ContainInConsecutiveOrder("--coverage-output", "\"Project.Tests.cobertura.xml\"") + .And.NotContain("--logger") + .And.NotContain("--collect"); + } + + [Fact] + public async Task RunAsync_UseMicrosoftTestingPlatformWithoutCoverage_OmitsCoverageArguments() + { + // Arrange + var calls = new List(); + var context = new MtpTestBuildContext(CreateCakeContext(calls), false); + var task = new TestTask(); + + // Act + await task.RunAsync(context); + + // Assert + calls + .Should() + .ContainSingle() + .Which + .Should() + .ContainInConsecutiveOrder("--project", "\"/repo/tests/Project.Tests.csproj\"") + .And.ContainInConsecutiveOrder("--report-trx", "--report-trx-filename", "\"Project.Tests.trx\"") + .And.NotContain("--coverage") + .And.NotContain("--coverage-output-format") + .And.NotContain("--coverage-output") + .And.NotContain("--logger") + .And.NotContain("--collect"); + } + + [Fact] + public async Task RunAsync_UseMicrosoftTestingPlatformWithMultipleProjects_PassesProjectSpecificArguments() + { + // Arrange + var calls = new List(); + var context = new MtpTestBuildContext(CreateCakeContext(calls), testProjects: + [ + new FilePath("/repo/tests/B.Tests.csproj"), + new FilePath("/repo/tests/A.Tests.csproj") + ]); + var task = new TestTask(); + + // Act + await task.RunAsync(context); + + // Assert + calls + .Should() + .HaveCount(2); + + calls[0] + .Should() + .ContainInConsecutiveOrder("--project", "\"/repo/tests/A.Tests.csproj\"") + .And.ContainInConsecutiveOrder("--report-trx-filename", "\"A.Tests.trx\""); + + calls[1] + .Should() + .ContainInConsecutiveOrder("--project", "\"/repo/tests/B.Tests.csproj\"") + .And.ContainInConsecutiveOrder("--report-trx-filename", "\"B.Tests.trx\""); + } + + [Fact] + public async Task RunAsync_UseMicrosoftTestingPlatformAfterBuildTask_PassesNoBuild() + { + // Arrange + var calls = new List(); + var context = new MtpTestBuildContext(CreateCakeContext(calls)); + context.AddExecutedTask(new BuildTask()); + var task = new TestTask(); + + // Act + await task.RunAsync(context); + + // Assert + calls + .Should() + .ContainSingle() + .Which + .Should() + .Contain("--no-build") + .And.ContainInConsecutiveOrder("--project", "\"/repo/tests/Project.Tests.csproj\"") + .And.Contain("--report-trx"); + } + + [Fact] + public async Task RunAsync_UseMicrosoftTestingPlatformWithApplyDotNetTestSettingsOverride_KeepsMtpArguments() + { + // Arrange + var calls = new List(); + var context = new MtpTestBuildContext(CreateCakeContext(calls)); + var task = new FrameworkTestTask(); + + // Act + await task.RunAsync(context); + + // Assert + calls + .Should() + .ContainSingle() + .Which + .Should() + .ContainInConsecutiveOrder("--framework", "net10.0") + .And.ContainInConsecutiveOrder("--report-trx", "--report-trx-filename", "\"Project.Tests.trx\"") + .And.ContainInConsecutiveOrder("--coverage", "--coverage-output-format", "cobertura"); + } + + [Fact] + public async Task RunAsync_UseMicrosoftTestingPlatformWithArgumentCustomizationOverride_KeepsBothArguments() + { + // Arrange + var calls = new List(); + var context = new MtpTestBuildContext(CreateCakeContext(calls)); + var task = new ArgumentCustomizationTestTask(); + + // Act + await task.RunAsync(context); + + // Assert + calls + .Should() + .ContainSingle() + .Which + .Should() + .Contain("--custom-argument") + .And.ContainInConsecutiveOrder("--report-trx", "--report-trx-filename", "\"Project.Tests.trx\"") + .And.ContainInConsecutiveOrder("--coverage", "--coverage-output-format", "cobertura"); + } }