Run test project target frameworks in parallel

src/Microsoft.DotNet.Arcade.Sdk.Tests/Microsoft.DotNet.Arcade.Sdk.Tests.csproj

@@ -23,6 +23,9 @@
     <Content Include="..\Microsoft.DotNet.Arcade.Sdk\tools\Tests.targets"
              Link="testassets\ArcadeSdkTools\Tests.targets"
              CopyToOutputDirectory="PreserveNewest" />
+    <Content Include="..\Microsoft.DotNet.Arcade.Sdk\tools\Tests.targets"
+             Link="testassets\ParallelTestTfms\Tests.targets"
+             CopyToOutputDirectory="PreserveNewest" />
     <Content Include="..\Microsoft.DotNet.Arcade.Sdk\tools\VSTest.targets"
              Link="testassets\ArcadeSdkTools\VSTest.targets"
              CopyToOutputDirectory="PreserveNewest" />

src/Microsoft.DotNet.Arcade.Sdk.Tests/ParallelTestTfmsTests.cs

@@ -0,0 +1,46 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics;
+using System.IO;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.DotNet.Arcade.Sdk.Tests
+{
+    [Collection(TestProjectCollection.Name)]
+    public class ParallelTestTfmsTests
+    {
+        private readonly ITestOutputHelper _output;
+        private readonly TestProjectFixture _fixture;
+
+        public ParallelTestTfmsTests(ITestOutputHelper output, TestProjectFixture fixture)
+        {
+            _output = output;
+            _fixture = fixture;
+        }
+
+        [Fact]
+        public void TargetFrameworksRunInParallel()
+        {
+            var app = _fixture.CreateTestApp("ParallelTestTfms");
+            var startInfo = new ProcessStartInfo
+            {
+                FileName = Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet",
+                WorkingDirectory = app.WorkingDirectory,
+            };
+            startInfo.ArgumentList.Add("msbuild");
+            startInfo.ArgumentList.Add("ParallelTestTfms.proj");
+            startInfo.ArgumentList.Add("/t:Test");
+            startInfo.ArgumentList.Add("/m:4");
+            startInfo.ArgumentList.Add("/nr:false");
+
+            var exitCode = app.Run(_output, startInfo);
+
+            Assert.Equal(0, exitCode);
+            Assert.True(File.Exists(Path.Combine(app.WorkingDirectory, "artifacts", "tmp", "test-tfm-a.started")));
+            Assert.True(File.Exists(Path.Combine(app.WorkingDirectory, "artifacts", "tmp", "test-tfm-b.started")));
+        }
+    }
+}

src/Microsoft.DotNet.Arcade.Sdk.Tests/TestArchitecturesTests.cs

@@ -145,8 +145,8 @@ public void TestsValidateRequestedArchitectureAgainstFixedHost()
                 .Single();
             Assert.DoesNotContain("TestArchitectures", initializationDependencies.Attribute("Condition")?.Value);
 
-            XElement testTarget = GetTarget(targets, "Test");
-            Assert.Contains("_ValidateTestArchitectures", testTarget.Attribute("DependsOnTargets")?.Value);
+            XElement innerRunTestsTarget = GetTarget(targets, "_InnerRunTests");
+            Assert.Contains("_ValidateTestArchitectures", innerRunTestsTarget.Attribute("DependsOnTargets")?.Value);
         }
 
         [Fact]

src/Microsoft.DotNet.Arcade.Sdk.Tests/Utilities/TestProjectFixture.cs

@@ -20,6 +20,7 @@ public class TestProjectFixture : IDisposable
     {
         private readonly ConcurrentQueue<IDisposable> _disposables = new ConcurrentQueue<IDisposable>();
         private readonly string _logOutputDir;
+        private readonly string _tempRoot;
         private readonly string _testAssets;
         private readonly string _boilerPlateDir;
 
@@ -31,7 +32,12 @@ public class TestProjectFixture : IDisposable
         public TestProjectFixture()
         {
             ClearPackages();
-            _logOutputDir = GetType().Assembly.GetCustomAttributes<AssemblyMetadataAttribute>().Single(m => m.Key == "LogOutputDir").Value;
+            var helixWorkItemRoot = Environment.GetEnvironmentVariable("HELIX_WORKITEM_ROOT");
+            helixWorkItemRoot = string.IsNullOrWhiteSpace(helixWorkItemRoot) ? null : helixWorkItemRoot;
+            _logOutputDir = helixWorkItemRoot == null
+                ? GetType().Assembly.GetCustomAttributes<AssemblyMetadataAttribute>().Single(m => m.Key == "LogOutputDir").Value
+                : Path.Combine(helixWorkItemRoot, "test-logs");
+            _tempRoot = helixWorkItemRoot ?? Path.GetTempPath();
             _testAssets = Path.Combine(AppContext.BaseDirectory, "testassets");
             _boilerPlateDir = Path.Combine(_testAssets, "boilerplate");
         }
@@ -40,7 +46,7 @@ public TestApp CreateTestApp(string name)
         {
             var testAppFiles = Path.Combine(_testAssets, name);
             var instanceName = Path.GetRandomFileName();
-            var tempDir = Path.Combine(Path.GetTempPath(), "arcade", instanceName);
+            var tempDir = Path.Combine(_tempRoot, "arcade", instanceName);
             var app = new TestApp(tempDir, _logOutputDir, new[] { testAppFiles, _boilerPlateDir });
             _disposables.Enqueue(app);
             return app;

src/Microsoft.DotNet.Arcade.Sdk.Tests/testassets/ParallelTestTfms/ParallelTestTfms.proj

@@ -0,0 +1,16 @@
+<Project>
+  <PropertyGroup>
+    <TargetFrameworks>test-tfm-a;test-tfm-b</TargetFrameworks>
+    <IsTestProject>true</IsTestProject>
+    <IsUnitTestProject>true</IsUnitTestProject>
+    <TestRunnerName>Parallel</TestRunnerName>
+    <RepositoryEngineeringDir>$(MSBuildThisFileDirectory)eng\</RepositoryEngineeringDir>
+    <TestRuntime>Core</TestRuntime>
+    <TestArchitectures>x64</TestArchitectures>
+    <TargetPath>$(MSBuildThisFileDirectory)$(TargetFramework)\ParallelTestTfms.dll</TargetPath>
+    <ArtifactsTmpDir>$(MSBuildThisFileDirectory)artifacts\tmp\</ArtifactsTmpDir>
+    <ArtifactsLogDir>$(MSBuildThisFileDirectory)artifacts\log\</ArtifactsLogDir>
+    <ArtifactsTestResultsDir>$(MSBuildThisFileDirectory)artifacts\TestResults\</ArtifactsTestResultsDir>
+  </PropertyGroup>
+  <Import Project="Tests.targets" />
+</Project>

src/Microsoft.DotNet.Arcade.Sdk.Tests/testassets/ParallelTestTfms/eng/Parallel/Parallel.targets

@@ -0,0 +1,53 @@
+<Project>
+  <UsingTask TaskName="WaitForFiles"
+             TaskFactory="RoslynCodeTaskFactory"
+             AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.Core.dll">
+    <ParameterGroup>
+      <Files ParameterType="Microsoft.Build.Framework.ITaskItem[]" Required="true" />
+      <TimeoutSeconds ParameterType="System.Int32" Required="true" />
+    </ParameterGroup>
+    <Task>
+      <Using Namespace="System" />
+      <Using Namespace="System.IO" />
+      <Using Namespace="System.Linq" />
+      <Using Namespace="System.Threading" />
+      <Code Type="Fragment" Language="cs">
+        <![CDATA[
+        var deadline = DateTime.UtcNow.AddSeconds(TimeoutSeconds);
+        while (Files.Any(file => !File.Exists(file.ItemSpec.Replace('\\', Path.DirectorySeparatorChar))))
+        {
+            if (DateTime.UtcNow >= deadline)
+            {
+                Log.LogError("Timed out waiting for all target frameworks to start.");
+                break;
+            }
+
+            Thread.Sleep(100);
+        }
+        ]]>
+      </Code>
+    </Task>
+  </UsingTask>
+
+  <Target Name="ComputeRunArguments" />
+
+  <Target Name="RunTests"
+          Outputs="%(TestToRun.ResultsStdOutPath)"
+          Condition="'@(TestToRun)' != ''">
+    <PropertyGroup>
+      <_StartedFile>$(ArtifactsTmpDir)$(TargetFramework).started</_StartedFile>
+    </PropertyGroup>
+    <ItemGroup>
+      <_ExpectedStartedFile Include="$(ArtifactsTmpDir)test-tfm-a.started" />
+      <_ExpectedStartedFile Include="$(ArtifactsTmpDir)test-tfm-b.started" />
+    </ItemGroup>
+    <MakeDir Directories="$(ArtifactsTmpDir);$(ArtifactsLogDir)" />
+    <WriteLinesToFile File="$(_StartedFile)" Lines="$(TargetFramework)" Overwrite="true" />
+    <WaitForFiles Files="@(_ExpectedStartedFile)" TimeoutSeconds="10" />
+    <WriteLinesToFile File="%(TestToRun.ResultsStdOutPath)" Lines="$(TargetFramework)" Overwrite="true" />
+    <ItemGroup>
+      <FileWrites Include="$(_StartedFile)" />
+      <FileWrites Include="%(TestToRun.ResultsStdOutPath)" />
+    </ItemGroup>
+  </Target>
+</Project>

src/Microsoft.DotNet.Arcade.Sdk/tools/Tests.targets

@@ -4,8 +4,8 @@
   <PropertyGroup Condition="'$(IsTestProject)' == 'true'">
     <AutoGenerateBindingRedirects Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">true</AutoGenerateBindingRedirects>
 
-    <_GetTestsToRunTarget Condition="'$(TargetFrameworks)' == ''">_InnerGetTestsToRun</_GetTestsToRunTarget>
-    <_GetTestsToRunTarget Condition="'$(TargetFrameworks)' != ''">_OuterGetTestsToRun</_GetTestsToRunTarget>
+    <_RunTestsTarget Condition="'$(TargetFrameworks)' == ''">_InnerRunTests</_RunTestsTarget>
+    <_RunTestsTarget Condition="'$(TargetFrameworks)' != ''">_OuterRunTests</_RunTestsTarget>
 
     <!-- The runtime to run tests on: 'Core', 'Mono', 'Full' (desktop FX). -->
     <TestRuntime Condition="'$(TestRuntime)' == '' and '$(TargetFrameworkIdentifier)' == '.NETCoreApp'">Core</TestRuntime>
@@ -27,8 +27,8 @@
     <Error Condition="!Exists($(TestRunnerTargets))" Text="Test runner $(TestRunnerName) is invalid."/>
   </Target>
 
-  <Target Name="Test" DependsOnTargets="ErrorForMissingTestRunner;$(_GetTestsToRunTarget);_ValidateTestArchitectures;RunTests" Condition="'$(IsUnitTestProject)' == 'true' or '$(IsPerformanceTestProject)' == 'true'" />
-  <Target Name="IntegrationTest" DependsOnTargets="ErrorForMissingTestRunner;$(_GetTestsToRunTarget);_ValidateTestArchitectures;RunTests" Condition="'$(IsIntegrationTestProject)' == 'true'" />
+  <Target Name="Test" DependsOnTargets="ErrorForMissingTestRunner;$(_RunTestsTarget)" Condition="'$(IsUnitTestProject)' == 'true' or '$(IsPerformanceTestProject)' == 'true'" />
+  <Target Name="IntegrationTest" DependsOnTargets="ErrorForMissingTestRunner;$(_RunTestsTarget)" Condition="'$(IsIntegrationTestProject)' == 'true'" />
 
   <PropertyGroup>
     <_InitializeTestArchitecturesDependsOn Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">
@@ -144,15 +144,20 @@
            Text="Test architecture '%(_MismatchedTestArchitecture.Architecture)' cannot use the '%(_MismatchedTestArchitecture.BuiltTestArchitecture)' test host for '%(_MismatchedTestArchitecture.Identity)'. Build an architecture-specific test output or set TestArchitectures to match the architecture selected by PlatformTarget, Prefer32Bit, or RuntimeIdentifier." />
   </Target>
 
-  <Target Name="_OuterGetTestsToRun" Returns="@(TestToRun)">
+  <Target Name="_InnerRunTests"
+          DependsOnTargets="_InnerGetTestsToRun;_ValidateTestArchitectures;RunTests"
+          Returns="@(FileWrites)" />
+
+  <Target Name="_OuterRunTests" Returns="@(FileWrites)">
     <ItemGroup>
       <_TargetFramework Include="$(TargetFrameworks)" />
+      <_InnerTestBuild Include="$(MSBuildProjectFile)"
+                       AdditionalProperties="TargetFramework=%(_TargetFramework.Identity)" />
     </ItemGroup>
-    <MSBuild Projects="$(MSBuildProjectFile)"
-             Condition="'$(TargetFrameworks)' != ''"
-             Targets="_InnerGetTestsToRun"
-             Properties="TargetFramework=%(_TargetFramework.Identity)">
-      <Output ItemName="TestToRun" TaskParameter="TargetOutputs" />
+    <MSBuild Projects="@(_InnerTestBuild)"
+             Targets="_InnerRunTests"
+             BuildInParallel="true">
+      <Output ItemName="FileWrites" TaskParameter="TargetOutputs" />
     </MSBuild>
   </Target>