Batch CI tests based on namespace

azure-pipelines-PR.yml

@@ -507,78 +507,29 @@ stages:
             continueOnError: true
             condition: failed()
 
-          # Windows With Compressed Metadata Desktop
-        - job: WindowsCompressedMetadata_Desktop
-          variables:
-          - name: XUNIT_LOGS
-            value: $(Build.SourcesDirectory)\artifacts\TestResults\Release
-          - name: __VSNeverShowWhatsNew
-            value: 1
-          pool:
-            name: $(DncEngPublicBuildPool)
-            demands: ImageOverride -equals $(_WindowsMachineQueueName)
-          timeoutInMinutes: 120
-
-          steps:
-          - checkout: self
-            clean: true
-
-          - script: eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktop
+          # Windows With Compressed Metadata Desktop (split into 3 batches)
+        - template: eng/templates/batched-test-job.yml
+          parameters:
+            jobName: WindowsCompressedMetadata_Desktop
+            pool:
+              name: $(DncEngPublicBuildPool)
+              demands: ImageOverride -equals $(_WindowsMachineQueueName)
+            buildCommand: 'eng\CIBuildNoPublish.cmd -compressallmetadata -configuration Release -testDesktopBatch $(batchNumber)'
+            batches:
+              Batch1: { batchNumber: 1 }
+              Batch2: { batchNumber: 2 }
+              Batch3: { batchNumber: 3 }
+            variables:
+            - name: __VSNeverShowWhatsNew
+              value: 1
             env:
               FSharp_CacheEvictionImmediate: true
               DOTNET_DbgEnableMiniDump: 1
-              DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing.
+              DOTNET_DbgMiniDumpType: 3
               DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp
               NativeToolsOnMachine: true
-            displayName: Build / Test
-
-          - task: PublishTestResults@2
-            displayName: Publish Test Results
-            inputs:
-              testResultsFormat: 'VSTest'
-              testRunTitle: WindowsCompressedMetadata testDesktop
-              mergeTestResults: true
-              testResultsFiles: '*.trx'
-              searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release'
-            continueOnError: true
-            condition: succeededOrFailed()
-
-          - task: PublishBuildArtifacts@1
-            displayName: Publish BinLog
-            continueOnError: true
-            inputs:
-                PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog'
-                ArtifactName: 'Windows testDesktop binlogs'
-                ArtifactType: Container
-                parallel: true
-          - task: PublishBuildArtifacts@1
-            displayName: Publish Dumps
-            condition: failed()
-            continueOnError: true
-            inputs:
-                PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release'
-                ArtifactName: 'Windows testDesktop process dumps'
-                ArtifactType: Container
-                parallel: true
-          - task: PublishBuildArtifacts@1
-            displayName: Publish Test Logs
-            inputs:
-              PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release'
-              ArtifactName: 'Windows testDesktop test logs'
-              publishLocation: Container
-            continueOnError: true
-            condition: always()
-          - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj
-            displayName: Dump NuGet cache contents
-            condition: failed()
-          - task: PublishBuildArtifacts@1
-            displayName: Publish NuGet cache contents
-            inputs:
-              PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents'
-              ArtifactName: 'NuGetPackageContents Windows testDesktop'
-              publishLocation: Container
-            continueOnError: true
-            condition: failed()
+            publishBinLog: true
+            publishDumps: true
 
         # Mock official build
         - job: MockOfficial
@@ -635,50 +586,23 @@ stages:
             continueOnError: true
             condition: failed()
 
-        # MacOS
-        - job: MacOS
-          pool:
-            vmImage: macos-latest
-          timeoutInMinutes: 120
-          variables:
-          - name: _SignType
-            value: Test
-          steps:
-          - checkout: self
-            clean: true
-          - script: ./eng/cibuild.sh --configuration $(_BuildConfig) --testcoreclr
+        # MacOS (split into 3 batches)
+        - template: eng/templates/batched-test-job.yml
+          parameters:
+            jobName: MacOS
+            pool:
+              vmImage: macos-latest
+            buildCommand: './eng/cibuild.sh --configuration $(_BuildConfig) --testcoreclrbatch $(batchNumber)'
+            configuration: $(_BuildConfig)
+            batches:
+              Batch1: { batchNumber: 1 }
+              Batch2: { batchNumber: 2 }
+              Batch3: { batchNumber: 3 }
+            variables:
+            - name: _SignType
+              value: Test
             env:
               COMPlus_DefaultStackSize: 1000000
-            displayName: Build / Test
-          - task: PublishTestResults@2
-            displayName: Publish Test Results
-            inputs:
-              testResultsFormat: 'VSTest'
-              testResultsFiles: '*.trx'
-              testRunTitle: MacOS
-              mergeTestResults: true
-              searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
-            continueOnError: true
-            condition: succeededOrFailed()
-          - task: PublishBuildArtifacts@1
-            displayName: Publish Test Logs
-            inputs:
-              PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
-              ArtifactName: 'MacOS $(_BuildConfig) test logs'
-              publishLocation: Container
-            continueOnError: true
-            condition: failed()
-          - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj
-            displayName: Dump NuGet cache contents
-            condition: failed()
-          - task: PublishBuildArtifacts@1
-            displayName: Publish NuGet cache contents
-            inputs:
-              PathtoPublish: '$(Build.SourcesDirectory)/artifacts/NugetPackageRootContents'
-              ArtifactName: 'NuGetPackageContents Mac'
-              publishLocation: Container
-            continueOnError: true
-            condition: failed()
 
         # End to end build
         - job: EndToEndBuildTests

eng/Build.ps1

@@ -49,6 +49,7 @@ param (
     [switch]$dontUseGlobalNuGetCache = $false,
     [switch]$warnAsError = $true,
     [switch][Alias('test')]$testDesktop,
+    [string]$testDesktopBatch = "",
     [switch]$testCoreClr,
     [switch]$testCambridge,
     [switch]$testCompiler,
@@ -121,6 +122,7 @@ function Print-Usage() {
     Write-Host "  -testCompilerService          Run FSharpCompilerService unit tests"
     Write-Host "  -testCompilerComponentTests   Run FSharpCompilerService component tests"
     Write-Host "  -testDesktop                  Run tests against full .NET Framework"
+    Write-Host "  -testDesktopBatch <1|2|3>       Run a specific batch of the desktop test split (implies -testDesktop)"
     Write-Host "  -testCoreClr                  Run tests against CoreCLR"
     Write-Host "  -testFSharpCore               Run FSharpCore unit tests"
     Write-Host "  -testIntegration              Run F# integration tests"
@@ -193,6 +195,10 @@ function Process-Arguments() {
         $script:testEditor = $True
     }
 
+    if ($script:testDesktopBatch -ne "") {
+        $script:testDesktop = $True
+    }
+
     if ([System.Boolean]::Parse($script:officialSkipTests)) {
         $script:testAll = $False
         $script:testAllButIntegration = $False
@@ -605,7 +611,23 @@ try {
     }
 
     if ($testDesktop) {
-        TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework
+        if ($testDesktopBatch -ne "") {
+            $dotnetPath = InitializeDotNetCli
+            $dotnetExe = Join-Path $dotnetPath "dotnet.exe"
+            $splitScript = Join-Path $RepoRoot "eng\tests\TestSplit.fsx"
+            $splitOutput = & $dotnetExe fsi $splitScript $testDesktopBatch
+            if ($LASTEXITCODE -ne 0) { throw "TestSplit.fsx failed with exit code $LASTEXITCODE" }
+            foreach ($line in $splitOutput) {
+                if ($line -match '^dotnet test (\S+) --no-build -c Release\s*(.*)$') {
+                    $proj = $Matches[1] -replace '/', '\'
+                    $projPath = Join-Path $RepoRoot $proj
+                    $settings = $Matches[2].Trim()
+                    TestUsingMSBuild -testProject $projPath -targetFramework $script:desktopTargetFramework -settings $settings
+                }
+            }
+        } else {
+            TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework
+        }
     }
 
     if ($testFSharpCore) {

eng/build.sh

@@ -25,6 +25,7 @@ usage()
   echo ""
   echo "Test actions:"
   echo "  --testcoreclr                  Run unit tests on .NET Core (short: --test, -t)"
+  echo "  --testcoreclrbatch <1|2|3>     Run a specific batch of CoreCLR tests"
   echo "  --testCompilerComponentTests   Run FSharp.Compiler.ComponentTests on .NET Core"
   echo "  --testBenchmarks               Build and Run Benchmark suite"
   echo "  --testScripting                Run FSharp.Private.ScriptingTests on .NET Core"
@@ -63,6 +64,7 @@ pack=false
 publish=false
 sign=false
 test_core_clr=false
+test_core_clr_batch=""
 test_compilercomponent_tests=false
 test_benchmarks=false
 test_scripting=false
@@ -142,6 +144,10 @@ while [[ $# > 0 ]]; do
     --testcoreclr|--test|-t)
       test_core_clr=true
       ;;
+    --testcoreclrbatch)
+      test_core_clr_batch=$2
+      shift
+      ;;
     --testcompilercomponenttests)
       test_compilercomponent_tests=true
       ;;
@@ -204,6 +210,7 @@ function Test() {
   BuildMessage="Error running tests"
   testproject=""
   targetframework=""
+  extraargs=""
   while [[ $# > 0 ]]; do
     opt="$(echo "$1" | awk '{print tolower($0)}')"
     case "$opt" in
@@ -215,6 +222,10 @@ function Test() {
         targetframework=$2
         shift
         ;;
+      --extraargs)
+        extraargs=$2
+        shift
+        ;;
       *)
         echo "Invalid argument: $1"
         exit 1
@@ -244,7 +255,8 @@ function Test() {
     reportargs="--report-xunit-trx --report-xunit-trx-filename $testlogfilename"
   fi
 
-  args=(test $testtarget "$testproject" --no-build -c "$configuration" -f "$targetframework" $reportargs --results-directory "$testresultsdir" --hangdump --hangdump-timeout 5m --hangdump-type Full)
+  # Note: $extraargs is intentionally unquoted to allow word splitting into separate args
+  args=(test $testtarget "$testproject" --no-build -c "$configuration" -f "$targetframework" $reportargs --results-directory "$testresultsdir" --hangdump --hangdump-timeout 5m --hangdump-type Full $extraargs)
 
   "$DOTNET_INSTALL_DIR/dotnet" "${args[@]}" || exit $?
 }
@@ -360,7 +372,16 @@ fi
 
 BuildSolution
 
-if [[ "$test_core_clr" == true ]]; then
+if [[ "$test_core_clr_batch" != "" ]]; then
+  coreclrtestframework=$tfm
+  while IFS= read -r line; do
+    if [[ "$line" == dotnet\ test\ * ]]; then
+      proj=$(echo "$line" | awk '{print $3}')
+      settings=$(echo "$line" | sed 's/^dotnet test [^ ]* --no-build -c Release *//')
+      Test --testproject "$repo_root/$proj" --targetframework "$coreclrtestframework" --extraargs "$settings"
+    fi
+  done < <("$DOTNET_INSTALL_DIR/dotnet" fsi "$scriptroot/tests/TestSplit.fsx" "$test_core_clr_batch" 2>/dev/null)
+elif [[ "$test_core_clr" == true ]]; then
   coreclrtestframework=$tfm
   # Note: FSharp.Test.Utilities is a utility library, not a test project. Its tests are disabled due to xUnit3 API incompatibilities.
   Test --testproject "$repo_root/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj" --targetframework $coreclrtestframework