Enable SuperPMI collection of CoreCLR test run

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -24,6 +24,7 @@ parameters:
   shouldContinueOnError: false
   dependsOn: []
   dependOnEvaluatePaths: false
+  SuperPmiCollect: false
 
 ### Test run job
 
@@ -76,6 +77,9 @@ jobs:
             - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
         - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
           - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }}
+      # SuperPMI collection needs to run mcs.exe on the AzDO machine. Assume that's an x64 machine, and download an x64 product build if needed.
+      - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}:
+          - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', '', parameters.osGroup, parameters.osSubgroup, 'x64', parameters.buildConfig) }}
 
     # Compute job name from template parameters
     ${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}:
@@ -147,6 +151,36 @@ jobs:
         - name: testTreeFilterArg
           value: '-tree:GC/Scenarios/GCSimulator'
 
+    # Variables used for SuperPMI collection
+    - ${{ if eq(parameters.SuperPmiCollect, true) }}:
+      - MchFileTag: '${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}'
+      - name: CollectionType
+        value: 'run'
+      - name: CollectionName
+        value: 'coreclr_tests'
+      - ${{ if eq(parameters.osGroup, 'windows') }}:
+        - name: PythonScript
+          value: 'py -3'
+        - name: PipScript
+          value: 'py -3 -m pip'
+        - name: MchFilesLocation
+          value: '$(Build.SourcesDirectory)\artifacts\helixresults\'
+        - name: MergedMchFileLocation
+          value: '$(Build.SourcesDirectory)\artifacts\spmi_collection\'
+        - name: SpmiLogsLocation
+          value: '$(Build.SourcesDirectory)\artifacts\spmi_logs\'
+      - ${{ if ne(parameters.osGroup, 'windows') }}:
+        - name: PythonScript
+          value: 'python3'
+        - name: PipScript
+          value: 'pip3'
+        - name: MchFilesLocation
+          value: '$(Build.SourcesDirectory)/artifacts/helixresults/'
+        - name: MergedMchFileLocation
+          value: '$(Build.SourcesDirectory)/artifacts/spmi_collection/'
+        - name: SpmiLogsLocation
+          value: '$(Build.SourcesDirectory)/artifacts/spmi_logs/'
+
     # Set job timeouts
     #
     # "timeoutPerTestCollectionInMinutes" is the time needed for the "biggest" xUnit test collection to complete.
@@ -298,6 +332,16 @@ jobs:
         displayName: 'native test artifacts'
 
 
+    # SuperPMI collection: Download x64 coreclr if running on non-x64 configuration (needed for mcs.exe)
+    - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}:
+      - template: /eng/pipelines/common/download-artifact-step.yml
+        parameters:
+          unpackFolder: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper)'
+          artifactFileName: 'CoreCLRProduct___$(osGroup)$(osSubgroup)_x64_$(buildConfig)$(archiveExtension)'
+          artifactName: 'CoreCLRProduct___$(osGroup)$(osSubgroup)_x64_$(buildConfig)'
+          displayName: 'CoreCLR product build (x64)'
+
+
     # Publish native test components to test output folder. Sadly we cannot do this
     # during product build (so that we could zip up the files in their final test location
     # and directly unzip them there after download). Unfortunately the logic to copy
@@ -353,6 +397,7 @@ jobs:
         runtimeFlavor: ${{ parameters.runtimeFlavor }}
         shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
         runtimeVariant: ${{ parameters.runtimeVariant }}
+        SuperPmiCollect: ${{ parameters.SuperPmiCollect }}
 
         ${{ if eq(variables['System.TeamProject'], 'public') }}:
           creator: $(Build.DefinitionName)
@@ -603,3 +648,74 @@ jobs:
         artifactName: '${{ parameters.runtimeFlavor }}_${{ parameters.runtimeVariant }}_$(LogNamePrefix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.testGroup }}'
       continueOnError: true
       condition: always()
+
+    ########################################################################################################
+    #
+    # Finalize SuperPMI collection: (1) merge all MCH files generated by all Helix jobs, (2) upload MCH file to Azure Storage, (3) upload log files
+    # Note that all these steps are "condition: always()" because we want to upload as much of the collection
+    # as possible, even if there were test failures.
+    #
+    ########################################################################################################
+
+    - ${{ if eq(parameters.SuperPmiCollect, true) }}:
+
+      # Create required directories for merged mch collection and superpmi logs
+      - ${{ if ne(parameters.osGroup, 'windows') }}:
+        - script: |
+            mkdir -p $(MergedMchFileLocation)
+            mkdir -p $(SpmiLogsLocation)
+          displayName: Create SuperPMI directories
+          condition: always()
+      - ${{ if eq(parameters.osGroup, 'windows') }}:
+        - script: |
+            mkdir $(MergedMchFileLocation)
+            mkdir $(SpmiLogsLocation)
+          displayName: Create SuperPMI directories
+          condition: always()
+
+      - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch  -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch
+        displayName: 'Merge $(CollectionName)-$(CollectionType) SuperPMI collections'
+        condition: always()
+
+      - template: /eng/pipelines/common/upload-artifact-step.yml
+        parameters:
+          rootFolder: $(MergedMchFileLocation)
+          includeRootFolder: false
+          archiveType: $(archiveType)
+          tarCompression: $(tarCompression)
+          archiveExtension: $(archiveExtension)
+          artifactName: 'SuperPMI_Collection_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
+          displayName: 'Upload artifacts SuperPMI $(CollectionName)-$(CollectionType) collection'
+          condition: always()
+
+      # Ensure the Python azure-storage-blob package is installed before doing the upload.
+      - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall
+        displayName: Upgrade Pip to latest and install azure-storage-blob Python package
+        condition: always()
+
+      - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch $(archType) -build_type $(buildConfig) -mch_files $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper)
+        displayName: 'Upload SuperPMI $(CollectionName)-$(CollectionType) collection to Azure Storage'
+        condition: always()
+        env:
+          CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline
+
+      - task: CopyFiles@2
+        displayName: Copying superpmi.log of all partitions
+        inputs:
+          sourceFolder: '$(MchFilesLocation)'
+          contents: '**/$(CollectionName).$(CollectionType)*.log'
+          targetFolder: '$(SpmiLogsLocation)'
+        condition: always()
+
+      - task: PublishPipelineArtifact@1
+        displayName: Publish SuperPMI logs
+        inputs:
+          targetPath: $(SpmiLogsLocation)
+          artifactName: 'SuperPMI_Logs_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
+        condition: always()
+
+    ########################################################################################################
+    #
+    # End of SuperPMI processing
+    #
+    ########################################################################################################

eng/pipelines/common/templates/runtimes/send-to-helix-step.yml

@@ -26,6 +26,7 @@ parameters:
   runtimeFlavor: 'CoreCLR'
   runtimeVariant: ''
   shouldContinueOnError: false
+  SuperPmiCollect: ''
 
 
 steps:
@@ -57,6 +58,7 @@ steps:
       _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
       RuntimeFlavor: ${{ parameters.runtimeFlavor }}
       _RuntimeVariant: ${{ parameters.runtimeVariant }}
+      _SuperPmiCollect: ${{ parameters.SuperPmiCollect }}
       ${{ if eq(parameters.publishTestResults, 'true') }}:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)
       # TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed

eng/pipelines/common/upload-artifact-step.yml

@@ -6,6 +6,7 @@ parameters:
   archiveExtension: ''
   artifactName: ''
   displayName: ''
+  condition: succeeded()
 
 steps:
   # Zip Artifact
@@ -17,9 +18,11 @@ steps:
       archiveType:       ${{ parameters.archiveType }}
       tarCompression:    ${{ parameters.tarCompression }}
       includeRootFolder: ${{ parameters.includeRootFolder }}
+    condition: ${{ parameters.condition }}
 
   - task: PublishBuildArtifacts@1
     displayName: 'Publish ${{ parameters.displayName }}'
     inputs:
       pathtoPublish: $(Build.StagingDirectory)/${{ parameters.artifactName }}${{ parameters.archiveExtension }}
       artifactName:  ${{ parameters.artifactName }}
+    condition: ${{ parameters.condition }}

eng/pipelines/coreclr/superpmi-collect.yml

@@ -46,6 +46,8 @@ jobs:
       - windows_x64
 
 # superpmi-collect-job that targets macOS/arm64 depends on coreclr binaries produced by the macOS/x64 job
+# We don't collect osx-x64 (it's essentially the same as linux-x64). If we did, we'd add OSX_x64 in the
+# build-coreclr-and-libraries-job.yml above, and remove this.
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
     jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml
@@ -69,8 +71,6 @@ jobs:
     jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml
     buildConfig: checked
     platforms:
-    # Linux tests are built on the OSX machines.
-    # - OSX_x64
     - OSX_arm64
     - Linux_arm
     - Linux_arm64
@@ -91,16 +91,13 @@ jobs:
     jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml
     buildConfig: checked
     platforms:
-    # Linux tests are built on the OSX machines.
-    # - OSX_x64
     - OSX_arm64
     - Linux_arm
     - Linux_arm64
     - Linux_x64
     - windows_x64
     - windows_x86
     - windows_arm64
-    - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
     helixQueueGroup: ci
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     jobParameters:
@@ -114,16 +111,13 @@ jobs:
     jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml
     buildConfig: checked
     platforms:
-    # Linux tests are built on the OSX machines.
-    # - OSX_x64
     - OSX_arm64
     - Linux_arm
     - Linux_arm64
     - Linux_x64
     - windows_x64
     - windows_x86
     - windows_arm64
-    - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
     helixQueueGroup: ci
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     jobParameters:
@@ -137,8 +131,6 @@ jobs:
     jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml
     buildConfig: checked
     platforms:
-    # Linux tests are built on the OSX machines.
-    # - OSX_x64
     - OSX_arm64
     - Linux_arm
     - Linux_arm64
@@ -159,8 +151,6 @@ jobs:
     jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml
     buildConfig: checked
     platforms:
-    # Linux tests are built on the OSX machines.
-    # - OSX_x64
     - OSX_arm64
     - Linux_arm
     - Linux_arm64
@@ -175,3 +165,25 @@ jobs:
       liveLibrariesBuildConfig: Release
       collectionType: run
       collectionName: benchmarks
+
+#
+# Collection of coreclr test run
+#
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
+    buildConfig: checked
+    platforms:
+    - OSX_arm64
+    - Linux_arm
+    - Linux_arm64
+    - Linux_x64
+    - windows_x64
+    - windows_x86
+    - windows_arm64
+    helixQueueGroup: superpmi
+    helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+    jobParameters:
+      testGroup: outerloop
+      liveLibrariesBuildConfig: Release
+      SuperPmiCollect: true

eng/pipelines/coreclr/templates/helix-queues-setup.yml

@@ -11,6 +11,13 @@ parameters:
   dependOnEvaluatePaths: false
   jobParameters: {}
 
+# parameters.jobParameters.helixQueueGroup values:
+# 'pr' -- pull request
+# 'ci' -- continuous integration ("merge")
+# 'libraries' -- libraries tests
+# 'cet' -- machines supporting CET technology
+# 'superpmi' -- for TeamProject 'internal', a smaller set of queues (one per architecture, not several) for SuperPMI collection
+
 jobs:
 - template: ${{ parameters.jobTemplate }}
   parameters:
@@ -45,7 +52,9 @@ jobs:
     - ${{ if eq(parameters.platform, 'Linux_arm') }}:
       - ${{ if eq(variables['System.TeamProject'], 'public') }}:
         - (Ubuntu.1804.Arm32.Open)Ubuntu.1804.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440
-      - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+      - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}:
+        - (Ubuntu.1804.Arm32)Ubuntu.1804.Armarch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440
+      - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}:
         - (Debian.10.Arm32)Ubuntu.1804.Armarch@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-arm32v7-20210304164340-6616c63
         - (Debian.11.Arm32)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-arm32v7-20210304164347-5a7c380
         - (Ubuntu.1804.Arm32)Ubuntu.1804.Armarch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440
@@ -57,7 +66,9 @@ jobs:
       - ${{ if and(eq(variables['System.TeamProject'], 'public'), notIn(parameters.jobParameters.helixQueueGroup, 'pr', 'ci', 'libraries')) }}:
         - (Debian.10.Arm64.Open)Ubuntu.1804.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-arm64v8-20220818195427-06f234f
         - (Debian.11.Arm64.Open)Ubuntu.1804.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-arm64v8-20220818195437-06f234f
-      - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+      - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}:
+        - (Ubuntu.1804.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8-20220824230426-06f234f
+      - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}:
         - (Debian.10.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-arm64v8-20220818195427-06f234f
         - (Debian.11.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-arm64v8-20220818195437-06f234f
         - (Ubuntu.1804.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8-20220824230426-06f234f
@@ -93,7 +104,9 @@ jobs:
         - Ubuntu.1804.Amd64.Open
         - (Centos.8.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-8-helix-20201229003624-c1bf759
         - RedHat.7.Amd64.Open
-      - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+      - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}:
+        - Ubuntu.1804.Amd64
+      - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}:
         - (Debian.10.Amd64)Ubuntu.1804.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64-20220810215022-f344011
         - (Debian.11.Amd64)Ubuntu.1804.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64-20220810215032-f344011
         - Ubuntu.1804.Amd64
@@ -126,7 +139,9 @@ jobs:
           - (Windows.Nano.1809.Amd64.Open)windows.10.amd64.serverrs5.open@mcr.microsoft.com/dotnet-buildtools/prereqs:nanoserver-1809-helix-amd64-08e8e40-20200107182504
           - Windows.7.Amd64.Open
           - Windows.10.Amd64.Open
-      - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+      - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}:
+        - Windows.10.Amd64.X86.Rt
+      - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}:
         - Windows.7.Amd64
         - Windows.81.Amd64
         - Windows.10.Amd64
@@ -140,7 +155,9 @@ jobs:
       - ${{ if and(eq(variables['System.TeamProject'], 'public'), notIn(parameters.jobParameters.helixQueueGroup, 'pr', 'ci', 'libraries')) }}:
         - Windows.7.Amd64.Open
         - Windows.10.Amd64.Open
-      - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+      - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}:
+        - Windows.10.Amd64.X86.Rt
+      - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}:
         - Windows.7.Amd64
         - Windows.81.Amd64
         - Windows.10.Amd64