diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 4bc3e519aec457..7e8fcaf4175f8b 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -24,6 +24,7 @@ parameters: shouldContinueOnError: false dependsOn: [] dependOnEvaluatePaths: false + SuperPmiCollect: false ### Test run job @@ -76,6 +77,9 @@ jobs: - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + # SuperPMI collection needs to run mcs.exe on the AzDO machine. Assume that's an x64 machine, and download an x64 product build if needed. + - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}: + - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', '', parameters.osGroup, parameters.osSubgroup, 'x64', parameters.buildConfig) }} # Compute job name from template parameters ${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: @@ -147,6 +151,36 @@ jobs: - name: testTreeFilterArg value: '-tree:GC/Scenarios/GCSimulator' + # Variables used for SuperPMI collection + - ${{ if eq(parameters.SuperPmiCollect, true) }}: + - MchFileTag: '${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}' + - name: CollectionType + value: 'run' + - name: CollectionName + value: 'coreclr_tests' + - ${{ if eq(parameters.osGroup, 'windows') }}: + - name: PythonScript + value: 'py -3' + - name: PipScript + value: 'py -3 -m pip' + - name: MchFilesLocation + value: '$(Build.SourcesDirectory)\artifacts\helixresults\' + - name: MergedMchFileLocation + value: '$(Build.SourcesDirectory)\artifacts\spmi_collection\' + - name: SpmiLogsLocation + value: '$(Build.SourcesDirectory)\artifacts\spmi_logs\' + - ${{ if ne(parameters.osGroup, 'windows') }}: + - name: PythonScript + value: 'python3' + - name: PipScript + value: 'pip3' + - name: MchFilesLocation + value: '$(Build.SourcesDirectory)/artifacts/helixresults/' + - name: MergedMchFileLocation + value: '$(Build.SourcesDirectory)/artifacts/spmi_collection/' + - name: SpmiLogsLocation + value: '$(Build.SourcesDirectory)/artifacts/spmi_logs/' + # Set job timeouts # # "timeoutPerTestCollectionInMinutes" is the time needed for the "biggest" xUnit test collection to complete. @@ -298,6 +332,16 @@ jobs: displayName: 'native test artifacts' + # SuperPMI collection: Download x64 coreclr if running on non-x64 configuration (needed for mcs.exe) + - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper)' + artifactFileName: 'CoreCLRProduct___$(osGroup)$(osSubgroup)_x64_$(buildConfig)$(archiveExtension)' + artifactName: 'CoreCLRProduct___$(osGroup)$(osSubgroup)_x64_$(buildConfig)' + displayName: 'CoreCLR product build (x64)' + + # Publish native test components to test output folder. Sadly we cannot do this # during product build (so that we could zip up the files in their final test location # and directly unzip them there after download). Unfortunately the logic to copy @@ -353,6 +397,7 @@ jobs: runtimeFlavor: ${{ parameters.runtimeFlavor }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} runtimeVariant: ${{ parameters.runtimeVariant }} + SuperPmiCollect: ${{ parameters.SuperPmiCollect }} ${{ if eq(variables['System.TeamProject'], 'public') }}: creator: $(Build.DefinitionName) @@ -603,3 +648,74 @@ jobs: artifactName: '${{ parameters.runtimeFlavor }}_${{ parameters.runtimeVariant }}_$(LogNamePrefix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.testGroup }}' continueOnError: true condition: always() + + ######################################################################################################## + # + # Finalize SuperPMI collection: (1) merge all MCH files generated by all Helix jobs, (2) upload MCH file to Azure Storage, (3) upload log files + # Note that all these steps are "condition: always()" because we want to upload as much of the collection + # as possible, even if there were test failures. + # + ######################################################################################################## + + - ${{ if eq(parameters.SuperPmiCollect, true) }}: + + # Create required directories for merged mch collection and superpmi logs + - ${{ if ne(parameters.osGroup, 'windows') }}: + - script: | + mkdir -p $(MergedMchFileLocation) + mkdir -p $(SpmiLogsLocation) + displayName: Create SuperPMI directories + condition: always() + - ${{ if eq(parameters.osGroup, 'windows') }}: + - script: | + mkdir $(MergedMchFileLocation) + mkdir $(SpmiLogsLocation) + displayName: Create SuperPMI directories + condition: always() + + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch + displayName: 'Merge $(CollectionName)-$(CollectionType) SuperPMI collections' + condition: always() + + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(MergedMchFileLocation) + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: 'SuperPMI_Collection_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + displayName: 'Upload artifacts SuperPMI $(CollectionName)-$(CollectionType) collection' + condition: always() + + # Ensure the Python azure-storage-blob package is installed before doing the upload. + - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall + displayName: Upgrade Pip to latest and install azure-storage-blob Python package + condition: always() + + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch $(archType) -build_type $(buildConfig) -mch_files $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper) + displayName: 'Upload SuperPMI $(CollectionName)-$(CollectionType) collection to Azure Storage' + condition: always() + env: + CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline + + - task: CopyFiles@2 + displayName: Copying superpmi.log of all partitions + inputs: + sourceFolder: '$(MchFilesLocation)' + contents: '**/$(CollectionName).$(CollectionType)*.log' + targetFolder: '$(SpmiLogsLocation)' + condition: always() + + - task: PublishPipelineArtifact@1 + displayName: Publish SuperPMI logs + inputs: + targetPath: $(SpmiLogsLocation) + artifactName: 'SuperPMI_Logs_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + condition: always() + + ######################################################################################################## + # + # End of SuperPMI processing + # + ######################################################################################################## diff --git a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml index 0a0242529eb9d5..dfad8c58d880ae 100644 --- a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml +++ b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml @@ -26,6 +26,7 @@ parameters: runtimeFlavor: 'CoreCLR' runtimeVariant: '' shouldContinueOnError: false + SuperPmiCollect: '' steps: @@ -57,6 +58,7 @@ steps: _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }} RuntimeFlavor: ${{ parameters.runtimeFlavor }} _RuntimeVariant: ${{ parameters.runtimeVariant }} + _SuperPmiCollect: ${{ parameters.SuperPmiCollect }} ${{ if eq(parameters.publishTestResults, 'true') }}: SYSTEM_ACCESSTOKEN: $(System.AccessToken) # TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed diff --git a/eng/pipelines/common/upload-artifact-step.yml b/eng/pipelines/common/upload-artifact-step.yml index df753b1ed0321e..249da066c7aae6 100644 --- a/eng/pipelines/common/upload-artifact-step.yml +++ b/eng/pipelines/common/upload-artifact-step.yml @@ -6,6 +6,7 @@ parameters: archiveExtension: '' artifactName: '' displayName: '' + condition: succeeded() steps: # Zip Artifact @@ -17,9 +18,11 @@ steps: archiveType: ${{ parameters.archiveType }} tarCompression: ${{ parameters.tarCompression }} includeRootFolder: ${{ parameters.includeRootFolder }} + condition: ${{ parameters.condition }} - task: PublishBuildArtifacts@1 displayName: 'Publish ${{ parameters.displayName }}' inputs: pathtoPublish: $(Build.StagingDirectory)/${{ parameters.artifactName }}${{ parameters.archiveExtension }} artifactName: ${{ parameters.artifactName }} + condition: ${{ parameters.condition }} diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 57093bb758a1af..2dc130c57ea656 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -46,6 +46,8 @@ jobs: - windows_x64 # superpmi-collect-job that targets macOS/arm64 depends on coreclr binaries produced by the macOS/x64 job +# We don't collect osx-x64 (it's essentially the same as linux-x64). If we did, we'd add OSX_x64 in the +# build-coreclr-and-libraries-job.yml above, and remove this. - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml @@ -69,8 +71,6 @@ jobs: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - # Linux tests are built on the OSX machines. - # - OSX_x64 - OSX_arm64 - Linux_arm - Linux_arm64 @@ -91,8 +91,6 @@ jobs: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - # Linux tests are built on the OSX machines. - # - OSX_x64 - OSX_arm64 - Linux_arm - Linux_arm64 @@ -100,7 +98,6 @@ jobs: - windows_x64 - windows_x86 - windows_arm64 - - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: @@ -114,8 +111,6 @@ jobs: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - # Linux tests are built on the OSX machines. - # - OSX_x64 - OSX_arm64 - Linux_arm - Linux_arm64 @@ -123,7 +118,6 @@ jobs: - windows_x64 - windows_x86 - windows_arm64 - - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: @@ -137,8 +131,6 @@ jobs: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - # Linux tests are built on the OSX machines. - # - OSX_x64 - OSX_arm64 - Linux_arm - Linux_arm64 @@ -159,8 +151,6 @@ jobs: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - # Linux tests are built on the OSX machines. - # - OSX_x64 - OSX_arm64 - Linux_arm - Linux_arm64 @@ -175,3 +165,25 @@ jobs: liveLibrariesBuildConfig: Release collectionType: run collectionName: benchmarks + +# +# Collection of coreclr test run +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + buildConfig: checked + platforms: + - OSX_arm64 + - Linux_arm + - Linux_arm64 + - Linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: superpmi + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + SuperPmiCollect: true \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/helix-queues-setup.yml b/eng/pipelines/coreclr/templates/helix-queues-setup.yml index 8160f934a9d046..796a36abbda258 100644 --- a/eng/pipelines/coreclr/templates/helix-queues-setup.yml +++ b/eng/pipelines/coreclr/templates/helix-queues-setup.yml @@ -11,6 +11,13 @@ parameters: dependOnEvaluatePaths: false jobParameters: {} +# parameters.jobParameters.helixQueueGroup values: +# 'pr' -- pull request +# 'ci' -- continuous integration ("merge") +# 'libraries' -- libraries tests +# 'cet' -- machines supporting CET technology +# 'superpmi' -- for TeamProject 'internal', a smaller set of queues (one per architecture, not several) for SuperPMI collection + jobs: - template: ${{ parameters.jobTemplate }} parameters: @@ -45,7 +52,9 @@ jobs: - ${{ if eq(parameters.platform, 'Linux_arm') }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - (Ubuntu.1804.Arm32.Open)Ubuntu.1804.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440 - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: + - (Ubuntu.1804.Arm32)Ubuntu.1804.Armarch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440 + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: - (Debian.10.Arm32)Ubuntu.1804.Armarch@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-arm32v7-20210304164340-6616c63 - (Debian.11.Arm32)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-arm32v7-20210304164347-5a7c380 - (Ubuntu.1804.Arm32)Ubuntu.1804.Armarch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440 @@ -57,7 +66,9 @@ jobs: - ${{ if and(eq(variables['System.TeamProject'], 'public'), notIn(parameters.jobParameters.helixQueueGroup, 'pr', 'ci', 'libraries')) }}: - (Debian.10.Arm64.Open)Ubuntu.1804.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-arm64v8-20220818195427-06f234f - (Debian.11.Arm64.Open)Ubuntu.1804.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-arm64v8-20220818195437-06f234f - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: + - (Ubuntu.1804.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8-20220824230426-06f234f + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: - (Debian.10.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-arm64v8-20220818195427-06f234f - (Debian.11.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-arm64v8-20220818195437-06f234f - (Ubuntu.1804.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8-20220824230426-06f234f @@ -93,7 +104,9 @@ jobs: - Ubuntu.1804.Amd64.Open - (Centos.8.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-8-helix-20201229003624-c1bf759 - RedHat.7.Amd64.Open - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: + - Ubuntu.1804.Amd64 + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: - (Debian.10.Amd64)Ubuntu.1804.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64-20220810215022-f344011 - (Debian.11.Amd64)Ubuntu.1804.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64-20220810215032-f344011 - Ubuntu.1804.Amd64 @@ -126,7 +139,9 @@ jobs: - (Windows.Nano.1809.Amd64.Open)windows.10.amd64.serverrs5.open@mcr.microsoft.com/dotnet-buildtools/prereqs:nanoserver-1809-helix-amd64-08e8e40-20200107182504 - Windows.7.Amd64.Open - Windows.10.Amd64.Open - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: + - Windows.10.Amd64.X86.Rt + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: - Windows.7.Amd64 - Windows.81.Amd64 - Windows.10.Amd64 @@ -140,7 +155,9 @@ jobs: - ${{ if and(eq(variables['System.TeamProject'], 'public'), notIn(parameters.jobParameters.helixQueueGroup, 'pr', 'ci', 'libraries')) }}: - Windows.7.Amd64.Open - Windows.10.Amd64.Open - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: + - Windows.10.Amd64.X86.Rt + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: - Windows.7.Amd64 - Windows.81.Amd64 - Windows.10.Amd64 diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 5ea9e3f03b1a6e..893857887ef28c 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -73,6 +73,8 @@ jobs: value: '$(Build.SourcesDirectory)\artifacts\spmi_collection\' - name: SpmiLogsLocation value: '$(Build.SourcesDirectory)\artifacts\spmi_logs\' + - name: PayloadLocation + value: '$(Build.SourcesDirectory)\payload' - ${{ if ne(parameters.osGroup, 'windows') }}: - name: PythonScript value: 'python3' @@ -86,6 +88,9 @@ jobs: value: '$(Build.SourcesDirectory)/artifacts/spmi_collection/' - name: SpmiLogsLocation value: '$(Build.SourcesDirectory)/artifacts/spmi_logs/' + - name: PayloadLocation + value: '$(Build.SourcesDirectory)/payload' + - ${{ if eq(parameters.collectionName, 'libraries') }}: - name: InputDirectory value: '$(Core_Root_Dir)' @@ -98,6 +103,7 @@ jobs: - ${{ if eq(parameters.collectionName, 'libraries_tests') }}: - name: InputDirectory value: '$(Build.SourcesDirectory)/artifacts/tests/libraries/$(osGroup).$(archType).$(buildConfigUpper)' + workspace: clean: all pool: @@ -106,7 +112,7 @@ jobs: steps: - ${{ parameters.steps }} - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} # Create required directories for merged mch collection and superpmi logs @@ -144,7 +150,7 @@ jobs: # Always run merge step even if collection of some partition fails so we can store collection # of the partitions that succeeded. If all the partitions fail, merge-mch would fail and we won't # run future steps like uploading superpmi collection. - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch --ci -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch displayName: ${{ format('Merge {0}-{1} SuperPMI collections', parameters.collectionName, parameters.collectionType) }} condition: always() diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index a2ca69af7f9957..85979cdc265afb 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -1,30 +1,40 @@ - - - + + + + --> \ @@ -33,62 +43,88 @@ / - + AssembliesPayload - Path that will be sent to helix machine to run collection on + AssembliesDirectoryOnHelix - Path on helix machine itself where superpmi.py will discover the sent assemblies. + --> - - - - - - + - + %HELIX_PYTHONPATH% - $(WorkItemDirectory)\pmiAssembliesDirectory - %HELIX_WORKITEM_PAYLOAD%\binaries %HELIX_CORRELATION_PAYLOAD%\superpmi - - - %HELIX_WORKITEM_PAYLOAD%\performance - %HELIX_WORKITEM_UPLOAD_ROOT% $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults - $(SuperPMIDirectory)\superpmi.py collect --clean -log_level DEBUG --$(CollectionType) -pmi_location $(SuperPMIDirectory)\pmi.dll -pmi_path @(PmiPathDirectories->'$(SuperPMIDirectory)\%(Identity)', ' ') $HELIX_PYTHONPATH - $(WorkItemDirectory)/pmiAssembliesDirectory - $HELIX_WORKITEM_PAYLOAD/binaries $HELIX_CORRELATION_PAYLOAD/superpmi - - - $HELIX_WORKITEM_PAYLOAD/performance - $HELIX_WORKITEM_UPLOAD_ROOT $(BUILD_SOURCESDIRECTORY)/artifacts/helixresults - $(SuperPMIDirectory)/superpmi.py collect --clean -log_level DEBUG --$(CollectionType) -pmi_location $(SuperPMIDirectory)/pmi.dll -pmi_path @(PmiPathDirectories->'$(SuperPMIDirectory)/%(Identity)', ' ') - - $(Python) $(WorkItemCommand) -assemblies $(PmiAssembliesDirectory) -arch $(Architecture) -build_type $(BuildConfig) -core_root $(SuperPMIDirectory) + + + + %HELIX_WORKITEM_PAYLOAD%\binaries + + + $HELIX_WORKITEM_PAYLOAD/binaries + + + + + + %HELIX_WORKITEM_PAYLOAD%\performance + + + $HELIX_WORKITEM_PAYLOAD/performance + + + + + + $(WorkItemDirectory)$(FileSeparatorChar)collectAssembliesDirectory$(FileSeparatorChar)$(CollectionName) + + + + + + + + + + + + -pmi_location $(SuperPMIDirectory)\pmi.dll -pmi_path @(PmiPathDirectories->'$(SuperPMIDirectory)\%(Identity)', ' ') + + + -pmi_location $(SuperPMIDirectory)/pmi.dll -pmi_path @(PmiPathDirectories->'$(SuperPMIDirectory)/%(Identity)', ' ') + + + + + + + + $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi.py collect --clean -log_level DEBUG --$(CollectionType) $(PmiArguments) -assemblies $(AssembliesDirectoryOnHelix) -arch $(Architecture) -build_type $(BuildConfig) -core_root $(SuperPMIDirectory) 2:00 - $(Python) $(SuperPMIDirectory)/superpmi_benchmarks.py -performance_directory $(PerformanceDirectory) -superpmi_directory $(SuperPMIDirectory) -core_root $(SuperPMIDirectory) -arch $(Architecture) + $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_benchmarks.py -performance_directory $(PerformanceDirectory) -superpmi_directory $(SuperPMIDirectory) -core_root $(SuperPMIDirectory) -arch $(Architecture) 3:00 @@ -101,12 +137,12 @@ + - + - + + 30 - + @@ -167,7 +209,7 @@ $(CollectionName).$(CollectionType).%(HelixWorkItem.PartitionId).$(MchFileTag) - $(PmiAssembliesPayload)$(FileSeparatorChar)$(CollectionName)$(FileSeparatorChar)%(HelixWorkItem.PmiAssemblies) + $(AssembliesPayload)$(FileSeparatorChar)%(HelixWorkItem.CollectAssemblies) $(WorkItemCommand) -output_mch_path $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).mch -log_file $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).log $(WorkItemTimeout) %(OutputFileName).mch;%(OutputFileName).mch.mct;%(OutputFileName).log diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py index b33983133b0aa7..627d71d7255c5a 100644 --- a/src/coreclr/scripts/superpmi.py +++ b/src/coreclr/scripts/superpmi.py @@ -295,12 +295,14 @@ def add_core_root_arguments(parser, build_type_default, build_type_help): collect_parser.add_argument("-mch_files", metavar="MCH_FILE", nargs='+', help="Pass a sequence of MCH files which will be merged. Required by --merge_mch_files.") collect_parser.add_argument("--use_zapdisable", action="store_true", help="Sets COMPlus_ZapDisable=1 and COMPlus_ReadyToRun=0 when doing collection to cause NGEN/ReadyToRun images to not be used, and thus causes JIT compilation and SuperPMI collection of these methods.") collect_parser.add_argument("--tiered_compilation", action="store_true", help="Sets COMPlus_TieredCompilation=1 when doing collections.") +collect_parser.add_argument("--ci", action="store_true", help="Special collection mode for handling zero-sized files in Azure DevOps + Helix pipelines collections.") # Allow for continuing a collection in progress collect_parser.add_argument("-temp_dir", help="Specify an existing temporary directory to use. Useful if continuing an ongoing collection process, or forcing a temporary directory to a particular hard drive. Optional; default is to create a temporary directory in the usual TEMP location.") collect_parser.add_argument("--clean", action="store_true", help="Clean the collection by removing contexts that fail to replay without error.") collect_parser.add_argument("--skip_collection_step", action="store_true", help="Do not run the collection step.") collect_parser.add_argument("--skip_merge_step", action="store_true", help="Do not run the merge step.") +collect_parser.add_argument("--skip_toc_step", action="store_true", help="Do not run the TOC creation step.") collect_parser.add_argument("--skip_collect_mc_files", action="store_true", help="Do not collect .MC files") # Create a set of arguments common to all SuperPMI replay commands, namely basic replay and ASM diffs. @@ -387,6 +389,7 @@ def add_core_root_arguments(parser, build_type_default, build_type_help): merge_mch_parser.add_argument("-output_mch_path", required=True, help="Location to place the final MCH file.") merge_mch_parser.add_argument("-pattern", required=True, help=merge_mch_pattern_help) +merge_mch_parser.add_argument("--ci", action="store_true", help="Special collection mode for handling zero-sized files in Azure DevOps + Helix pipelines collections.") ################################################################################ # Helper functions @@ -731,12 +734,16 @@ def collect(self): else: self.__copy_to_final_mch_file__() - self.__create_toc__() + if not self.coreclr_args.skip_toc_step: + self.__create_toc__() if self.coreclr_args.clean: # There is no point to verify unless we have run the clean step. self.__verify_final_mch__() + if self.coreclr_args.ci: + self.__process_for_ci__() + passed = True except Exception as exception: @@ -1175,6 +1182,21 @@ def __verify_final_mch__(self): if not passed: raise RuntimeError("Error, unclean replay.") + def __process_for_ci__(self): + """ Helix doesn't upload zero-sized files. Sometimes we end up with zero-sized .mch files if + there is no data collected. Convert these to special "sentinel" files that are later processed + by "merge-mch" by deleting them. This prevents the + file download to succeed (because the file exists) and the MCH merge to also succeed. + """ + + logging.info("Process MCH files for CI") + + if os.path.getsize(self.final_mch_file) == 0: + # Convert to sentinel file + logging.info("Converting zero-length MCH file {} to ZEROLENGTH sentinel file".format(self.final_mch_file)) + with open(self.final_mch_file, "w") as write_fh: + write_fh.write("ZEROLENGTH") + ################################################################################ # SuperPMI Replay helpers ################################################################################ @@ -2909,6 +2931,12 @@ def merge_mch(coreclr_args): mcs -merge -recursive -dedup -thin mcs -toc + If `--ci` is passed, it looks for special "ZEROLENGTH" files and deletes them + before invoking the merge. This is to handle a CI issue where we generate zero-length + .MCH files in Helix, convert them to non-zero-length sentinel files so they will be + successfully uploaded to artifacts storage, then downloaded on AzDO using the + HelixWorkItem.DownloadFilesFromResults mechanism. Then, we delete them before merging. + Args: coreclr_args (CoreclrArguments) : parsed args @@ -2916,6 +2944,23 @@ def merge_mch(coreclr_args): True on success, else False """ + if coreclr_args.ci: + # Handle zero-length sentinel files. + # Recurse looking for small files that have the text ZEROLENGTH, and delete them. + try: + for dirpath, _, filenames in os.walk(os.path.dirname(coreclr_args.pattern)): + for file_name in filenames: + file_path = os.path.join(dirpath, file_name) + if os.path.getsize(file_path) < 15: + with open(file_path, "rb") as fh: + contents = fh.read() + match = re.search(b'ZEROLENGTH', contents) + if match is not None: + logging.info("Removing zero-length MCH sentinel file {}".format(file_path)) + os.remove(file_path) + except Exception: + pass + logging.info("Merging %s -> %s", coreclr_args.pattern, coreclr_args.output_mch_path) mcs_path = determine_mcs_tool_path(coreclr_args) command = [mcs_path, "-merge", coreclr_args.output_mch_path, coreclr_args.pattern, "-recursive", "-dedup", "-thin"] @@ -3523,6 +3568,16 @@ def verify_base_diff_args(): lambda unused: True, "Unable to set skip_merge_step.") + coreclr_args.verify(args, + "skip_toc_step", + lambda unused: True, + "Unable to set skip_toc_step.") + + coreclr_args.verify(args, + "ci", + lambda unused: True, + "Unable to set ci.") + coreclr_args.verify(args, "clean", lambda unused: True, @@ -3543,8 +3598,8 @@ def verify_base_diff_args(): lambda unused: True, "Unable to set pmi_path") - if (args.collection_command is None) and (args.pmi is False) and (args.crossgen2 is False): - print("Either a collection command or `--pmi` or `--crossgen2` must be specified") + if (args.collection_command is None) and (args.pmi is False) and (args.crossgen2 is False) and not coreclr_args.skip_collection_step: + print("Either a collection command or `--pmi` or `--crossgen2` or `--skip_collection_step` must be specified") sys.exit(1) if (args.collection_command is not None) and (len(args.assemblies) > 0): @@ -3565,7 +3620,7 @@ def verify_base_diff_args(): if args.pmi_location is not None: logging.warning("Warning: -pmi_location is set but --pmi is not.") - if args.collection_command is None and args.merge_mch_files is not True: + if args.collection_command is None and args.merge_mch_files is not True and not coreclr_args.skip_collection_step: assert args.collection_args is None assert (args.pmi is True) or (args.crossgen2 is True) assert len(args.assemblies) > 0 @@ -3872,6 +3927,11 @@ def verify_base_diff_args(): lambda unused: True, "Unable to set pattern") + coreclr_args.verify(args, + "ci", + lambda unused: True, + "Unable to set ci.") + if coreclr_args.mode == "replay" or coreclr_args.mode == "asmdiffs" or coreclr_args.mode == "tpdiff" or coreclr_args.mode == "download": if hasattr(coreclr_args, "private_store") and coreclr_args.private_store is not None: logging.info("Using private stores:") diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 497ff9f64d57bd..815041c97685d0 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -9,22 +9,25 @@ # # Script to setup directory structure required to perform SuperPMI collection in CI. # It does the following steps: -# 1. It creates `correlation_payload_directory` that contains files from CORE_ROOT, src\coreclr\scripts. -# This directory is the one that is sent to all the helix machines that performs SPMI collection. -# 2. It clones dotnet/jitutils, builds it and then copies the `pmi.dll` to `correlation_payload_directory` folder. -# This file is needed to do pmi SPMI runs. -# 3. The script takes `input_artifacts` parameter which contains managed .dlls and .exes on -# which SPMI needs to be run. This script will partition these folders into equal buckets of approximately `max_size` -# bytes and stores them under `payload` directory. Each sub-folder inside `payload` directory is sent to individual -# helix machine to do SPMI collection on. E.g. for `input_artifacts` to be run on libraries, the parameter would be path to -# `CORE_ROOT` folder and this script will copy `max_size` bytes of those files under `payload/libraries/0/binaries`, -# `payload/libraries/1/binaries` and so forth. -# 4. Lastly, it sets the pipeline variables. +# 1. Create `correlation_payload_directory` that contains files from CORE_ROOT, src\coreclr\scripts. +# This directory is the one that is sent to all the helix machines that perform SPMI collections. +# 2. For PMI collections, clone dotnet/jitutils, build it and then copy the `pmi.dll` to +# `correlation_payload_directory` folder. +# 3. For PMI/crossgen2 collections, the `input_directory` directory contains the set of assemblies +# to collect over. This script will partition these folders into equal buckets of approximately +# `max_size` bytes and stores them under the workitem payload directory. Each sub-folder inside +# this directory is sent to an individual helix machine to do SPMI collection on. E.g. for +# `input_directory` to be run on libraries, the parameter is the path to `CORE_ROOT` folder and +# this script will copy `max_size` bytes of those files under +# `payload/collectAssembliesDirectory/libraries/0/binaries`, +# `payload/collectAssembliesDirectory/libraries/1/binaries` and so forth. +# 4. For benchmarks collections, a specialized script is called to set up the benchmarks collection. +# 5. Lastly, it sets the pipeline variables. # # Below are the helix queues it sets depending on the OS/architecture: # | Arch | windows | Linux | macOS | # |-------|-------------------------|--------------------------------------------------------------------------------------------------------------------------------------|----------------| -# | x86 | Windows.10.Amd64.X86.Rt | | - | +# | x86 | Windows.10.Amd64.X86.Rt | - | - | # | x64 | Windows.10.Amd64.X86.Rt | Ubuntu.1804.Amd64 | OSX.1014.Amd64 | # | arm | - | (Ubuntu.1804.Arm32)Ubuntu.1804.Armarch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440 | - | # | arm64 | Windows.10.Arm64 | (Ubuntu.1804.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8-20210531091519-97d8652 | OSX.1100.ARM64 | @@ -39,22 +42,25 @@ from coreclr_arguments import * from jitutil import run_command, copy_directory, copy_files, set_pipeline_variable, ChangeDir, TempDir - # Start of parser object creation. parser = argparse.ArgumentParser(description="description") -parser.add_argument("-source_directory", help="path to source directory") -parser.add_argument("-core_root_directory", help="path to core_root directory") -parser.add_argument("-arch", help="Architecture") -parser.add_argument("-platform", help="OS platform") +parser.add_argument("-collection_type", required=True, help="Type of the SPMI collection to be done (crossgen2, pmi, run)") +parser.add_argument("-collection_name", required=True, help="Name of the SPMI collection to be done (e.g., libraries, libraries_tests, coreclr_tests, benchmarks)") +parser.add_argument("-payload_directory", required=True, help="Path to payload directory to create: subdirectories are created for the correlation payload as well as the per-partition work items") +parser.add_argument("-source_directory", required=True, help="Path to source directory") +parser.add_argument("-core_root_directory", required=True, help="Path to Core_Root directory") +parser.add_argument("-arch", required=True, help="Architecture") +parser.add_argument("-platform", required=True, help="OS platform") parser.add_argument("-mch_file_tag", help="Tag to be used to mch files") -parser.add_argument("-collection_name", help="Name of the SPMI collection to be done (e.g., libraries, tests)") -parser.add_argument("-collection_type", help="Type of the SPMI collection to be done (crossgen, crossgen2, pmi)") -parser.add_argument("-input_directory", help="directory containing assemblies for which superpmi collection to be done") -parser.add_argument("-max_size", help="Max size of each partition in MB") +parser.add_argument("-input_directory", help="Directory containing assemblies which SuperPMI will use for collection (for pmi/crossgen2 collections)") +parser.add_argument("-max_size", help="Max size of each partition in MB (for pmi/crossgen2 collections)") + is_windows = platform.system() == "Windows" +legal_collection_types = [ "crossgen2", "pmi", "run" ] + native_binaries_to_ignore = [ "api-ms-win-core-console-l1-1-0.dll", "api-ms-win-core-datetime-l1-1-0.dll", @@ -187,15 +193,28 @@ def setup_args(args): coreclr_args = CoreclrArguments(args, require_built_core_root=False, require_built_product_dir=False, require_built_test_dir=False, default_build_type="Checked") + coreclr_args.verify(args, + "payload_directory", + lambda unused: True, + "Unable to set payload_directory", + modify_arg=lambda payload_directory: os.path.abspath(payload_directory)) + coreclr_args.verify(args, "source_directory", lambda source_directory: os.path.isdir(source_directory), - "source_directory doesn't exist") + "source_directory doesn't exist", + modify_arg=lambda source_directory: os.path.abspath(source_directory)) + + check_dir = os.path.join(coreclr_args.source_directory, 'src', 'coreclr', 'scripts') + if not os.path.isdir(check_dir): + print("Specified directory {0} doesn't looks like a source directory".format(coreclr_args.source_directory)) + sys.exit(1) coreclr_args.verify(args, "core_root_directory", lambda core_root_directory: os.path.isdir(core_root_directory), - "core_root_directory doesn't exist") + "core_root_directory doesn't exist", + modify_arg=lambda core_root_directory: os.path.abspath(core_root_directory)) coreclr_args.verify(args, "arch", @@ -219,17 +238,18 @@ def setup_args(args): coreclr_args.verify(args, "collection_type", - lambda unused: True, - "Unable to set collection_type") + lambda collection_type: collection_type in legal_collection_types, + "Please specify one of the allowed collection types: " + ' '.join(legal_collection_types)) coreclr_args.verify(args, "input_directory", - lambda input_directory: os.path.isdir(input_directory), - "input_directory doesn't exist") + lambda input_directory: coreclr_args.collection_type not in [ "pmi", "crossgen2" ] or os.path.isdir(input_directory), + "input_directory doesn't exist", + modify_arg=lambda input_directory: None if input_directory is None else os.path.abspath(input_directory)) coreclr_args.verify(args, "max_size", - lambda max_size: max_size > 0, + lambda max_size: coreclr_args.collection_type not in [ "pmi", "crossgen2" ] or max_size > 0, "Please enter valid positive numeric max_size", modify_arg=lambda max_size: int( max_size) * 1000 * 1000 if max_size is not None and max_size.isnumeric() else 0 @@ -390,15 +410,31 @@ def main(main_args): coreclr_args = setup_args(main_args) source_directory = coreclr_args.source_directory - # CorrelationPayload directories - correlation_payload_directory = os.path.join(coreclr_args.source_directory, "payload") + # If the payload directory doesn't already exist (it probably shouldn't) then create it. + if not os.path.isdir(coreclr_args.payload_directory): + os.makedirs(coreclr_args.payload_directory) + + correlation_payload_directory = os.path.join(coreclr_args.payload_directory, 'correlation') + workitem_payload_directory = os.path.join(coreclr_args.payload_directory, 'workitem') + superpmi_src_directory = os.path.join(source_directory, 'src', 'coreclr', 'scripts') + + # Correlation payload directories (sent to every Helix machine). + # Currently, all the Core_Root files, superpmi script files, and pmi.dll go in the same place. superpmi_dst_directory = os.path.join(correlation_payload_directory, "superpmi") + core_root_dst_directory = superpmi_dst_directory + + # Workitem directories + # input_artifacts is only used for pmi/crossgen2 collections. + input_artifacts = "" + arch = coreclr_args.arch platform_name = coreclr_args.platform.lower() helix_source_prefix = "official" creator = "" ci = True + + # Determine the Helix queue name to use when running jobs. if platform_name == "windows": helix_queue = "Windows.10.Arm64" if arch == "arm64" else "Windows.10.Amd64.X86.Rt" elif platform_name == "linux": @@ -411,10 +447,13 @@ def main(main_args): elif platform_name == "osx": helix_queue = "OSX.1100.ARM64" if arch == "arm64" else "OSX.1014.Amd64" - # create superpmi directory + # Copy the superpmi scripts + print('Copying {} -> {}'.format(superpmi_src_directory, superpmi_dst_directory)) copy_directory(superpmi_src_directory, superpmi_dst_directory, verbose_output=True, match_func=lambda path: any(path.endswith(extension) for extension in [".py"])) + # Copy Core_Root + if platform_name == "windows": acceptable_copy = lambda path: any(path.endswith(extension) for extension in [".py", ".dll", ".exe", ".json"]) else: @@ -423,80 +462,78 @@ def main(main_args): # Need to accept files without any extension, which is how executable file's names look. acceptable_copy = lambda path: (os.path.basename(path).find(".") == -1) or any(path.endswith(extension) for extension in acceptable_extensions) - print('Copying {} -> {}'.format(coreclr_args.core_root_directory, superpmi_dst_directory)) - copy_directory(coreclr_args.core_root_directory, superpmi_dst_directory, verbose_output=True, match_func=acceptable_copy) - - # Copy all the test files to CORE_ROOT - # The reason is there are lot of dependencies with *.Tests.dll and to ensure we do not get - # Reflection errors, just copy everything to CORE_ROOT so for all individual partitions, the - # references will be present in CORE_ROOT. - if coreclr_args.collection_name == "libraries_tests": - print('Copying {} -> {}'.format(coreclr_args.input_directory, superpmi_dst_directory)) - - def make_readable(folder_name): - """Make file executable by changing the permission - - Args: - folder_name (string): folder to mark with 744 - """ - if is_windows: - return - - print("Inside make_readable") - run_command(["ls", "-l", folder_name]) - for file_path, dirs, files in os.walk(folder_name, topdown=True): - for d in dirs: - os.chmod(os.path.join(file_path, d), - # read+write+execute for owner - (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) | - # read for group - (stat.S_IRGRP) | - # read for other - (stat.S_IROTH)) - - for f in files: - os.chmod(os.path.join(file_path, f), - # read+write+execute for owner - (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) | - # read for group - (stat.S_IRGRP) | - # read for other - (stat.S_IROTH)) - run_command(["ls", "-l", folder_name]) - - make_readable(coreclr_args.input_directory) - copy_directory(coreclr_args.input_directory, superpmi_dst_directory, verbose_output=True, match_func=acceptable_copy) - - # Workitem directories - workitem_directory = os.path.join(source_directory, "workitem") - input_artifacts = "" + print('Copying {} -> {}'.format(coreclr_args.core_root_directory, core_root_dst_directory)) + copy_directory(coreclr_args.core_root_directory, core_root_dst_directory, verbose_output=True, match_func=acceptable_copy) if coreclr_args.collection_name == "benchmarks": # Setup microbenchmarks - setup_microbenchmark(workitem_directory, arch) + setup_microbenchmark(workitem_payload_directory, arch) else: - # Setup for pmi/crossgen runs + # Setup for pmi/crossgen2 runs - # Clone and build jitutils - try: - with TempDir() as jitutils_directory: - run_command( - ["git", "clone", "--quiet", "--depth", "1", "https://github.com/dotnet/jitutils", jitutils_directory]) - - # Make sure ".dotnet" directory exists, by running the script at least once - dotnet_script_name = "dotnet.cmd" if is_windows else "dotnet.sh" - dotnet_script_path = os.path.join(source_directory, dotnet_script_name) - run_command([dotnet_script_path, "--info"], jitutils_directory) - - # Set dotnet path to run build - os.environ["PATH"] = os.path.join(source_directory, ".dotnet") + os.pathsep + os.environ["PATH"] - build_file = "build.cmd" if is_windows else "build.sh" - run_command([os.path.join(jitutils_directory, build_file), "-p"], jitutils_directory) + # For libraries tests, copy all the test files to the single + # The reason is there are lot of dependencies with *.Tests.dll and to ensure we do not get + # Reflection errors, just copy everything to CORE_ROOT so for all individual partitions, the + # references will be present in CORE_ROOT. + if coreclr_args.collection_name == "libraries_tests": - copy_files(os.path.join(jitutils_directory, "bin"), superpmi_dst_directory, [os.path.join(jitutils_directory, "bin", "pmi.dll")]) - except PermissionError as pe_error: - # Details: https://bugs.python.org/issue26660 - print('Ignoring PermissionError: {0}'.format(pe_error)) + def make_readable(folder_name): + """Make file executable by changing the permission + + Args: + folder_name (string): folder to mark with 744 + """ + if is_windows: + return + + print("Inside make_readable") + run_command(["ls", "-l", folder_name]) + for file_path, dirs, files in os.walk(folder_name, topdown=True): + for d in dirs: + os.chmod(os.path.join(file_path, d), + # read+write+execute for owner + (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) | + # read for group + (stat.S_IRGRP) | + # read for other + (stat.S_IROTH)) + + for f in files: + os.chmod(os.path.join(file_path, f), + # read+write+execute for owner + (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) | + # read for group + (stat.S_IRGRP) | + # read for other + (stat.S_IROTH)) + run_command(["ls", "-l", folder_name]) + + make_readable(coreclr_args.input_directory) + print('Copying {} -> {}'.format(coreclr_args.input_directory, core_root_dst_directory)) + copy_directory(coreclr_args.input_directory, core_root_dst_directory, verbose_output=True, match_func=acceptable_copy) + + # We need the PMI tool if we're doing a PMI collection. We could download a cached copy from Azure DevOps JIT blob + # storage, but instead we clone and build jitutils to build pmi.dll. + if coreclr_args.collection_type == "pmi": + try: + with TempDir() as jitutils_directory: + run_command( + ["git", "clone", "--quiet", "--depth", "1", "https://github.com/dotnet/jitutils", jitutils_directory]) + + # Make sure ".dotnet" directory exists, by running the script at least once + dotnet_script_name = "dotnet.cmd" if is_windows else "dotnet.sh" + dotnet_script_path = os.path.join(source_directory, dotnet_script_name) + run_command([dotnet_script_path, "--info"], jitutils_directory) + + # Set dotnet path to run build + os.environ["PATH"] = os.path.join(source_directory, ".dotnet") + os.pathsep + os.environ["PATH"] + build_file = "build.cmd" if is_windows else "build.sh" + run_command([os.path.join(jitutils_directory, build_file), "-p"], jitutils_directory) + + copy_files(os.path.join(jitutils_directory, "bin"), core_root_dst_directory, [os.path.join(jitutils_directory, "bin", "pmi.dll")]) + except PermissionError as pe_error: + # Details: https://bugs.python.org/issue26660 + print('Ignoring PermissionError: {0}'.format(pe_error)) # NOTE: we can't use the build machine ".dotnet" to run on all platforms. E.g., the Windows x86 build uses a # Windows x64 .dotnet\dotnet.exe that can't load a 32-bit shim. Thus, we always use corerun from Core_Root to invoke crossgen2. @@ -509,9 +546,7 @@ def make_readable(folder_name): # print('Copying {} -> {}'.format(dotnet_src_directory, dotnet_dst_directory)) # copy_directory(dotnet_src_directory, dotnet_dst_directory, verbose_output=False) - # payload - pmiassemblies_directory = os.path.join(workitem_directory, "pmiAssembliesDirectory") - input_artifacts = os.path.join(pmiassemblies_directory, coreclr_args.collection_name) + input_artifacts = os.path.join(workitem_payload_directory, "collectAssembliesDirectory", coreclr_args.collection_name) exclude_directory = ['Core_Root'] if coreclr_args.collection_name == "coreclr_tests" else [] exclude_files = native_binaries_to_ignore if coreclr_args.collection_type == "crossgen2": @@ -531,7 +566,7 @@ def make_readable(folder_name): # Set variables print('Setting pipeline variables:') set_pipeline_variable("CorrelationPayloadDirectory", correlation_payload_directory) - set_pipeline_variable("WorkItemDirectory", workitem_directory) + set_pipeline_variable("WorkItemDirectory", workitem_payload_directory) set_pipeline_variable("InputArtifacts", input_artifacts) set_pipeline_variable("Python", ' '.join(get_python_name())) set_pipeline_variable("Architecture", arch) diff --git a/src/coreclr/tools/superpmi/mcs/commandline.cpp b/src/coreclr/tools/superpmi/mcs/commandline.cpp index d7b29b2c279bb7..86ce6023375d1f 100644 --- a/src/coreclr/tools/superpmi/mcs/commandline.cpp +++ b/src/coreclr/tools/superpmi/mcs/commandline.cpp @@ -121,6 +121,8 @@ void CommandLine::DumpHelp(const char* program) printf(" -strip range inputfile outputfile\n"); printf(" Copy method contexts from one file to another, skipping ranged items.\n"); printf(" inputfile is read and records not in range are written to outputfile.\n"); + printf(" If range is empty (e.g., from an empty .mcl file due to a clean replay),\n"); + printf(" the file is simply copied.\n"); printf(" e.g. -strip 2 a.mc b.mc\n"); printf("\n"); printf(" -toc inputfile\n"); @@ -627,12 +629,6 @@ bool CommandLine::Parse(int argc, char* argv[], /* OUT */ Options* o) DumpHelp(argv[0]); return false; } - if (o->indexCount == 0) - { - LogError("CommandLine::Parse() -strip requires a range."); - DumpHelp(argv[0]); - return false; - } return true; } if (o->actionPrintJITEEVersion) diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp index 3d7b40ff375f09..b25c8aad527a45 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp @@ -7001,8 +7001,15 @@ int MethodContext::dumpMethodIdentityInfoToBuffer(char* buff, int len, bool igno char* obuff = buff; - // Add the Method Signature - int t = sprintf_s(buff, len, "%s -- ", CallUtils::GetMethodFullName(this, pInfo->ftn, pInfo->args, ignoreMethodName)); + // Add the Method Signature. Be careful about potentially huge method signatures; truncate if necessary. + const char* methodFullName = CallUtils::GetMethodFullName(this, pInfo->ftn, pInfo->args, ignoreMethodName); + int t = _snprintf_s(buff, len - METHOD_IDENTITY_INFO_NON_NAME_RESERVE, _TRUNCATE, "%s -- ", methodFullName); + if (t == -1) + { + // We truncated the name string, meaning we wrote exactly `len - METHOD_IDENTITY_INFO_NON_NAME_RESERVE` characters + // (including the terminating null). We advance the buffer pointer by this amount, not including that terminating null. + t = len - METHOD_IDENTITY_INFO_NON_NAME_RESERVE - 1; + } buff += t; len -= t; diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h index 3fc5247ef66c20..b14d67b6b678c9 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h @@ -42,6 +42,7 @@ extern bool g_debugRep; const char* toString(CorInfoType cit); #define METHOD_IDENTITY_INFO_SIZE 0x10000 // We assume that the METHOD_IDENTITY_INFO_SIZE will not exceed 64KB +#define METHOD_IDENTITY_INFO_NON_NAME_RESERVE 0x400 // Reserve 1KB of METHOD_IDENTITY_INFO_SIZE for everything except for the method name. // Special "jit flags" for noting some method context features diff --git a/src/tests/Common/CLRTest.Execute.Bash.targets b/src/tests/Common/CLRTest.Execute.Bash.targets index ac0a86a8699a58..1c20d1e730c816 100644 --- a/src/tests/Common/CLRTest.Execute.Bash.targets +++ b/src/tests/Common/CLRTest.Execute.Bash.targets @@ -53,7 +53,7 @@ WARNING: When setting properties based on their current state (for example: + DependsOnTargets="$(BashScriptSnippetGen);GetIlasmRoundTripBashScript;GetSuperPMICollectionBashScript"> @@ -534,6 +534,7 @@ $(BashCLRTestEnvironmentCompatibilityCheck) $(BashCLRTestArgPrep) $(BashCLRTestExitCodePrep) $(IlasmRoundTripBashScript) +$(SuperPMICollectionBashScript) # Allow precommands to override the ExePath ExePath=$(InputAssemblyName) export TestExclusionListPath=$CORE_ROOT/TestExclusionList.txt diff --git a/src/tests/Common/CLRTest.Execute.Batch.targets b/src/tests/Common/CLRTest.Execute.Batch.targets index 2907e65e394f5a..04c9e017d7e513 100644 --- a/src/tests/Common/CLRTest.Execute.Batch.targets +++ b/src/tests/Common/CLRTest.Execute.Batch.targets @@ -52,7 +52,7 @@ WARNING: When setting properties based on their current state (for example: + DependsOnTargets="$(BatchScriptSnippetGen);GetIlasmRoundTripBatchScript;GetSuperPMICollectionBatchScript"> @@ -434,6 +434,8 @@ $(BatchCLRTestEnvironmentCompatibilityCheck) $(IlasmRoundTripBatchScript) +$(SuperPMICollectionBatchScript) + REM Allow precommands to override the ExePath set ExePath=$(InputAssemblyName) set TestExclusionListPath=%CORE_ROOT%\TestExclusionList.txt diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index f4cfed6594cf9f..df93c54cd7e4d6 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -187,6 +187,72 @@ IF NOT DEFINED DoLink ( + + + + + + + + + + + + + + + + + + + + @@ -44,7 +100,8 @@ RuntimeVariant=$(_RuntimeVariant); BundledNETCoreAppPackageVersion=$(BundledNETCoreAppPackageVersion); HelixRuntimeRid=$(HelixRuntimeRid); - PALTestsDir=$(_PALTestsDir) + PALTestsDir=$(_PALTestsDir); + SuperPmiCollect=$(_SuperPmiCollect) <_PropertiesToPass Condition="'$(TargetOS)' == 'Browser' Or '$(TargetsAndroid)' == 'true'"> @@ -67,7 +124,7 @@ - + @@ -230,6 +287,13 @@ + + + + <_SuperPmiScriptsFiles Include="$(RepoRoot)src\coreclr\scripts\*.py" /> + + + @@ -298,6 +362,8 @@ <_MergedWrapperParentDirectory>$([System.IO.Path]::GetDirectoryName('$(_MergedWrapperDirectory)')) <_MergedWrapperName>%(_MergedWrapperRunScript.FileName) <_MergedWrapperRunScriptRelative Condition="'%(_MergedWrapperRunScript.Identity)' != ''">$([System.IO.Path]::GetRelativePath($(TestBinDir), %(_MergedWrapperRunScript.FullPath))) + + <_MergedWrapperRunScriptPrefix Condition="'$(TestWrapperTargetsWindows)' == 'true'">call <_MergedWrapperOutOfProcessTestMarkers Include="$(_MergedWrapperParentDirectory)/**/*.OutOfProcessTest" /> @@ -327,8 +393,8 @@ - - + + @@ -502,12 +568,26 @@ <_XUnitParallelMode Condition=" '$(LongRunningGCTests)' == 'true' ">none <_XUnitParallelMode Condition=" '$(GcSimulatorTests)' == 'true' ">none -parallel $(_XUnitParallelMode) -nocolor -noshadow -xml testResults.xml + false + + + + + $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults + + + + $(BUILD_SOURCESDIRECTORY)/artifacts/helixresults + + - + @@ -526,10 +606,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + @@ -547,6 +654,27 @@ + + + + + + + + + + + + + + + + + + + + + @(HelixPreCommand) @(HelixPostCommand) @@ -616,12 +744,14 @@ dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) -trait TestGroup=%(TestGroup) $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) + coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).log %(PayloadDirectory) %(MergedTestHelixCommand) $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) + coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).log