From 6d8515a2b6a51811d6908db407413ec2431e0ab9 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 25 Aug 2022 10:21:48 -0700 Subject: [PATCH 01/23] Clean up SuperPMI collection scripts The SuperPMI collection scripts were initially written to do PMI-based collections, and crossgen2 and benchmarks were added later. This change cleans up the scripts to only do what is required for a particular collection type. E.g., don't clone and build jitutils to get pmi.dll for crossgen2 collections, where it is not needed. Additional documentation is added and things are renamed to be more clear, if possible. Also, don't pass PMI-specific arguments to superpmi.py for crossgen2 collections (where they are ignored, but might be confusing to readers). In addition, the superpmi_collect_setup.py script gets more argument validation. This isn't terribly necessary since it's only called in one place in the CI scripts, but it is useful for documentation, and helps when you are calling the script manually as part of testing changes. I added a `-payload_directory` argument that specifies where the correlation and work item payloads should be placed, instead of assuming they should be in "well-known" directories in the source tree. Once again, this is useful for testing. --- eng/pipelines/coreclr/superpmi-collect.yml | 14 +- .../templates/run-superpmi-collect-job.yml | 8 +- src/coreclr/scripts/superpmi-collect.proj | 156 +++++++----- src/coreclr/scripts/superpmi_collect_setup.py | 241 ++++++++++-------- 4 files changed, 246 insertions(+), 173 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 57093bb758a1af..e85727c8c09625 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -46,6 +46,8 @@ jobs: - windows_x64 # superpmi-collect-job that targets macOS/arm64 depends on coreclr binaries produced by the macOS/x64 job +# We don't collect osx-x64 (it's essentially the same as linux-x64). If we did, we'd add OSX_x64 in the +# build-coreclr-and-libraries-job.yml above, and remove this. - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml @@ -69,8 +71,6 @@ jobs: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - # Linux tests are built on the OSX machines. - # - OSX_x64 - OSX_arm64 - Linux_arm - Linux_arm64 @@ -91,8 +91,6 @@ jobs: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - # Linux tests are built on the OSX machines. - # - OSX_x64 - OSX_arm64 - Linux_arm - Linux_arm64 @@ -100,7 +98,6 @@ jobs: - windows_x64 - windows_x86 - windows_arm64 - - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: @@ -114,8 +111,6 @@ jobs: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - # Linux tests are built on the OSX machines. - # - OSX_x64 - OSX_arm64 - Linux_arm - Linux_arm64 @@ -123,7 +118,6 @@ jobs: - windows_x64 - windows_x86 - windows_arm64 - - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: @@ -137,8 +131,6 @@ jobs: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - # Linux tests are built on the OSX machines. - # - OSX_x64 - OSX_arm64 - Linux_arm - Linux_arm64 @@ -159,8 +151,6 @@ jobs: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - # Linux tests are built on the OSX machines. - # - OSX_x64 - OSX_arm64 - Linux_arm - Linux_arm64 diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 5ea9e3f03b1a6e..fee9eb24ff4395 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -73,6 +73,8 @@ jobs: value: '$(Build.SourcesDirectory)\artifacts\spmi_collection\' - name: SpmiLogsLocation value: '$(Build.SourcesDirectory)\artifacts\spmi_logs\' + - name: PayloadLocation + value: '$(Build.SourcesDirectory)\payload' - ${{ if ne(parameters.osGroup, 'windows') }}: - name: PythonScript value: 'python3' @@ -86,6 +88,9 @@ jobs: value: '$(Build.SourcesDirectory)/artifacts/spmi_collection/' - name: SpmiLogsLocation value: '$(Build.SourcesDirectory)/artifacts/spmi_logs/' + - name: PayloadLocation + value: '$(Build.SourcesDirectory)/payload' + - ${{ if eq(parameters.collectionName, 'libraries') }}: - name: InputDirectory value: '$(Core_Root_Dir)' @@ -98,6 +103,7 @@ jobs: - ${{ if eq(parameters.collectionName, 'libraries_tests') }}: - name: InputDirectory value: '$(Build.SourcesDirectory)/artifacts/tests/libraries/$(osGroup).$(archType).$(buildConfigUpper)' + workspace: clean: all pool: @@ -106,7 +112,7 @@ jobs: steps: - ${{ parameters.steps }} - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} # Create required directories for merged mch collection and superpmi logs diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index a2ca69af7f9957..97162a7c79dae8 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -1,30 +1,40 @@ - - - + + + + --> \ @@ -33,62 +43,88 @@ / - + AssembliesPayload - Path that will be sent to helix machine to run collection on + AssembliesDirectoryOnHelix - Path on helix machine itself where superpmi.py will discover the sent assemblies. + --> - - - - - - + - + %HELIX_PYTHONPATH% - $(WorkItemDirectory)\pmiAssembliesDirectory - %HELIX_WORKITEM_PAYLOAD%\binaries %HELIX_CORRELATION_PAYLOAD%\superpmi - - - %HELIX_WORKITEM_PAYLOAD%\performance - %HELIX_WORKITEM_UPLOAD_ROOT% $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults - $(SuperPMIDirectory)\superpmi.py collect --clean -log_level DEBUG --$(CollectionType) -pmi_location $(SuperPMIDirectory)\pmi.dll -pmi_path @(PmiPathDirectories->'$(SuperPMIDirectory)\%(Identity)', ' ') $HELIX_PYTHONPATH - $(WorkItemDirectory)/pmiAssembliesDirectory - $HELIX_WORKITEM_PAYLOAD/binaries $HELIX_CORRELATION_PAYLOAD/superpmi - - - $HELIX_WORKITEM_PAYLOAD/performance - $HELIX_WORKITEM_UPLOAD_ROOT $(BUILD_SOURCESDIRECTORY)/artifacts/helixresults - $(SuperPMIDirectory)/superpmi.py collect --clean -log_level DEBUG --$(CollectionType) -pmi_location $(SuperPMIDirectory)/pmi.dll -pmi_path @(PmiPathDirectories->'$(SuperPMIDirectory)/%(Identity)', ' ') - - $(Python) $(WorkItemCommand) -assemblies $(PmiAssembliesDirectory) -arch $(Architecture) -build_type $(BuildConfig) -core_root $(SuperPMIDirectory) + + + + %HELIX_WORKITEM_PAYLOAD%\binaries + + + $HELIX_WORKITEM_PAYLOAD/binaries + + + + + + %HELIX_WORKITEM_PAYLOAD%\performance + + + $HELIX_WORKITEM_PAYLOAD/performance + + + + + + $(WorkItemDirectory)$(FileSeparatorChar)collectAssembliesDirectory$(FileSeparatorChar)$(CollectionName) + + + + + + + + + + + + -pmi_location $(SuperPMIDirectory)\pmi.dll -pmi_path @(PmiPathDirectories->'$(SuperPMIDirectory)\%(Identity)', ' ') + + + -pmi_location $(SuperPMIDirectory)/pmi.dll -pmi_path @(PmiPathDirectories->'$(SuperPMIDirectory)/%(Identity)', ' ') + + + + + + + + $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi.py collect --clean -log_level DEBUG --$(CollectionType) $(PmiArguments) -assemblies $(AssembliesDirectoryOnHelix) -arch $(Architecture) -build_type $(BuildConfig) -core_root $(SuperPMIDirectory) 2:00 - $(Python) $(SuperPMIDirectory)/superpmi_benchmarks.py -performance_directory $(PerformanceDirectory) -superpmi_directory $(SuperPMIDirectory) -core_root $(SuperPMIDirectory) -arch $(Architecture) + $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_benchmarks.py -performance_directory $(PerformanceDirectory) -superpmi_directory $(SuperPMIDirectory) -core_root $(SuperPMIDirectory) -arch $(Architecture) 3:00 @@ -101,12 +137,12 @@ + - + - + + 30 - + @@ -167,7 +209,7 @@ $(CollectionName).$(CollectionType).%(HelixWorkItem.PartitionId).$(MchFileTag) - $(PmiAssembliesPayload)$(FileSeparatorChar)$(CollectionName)$(FileSeparatorChar)%(HelixWorkItem.PmiAssemblies) + $(AssembliesPayload)$(FileSeparatorChar)%(HelixWorkItem.CollectAssemblies) $(WorkItemCommand) -output_mch_path $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).mch -log_file $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).log $(WorkItemTimeout) %(OutputFileName).mch;%(OutputFileName).mch.mct;%(OutputFileName).log diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 497ff9f64d57bd..815041c97685d0 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -9,22 +9,25 @@ # # Script to setup directory structure required to perform SuperPMI collection in CI. # It does the following steps: -# 1. It creates `correlation_payload_directory` that contains files from CORE_ROOT, src\coreclr\scripts. -# This directory is the one that is sent to all the helix machines that performs SPMI collection. -# 2. It clones dotnet/jitutils, builds it and then copies the `pmi.dll` to `correlation_payload_directory` folder. -# This file is needed to do pmi SPMI runs. -# 3. The script takes `input_artifacts` parameter which contains managed .dlls and .exes on -# which SPMI needs to be run. This script will partition these folders into equal buckets of approximately `max_size` -# bytes and stores them under `payload` directory. Each sub-folder inside `payload` directory is sent to individual -# helix machine to do SPMI collection on. E.g. for `input_artifacts` to be run on libraries, the parameter would be path to -# `CORE_ROOT` folder and this script will copy `max_size` bytes of those files under `payload/libraries/0/binaries`, -# `payload/libraries/1/binaries` and so forth. -# 4. Lastly, it sets the pipeline variables. +# 1. Create `correlation_payload_directory` that contains files from CORE_ROOT, src\coreclr\scripts. +# This directory is the one that is sent to all the helix machines that perform SPMI collections. +# 2. For PMI collections, clone dotnet/jitutils, build it and then copy the `pmi.dll` to +# `correlation_payload_directory` folder. +# 3. For PMI/crossgen2 collections, the `input_directory` directory contains the set of assemblies +# to collect over. This script will partition these folders into equal buckets of approximately +# `max_size` bytes and stores them under the workitem payload directory. Each sub-folder inside +# this directory is sent to an individual helix machine to do SPMI collection on. E.g. for +# `input_directory` to be run on libraries, the parameter is the path to `CORE_ROOT` folder and +# this script will copy `max_size` bytes of those files under +# `payload/collectAssembliesDirectory/libraries/0/binaries`, +# `payload/collectAssembliesDirectory/libraries/1/binaries` and so forth. +# 4. For benchmarks collections, a specialized script is called to set up the benchmarks collection. +# 5. Lastly, it sets the pipeline variables. # # Below are the helix queues it sets depending on the OS/architecture: # | Arch | windows | Linux | macOS | # |-------|-------------------------|--------------------------------------------------------------------------------------------------------------------------------------|----------------| -# | x86 | Windows.10.Amd64.X86.Rt | | - | +# | x86 | Windows.10.Amd64.X86.Rt | - | - | # | x64 | Windows.10.Amd64.X86.Rt | Ubuntu.1804.Amd64 | OSX.1014.Amd64 | # | arm | - | (Ubuntu.1804.Arm32)Ubuntu.1804.Armarch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440 | - | # | arm64 | Windows.10.Arm64 | (Ubuntu.1804.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8-20210531091519-97d8652 | OSX.1100.ARM64 | @@ -39,22 +42,25 @@ from coreclr_arguments import * from jitutil import run_command, copy_directory, copy_files, set_pipeline_variable, ChangeDir, TempDir - # Start of parser object creation. parser = argparse.ArgumentParser(description="description") -parser.add_argument("-source_directory", help="path to source directory") -parser.add_argument("-core_root_directory", help="path to core_root directory") -parser.add_argument("-arch", help="Architecture") -parser.add_argument("-platform", help="OS platform") +parser.add_argument("-collection_type", required=True, help="Type of the SPMI collection to be done (crossgen2, pmi, run)") +parser.add_argument("-collection_name", required=True, help="Name of the SPMI collection to be done (e.g., libraries, libraries_tests, coreclr_tests, benchmarks)") +parser.add_argument("-payload_directory", required=True, help="Path to payload directory to create: subdirectories are created for the correlation payload as well as the per-partition work items") +parser.add_argument("-source_directory", required=True, help="Path to source directory") +parser.add_argument("-core_root_directory", required=True, help="Path to Core_Root directory") +parser.add_argument("-arch", required=True, help="Architecture") +parser.add_argument("-platform", required=True, help="OS platform") parser.add_argument("-mch_file_tag", help="Tag to be used to mch files") -parser.add_argument("-collection_name", help="Name of the SPMI collection to be done (e.g., libraries, tests)") -parser.add_argument("-collection_type", help="Type of the SPMI collection to be done (crossgen, crossgen2, pmi)") -parser.add_argument("-input_directory", help="directory containing assemblies for which superpmi collection to be done") -parser.add_argument("-max_size", help="Max size of each partition in MB") +parser.add_argument("-input_directory", help="Directory containing assemblies which SuperPMI will use for collection (for pmi/crossgen2 collections)") +parser.add_argument("-max_size", help="Max size of each partition in MB (for pmi/crossgen2 collections)") + is_windows = platform.system() == "Windows" +legal_collection_types = [ "crossgen2", "pmi", "run" ] + native_binaries_to_ignore = [ "api-ms-win-core-console-l1-1-0.dll", "api-ms-win-core-datetime-l1-1-0.dll", @@ -187,15 +193,28 @@ def setup_args(args): coreclr_args = CoreclrArguments(args, require_built_core_root=False, require_built_product_dir=False, require_built_test_dir=False, default_build_type="Checked") + coreclr_args.verify(args, + "payload_directory", + lambda unused: True, + "Unable to set payload_directory", + modify_arg=lambda payload_directory: os.path.abspath(payload_directory)) + coreclr_args.verify(args, "source_directory", lambda source_directory: os.path.isdir(source_directory), - "source_directory doesn't exist") + "source_directory doesn't exist", + modify_arg=lambda source_directory: os.path.abspath(source_directory)) + + check_dir = os.path.join(coreclr_args.source_directory, 'src', 'coreclr', 'scripts') + if not os.path.isdir(check_dir): + print("Specified directory {0} doesn't looks like a source directory".format(coreclr_args.source_directory)) + sys.exit(1) coreclr_args.verify(args, "core_root_directory", lambda core_root_directory: os.path.isdir(core_root_directory), - "core_root_directory doesn't exist") + "core_root_directory doesn't exist", + modify_arg=lambda core_root_directory: os.path.abspath(core_root_directory)) coreclr_args.verify(args, "arch", @@ -219,17 +238,18 @@ def setup_args(args): coreclr_args.verify(args, "collection_type", - lambda unused: True, - "Unable to set collection_type") + lambda collection_type: collection_type in legal_collection_types, + "Please specify one of the allowed collection types: " + ' '.join(legal_collection_types)) coreclr_args.verify(args, "input_directory", - lambda input_directory: os.path.isdir(input_directory), - "input_directory doesn't exist") + lambda input_directory: coreclr_args.collection_type not in [ "pmi", "crossgen2" ] or os.path.isdir(input_directory), + "input_directory doesn't exist", + modify_arg=lambda input_directory: None if input_directory is None else os.path.abspath(input_directory)) coreclr_args.verify(args, "max_size", - lambda max_size: max_size > 0, + lambda max_size: coreclr_args.collection_type not in [ "pmi", "crossgen2" ] or max_size > 0, "Please enter valid positive numeric max_size", modify_arg=lambda max_size: int( max_size) * 1000 * 1000 if max_size is not None and max_size.isnumeric() else 0 @@ -390,15 +410,31 @@ def main(main_args): coreclr_args = setup_args(main_args) source_directory = coreclr_args.source_directory - # CorrelationPayload directories - correlation_payload_directory = os.path.join(coreclr_args.source_directory, "payload") + # If the payload directory doesn't already exist (it probably shouldn't) then create it. + if not os.path.isdir(coreclr_args.payload_directory): + os.makedirs(coreclr_args.payload_directory) + + correlation_payload_directory = os.path.join(coreclr_args.payload_directory, 'correlation') + workitem_payload_directory = os.path.join(coreclr_args.payload_directory, 'workitem') + superpmi_src_directory = os.path.join(source_directory, 'src', 'coreclr', 'scripts') + + # Correlation payload directories (sent to every Helix machine). + # Currently, all the Core_Root files, superpmi script files, and pmi.dll go in the same place. superpmi_dst_directory = os.path.join(correlation_payload_directory, "superpmi") + core_root_dst_directory = superpmi_dst_directory + + # Workitem directories + # input_artifacts is only used for pmi/crossgen2 collections. + input_artifacts = "" + arch = coreclr_args.arch platform_name = coreclr_args.platform.lower() helix_source_prefix = "official" creator = "" ci = True + + # Determine the Helix queue name to use when running jobs. if platform_name == "windows": helix_queue = "Windows.10.Arm64" if arch == "arm64" else "Windows.10.Amd64.X86.Rt" elif platform_name == "linux": @@ -411,10 +447,13 @@ def main(main_args): elif platform_name == "osx": helix_queue = "OSX.1100.ARM64" if arch == "arm64" else "OSX.1014.Amd64" - # create superpmi directory + # Copy the superpmi scripts + print('Copying {} -> {}'.format(superpmi_src_directory, superpmi_dst_directory)) copy_directory(superpmi_src_directory, superpmi_dst_directory, verbose_output=True, match_func=lambda path: any(path.endswith(extension) for extension in [".py"])) + # Copy Core_Root + if platform_name == "windows": acceptable_copy = lambda path: any(path.endswith(extension) for extension in [".py", ".dll", ".exe", ".json"]) else: @@ -423,80 +462,78 @@ def main(main_args): # Need to accept files without any extension, which is how executable file's names look. acceptable_copy = lambda path: (os.path.basename(path).find(".") == -1) or any(path.endswith(extension) for extension in acceptable_extensions) - print('Copying {} -> {}'.format(coreclr_args.core_root_directory, superpmi_dst_directory)) - copy_directory(coreclr_args.core_root_directory, superpmi_dst_directory, verbose_output=True, match_func=acceptable_copy) - - # Copy all the test files to CORE_ROOT - # The reason is there are lot of dependencies with *.Tests.dll and to ensure we do not get - # Reflection errors, just copy everything to CORE_ROOT so for all individual partitions, the - # references will be present in CORE_ROOT. - if coreclr_args.collection_name == "libraries_tests": - print('Copying {} -> {}'.format(coreclr_args.input_directory, superpmi_dst_directory)) - - def make_readable(folder_name): - """Make file executable by changing the permission - - Args: - folder_name (string): folder to mark with 744 - """ - if is_windows: - return - - print("Inside make_readable") - run_command(["ls", "-l", folder_name]) - for file_path, dirs, files in os.walk(folder_name, topdown=True): - for d in dirs: - os.chmod(os.path.join(file_path, d), - # read+write+execute for owner - (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) | - # read for group - (stat.S_IRGRP) | - # read for other - (stat.S_IROTH)) - - for f in files: - os.chmod(os.path.join(file_path, f), - # read+write+execute for owner - (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) | - # read for group - (stat.S_IRGRP) | - # read for other - (stat.S_IROTH)) - run_command(["ls", "-l", folder_name]) - - make_readable(coreclr_args.input_directory) - copy_directory(coreclr_args.input_directory, superpmi_dst_directory, verbose_output=True, match_func=acceptable_copy) - - # Workitem directories - workitem_directory = os.path.join(source_directory, "workitem") - input_artifacts = "" + print('Copying {} -> {}'.format(coreclr_args.core_root_directory, core_root_dst_directory)) + copy_directory(coreclr_args.core_root_directory, core_root_dst_directory, verbose_output=True, match_func=acceptable_copy) if coreclr_args.collection_name == "benchmarks": # Setup microbenchmarks - setup_microbenchmark(workitem_directory, arch) + setup_microbenchmark(workitem_payload_directory, arch) else: - # Setup for pmi/crossgen runs + # Setup for pmi/crossgen2 runs - # Clone and build jitutils - try: - with TempDir() as jitutils_directory: - run_command( - ["git", "clone", "--quiet", "--depth", "1", "https://github.com/dotnet/jitutils", jitutils_directory]) - - # Make sure ".dotnet" directory exists, by running the script at least once - dotnet_script_name = "dotnet.cmd" if is_windows else "dotnet.sh" - dotnet_script_path = os.path.join(source_directory, dotnet_script_name) - run_command([dotnet_script_path, "--info"], jitutils_directory) - - # Set dotnet path to run build - os.environ["PATH"] = os.path.join(source_directory, ".dotnet") + os.pathsep + os.environ["PATH"] - build_file = "build.cmd" if is_windows else "build.sh" - run_command([os.path.join(jitutils_directory, build_file), "-p"], jitutils_directory) + # For libraries tests, copy all the test files to the single + # The reason is there are lot of dependencies with *.Tests.dll and to ensure we do not get + # Reflection errors, just copy everything to CORE_ROOT so for all individual partitions, the + # references will be present in CORE_ROOT. + if coreclr_args.collection_name == "libraries_tests": - copy_files(os.path.join(jitutils_directory, "bin"), superpmi_dst_directory, [os.path.join(jitutils_directory, "bin", "pmi.dll")]) - except PermissionError as pe_error: - # Details: https://bugs.python.org/issue26660 - print('Ignoring PermissionError: {0}'.format(pe_error)) + def make_readable(folder_name): + """Make file executable by changing the permission + + Args: + folder_name (string): folder to mark with 744 + """ + if is_windows: + return + + print("Inside make_readable") + run_command(["ls", "-l", folder_name]) + for file_path, dirs, files in os.walk(folder_name, topdown=True): + for d in dirs: + os.chmod(os.path.join(file_path, d), + # read+write+execute for owner + (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) | + # read for group + (stat.S_IRGRP) | + # read for other + (stat.S_IROTH)) + + for f in files: + os.chmod(os.path.join(file_path, f), + # read+write+execute for owner + (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) | + # read for group + (stat.S_IRGRP) | + # read for other + (stat.S_IROTH)) + run_command(["ls", "-l", folder_name]) + + make_readable(coreclr_args.input_directory) + print('Copying {} -> {}'.format(coreclr_args.input_directory, core_root_dst_directory)) + copy_directory(coreclr_args.input_directory, core_root_dst_directory, verbose_output=True, match_func=acceptable_copy) + + # We need the PMI tool if we're doing a PMI collection. We could download a cached copy from Azure DevOps JIT blob + # storage, but instead we clone and build jitutils to build pmi.dll. + if coreclr_args.collection_type == "pmi": + try: + with TempDir() as jitutils_directory: + run_command( + ["git", "clone", "--quiet", "--depth", "1", "https://github.com/dotnet/jitutils", jitutils_directory]) + + # Make sure ".dotnet" directory exists, by running the script at least once + dotnet_script_name = "dotnet.cmd" if is_windows else "dotnet.sh" + dotnet_script_path = os.path.join(source_directory, dotnet_script_name) + run_command([dotnet_script_path, "--info"], jitutils_directory) + + # Set dotnet path to run build + os.environ["PATH"] = os.path.join(source_directory, ".dotnet") + os.pathsep + os.environ["PATH"] + build_file = "build.cmd" if is_windows else "build.sh" + run_command([os.path.join(jitutils_directory, build_file), "-p"], jitutils_directory) + + copy_files(os.path.join(jitutils_directory, "bin"), core_root_dst_directory, [os.path.join(jitutils_directory, "bin", "pmi.dll")]) + except PermissionError as pe_error: + # Details: https://bugs.python.org/issue26660 + print('Ignoring PermissionError: {0}'.format(pe_error)) # NOTE: we can't use the build machine ".dotnet" to run on all platforms. E.g., the Windows x86 build uses a # Windows x64 .dotnet\dotnet.exe that can't load a 32-bit shim. Thus, we always use corerun from Core_Root to invoke crossgen2. @@ -509,9 +546,7 @@ def make_readable(folder_name): # print('Copying {} -> {}'.format(dotnet_src_directory, dotnet_dst_directory)) # copy_directory(dotnet_src_directory, dotnet_dst_directory, verbose_output=False) - # payload - pmiassemblies_directory = os.path.join(workitem_directory, "pmiAssembliesDirectory") - input_artifacts = os.path.join(pmiassemblies_directory, coreclr_args.collection_name) + input_artifacts = os.path.join(workitem_payload_directory, "collectAssembliesDirectory", coreclr_args.collection_name) exclude_directory = ['Core_Root'] if coreclr_args.collection_name == "coreclr_tests" else [] exclude_files = native_binaries_to_ignore if coreclr_args.collection_type == "crossgen2": @@ -531,7 +566,7 @@ def make_readable(folder_name): # Set variables print('Setting pipeline variables:') set_pipeline_variable("CorrelationPayloadDirectory", correlation_payload_directory) - set_pipeline_variable("WorkItemDirectory", workitem_directory) + set_pipeline_variable("WorkItemDirectory", workitem_payload_directory) set_pipeline_variable("InputArtifacts", input_artifacts) set_pipeline_variable("Python", ' '.join(get_python_name())) set_pipeline_variable("Architecture", arch) From 49f2576563eef4fd67651ee4bcada38555adfbaf Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Fri, 26 Aug 2022 17:26:28 -0700 Subject: [PATCH 02/23] Enable SuperPMI collection of CoreCLR tests Instead of using our existing SuperPMI collection scripts which does test partitioning for PMI and crossgen2 collections, we use the existing (and very complex) test partitioning and running scripting, and add before steps to enable SuperPMI collection and after steps to process the results. The process is as follows: 1. The top-level superpmi-collect.yml file has a new task, which invokes the test run scripting /eng/pipelines/common/templates/runtimes/run-test-job.yml with a new argument: `SuperPmiCollect: true` 2. run-test-job.yml then adds a new dependency and various variable definitions, and invokes the send-to-helix-step.yml file with the new SuperPmiCollect variable. It also adds a number of post-Helix steps to (a) merge the MCH files from all the Helix jobs, (b) upload the merged MCH result to Azure Storage, (c) upload SuperPMI log files to artifacts. 3. The Helix invocation ends up at helixpublishwitharcade.proj, which runs a set of tests. It adds pre-commands to set environment variables to trigger SuperPMI collection and post-commands to merge all the generated MC files to a single MCH file. The environment variable `spmi_enable_collection` is set to trigger the collection. 4. The batch/bash wrapper scripts key on this variable and set up the SuperPMI collection variables. The idea is to set the SuperPMI collection variables as late as possible, namely, immediately before corerun.exe is invoked on the test. Specifically, we don't set the variables earlier, before dotnet.exe and xunit are invoked. Note: this collection process doesn't currently handle the new "merged" tests. --- .../templates/runtimes/run-test-job.yml | 107 ++++++++++ .../templates/runtimes/send-to-helix-step.yml | 2 + eng/pipelines/coreclr/superpmi-collect.yml | 196 ++++++++++-------- src/coreclr/scripts/superpmi-collect.proj | 2 +- src/coreclr/scripts/superpmi.py | 6 +- .../tools/superpmi/mcs/commandline.cpp | 8 +- src/tests/Common/CLRTest.Execute.Bash.targets | 3 +- .../Common/CLRTest.Execute.Batch.targets | 4 +- src/tests/Common/CLRTest.Jit.targets | 63 ++++++ src/tests/Common/helixpublishwitharcade.proj | 59 +++++- 10 files changed, 348 insertions(+), 102 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 4bc3e519aec457..5b405b16d9a7b6 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -24,6 +24,7 @@ parameters: shouldContinueOnError: false dependsOn: [] dependOnEvaluatePaths: false + SuperPmiCollect: false ### Test run job @@ -76,6 +77,10 @@ jobs: - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + # SuperPMI collection needs to run mcs.exe on the AzDO machine. Assume that's an x64 machine, and download an x64 product build if needed. + - ${{ if eq(parameters.SuperPmiCollect, true) }}: + - ${{ if ne(parameters.archType, 'x64') }}: + - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', '', parameters.osGroup, parameters.osSubgroup, 'x64', parameters.buildConfig) }} # Compute job name from template parameters ${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: @@ -147,6 +152,35 @@ jobs: - name: testTreeFilterArg value: '-tree:GC/Scenarios/GCSimulator' + # Variables used for SuperPMI collection + - ${{ if eq(parameters.SuperPmiCollect, true) }}: + - name: CollectionType + value: 'run' + - name: CollectionName + value: 'coreclr_tests' + - ${{ if eq(parameters.osGroup, 'windows') }}: + - name: PythonScript + value: 'py -3' + - name: PipScript + value: 'py -3 -m pip' + - name: MchFilesLocation + value: '$(Build.SourcesDirectory)\artifacts\helixresults\' + - name: MergedMchFileLocation + value: '$(Build.SourcesDirectory)\artifacts\spmi_collection\' + - name: SpmiLogsLocation + value: '$(Build.SourcesDirectory)\artifacts\spmi_logs\' + - ${{ if ne(parameters.osGroup, 'windows') }}: + - name: PythonScript + value: 'python3' + - name: PipScript + value: 'pip3' + - name: MchFilesLocation + value: '$(Build.SourcesDirectory)/artifacts/helixresults/' + - name: MergedMchFileLocation + value: '$(Build.SourcesDirectory)/artifacts/spmi_collection/' + - name: SpmiLogsLocation + value: '$(Build.SourcesDirectory)/artifacts/spmi_logs/' + # Set job timeouts # # "timeoutPerTestCollectionInMinutes" is the time needed for the "biggest" xUnit test collection to complete. @@ -298,6 +332,17 @@ jobs: displayName: 'native test artifacts' + # SuperPMI collection: Download x64 coreclr if running on non-x64 configuration (needed for mcs.exe) + - ${{ if eq(parameters.SuperPmiCollect, true) }}: + - ${{ if ne(parameters.archType, 'x64') }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper)' + artifactFileName: 'CoreCLRProduct___$(osGroup)$(osSubgroup)_x64_$(buildConfig)$(archiveExtension)' + artifactName: 'CoreCLRProduct___$(osGroup)$(osSubgroup)_x64_$(buildConfig)' + displayName: 'Coreclr product build (x64)' + + # Publish native test components to test output folder. Sadly we cannot do this # during product build (so that we could zip up the files in their final test location # and directly unzip them there after download). Unfortunately the logic to copy @@ -353,6 +398,7 @@ jobs: runtimeFlavor: ${{ parameters.runtimeFlavor }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} runtimeVariant: ${{ parameters.runtimeVariant }} + SuperPmiCollect: ${{ parameters.SuperPmiCollect }} ${{ if eq(variables['System.TeamProject'], 'public') }}: creator: $(Build.DefinitionName) @@ -603,3 +649,64 @@ jobs: artifactName: '${{ parameters.runtimeFlavor }}_${{ parameters.runtimeVariant }}_$(LogNamePrefix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.testGroup }}' continueOnError: true condition: always() + + # + # Finalize SuperPMI collection: (1) merge all MCH files generated by all Helix jobs, (2) upload MCH file to Azure Storage, (3) upload log files + # + + - ${{ if eq(parameters.SuperPmiCollect, true) }}: + + # Create required directories for merged mch collection and superpmi logs + - ${{ if ne(parameters.osGroup, 'windows') }}: + - script: | + mkdir -p $(MergedMchFileLocation) + mkdir -p $(SpmiLogsLocation) + displayName: Create SuperPMI directories + - ${{ if eq(parameters.osGroup, 'windows') }}: + - script: | + mkdir $(MergedMchFileLocation) + mkdir $(SpmiLogsLocation) + displayName: Create SuperPMI directories + + # Always run merge step even if collection of some partition fails so we can store collection of the partitions that succeeded. + # If all the partitions fail, merge-mch would fail and we won't run future steps like uploading superpmi collection. + # This depends on mcs.exe (from artifacts Core_Root or product binaries or PATH). + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch + displayName: ${{ format('Merge {0}-{1} SuperPMI collections', CollectionName, CollectionType) }} + condition: always() + + # If merge step above fails, then skip "Upload as artifact" and "Upload to Azure storage" + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(MergedMchFileLocation) + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: 'SuperPMI_Collection_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + displayName: ${{ format('Upload artifacts SuperPMI {0}-{1} collection', CollectionName, CollectionType) }} + + # Ensure the Python azure-storage-blob package is installed before doing the upload. + - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall + displayName: Upgrade Pip to latest and install azure-storage-blob Python package + + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch $(archType) -build_type $(buildConfig) -mch_files $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper) + displayName: ${{ format('Upload SuperPMI {0}-{1} collection to Azure Storage', CollectionName, CollectionType) }} + env: + CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline + + # Always upload the available logs for diagnostics + - task: CopyFiles@2 + displayName: Copying superpmi.log of all partitions + inputs: + sourceFolder: '$(MchFilesLocation)' + contents: '**/$(CollectionName).$(CollectionType)*.log' + targetFolder: '$(SpmiLogsLocation)' + condition: always() + + - task: PublishPipelineArtifact@1 + displayName: Publish SuperPMI logs + inputs: + targetPath: $(SpmiLogsLocation) + artifactName: 'SuperPMI_Logs_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + condition: always() \ No newline at end of file diff --git a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml index 0a0242529eb9d5..dfad8c58d880ae 100644 --- a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml +++ b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml @@ -26,6 +26,7 @@ parameters: runtimeFlavor: 'CoreCLR' runtimeVariant: '' shouldContinueOnError: false + SuperPmiCollect: '' steps: @@ -57,6 +58,7 @@ steps: _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }} RuntimeFlavor: ${{ parameters.runtimeFlavor }} _RuntimeVariant: ${{ parameters.runtimeVariant }} + _SuperPmiCollect: ${{ parameters.SuperPmiCollect }} ${{ if eq(parameters.publishTestResults, 'true') }}: SYSTEM_ACCESSTOKEN: $(System.AccessToken) # TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index e85727c8c09625..35c0ee681e2135 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -66,102 +66,122 @@ jobs: jobParameters: testGroup: outerloop -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - OSX_arm64 - - Linux_arm - - Linux_arm64 - - Linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: pmi - collectionName: libraries +############################################################### disable the following to test coreclr test run collection +# - template: /eng/pipelines/common/platform-matrix.yml +# parameters: +# jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml +# buildConfig: checked +# platforms: +# - OSX_arm64 +# - Linux_arm +# - Linux_arm64 +# - Linux_x64 +# - windows_x64 +# - windows_x86 +# - windows_arm64 +# helixQueueGroup: ci +# helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml +# jobParameters: +# testGroup: outerloop +# liveLibrariesBuildConfig: Release +# collectionType: pmi +# collectionName: libraries +# +# - template: /eng/pipelines/common/platform-matrix.yml +# parameters: +# jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml +# buildConfig: checked +# platforms: +# - OSX_arm64 +# - Linux_arm +# - Linux_arm64 +# - Linux_x64 +# - windows_x64 +# - windows_x86 +# - windows_arm64 +# helixQueueGroup: ci +# helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml +# jobParameters: +# testGroup: outerloop +# liveLibrariesBuildConfig: Release +# collectionType: pmi +# collectionName: coreclr_tests +# +# - template: /eng/pipelines/common/platform-matrix.yml +# parameters: +# jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml +# buildConfig: checked +# platforms: +# - OSX_arm64 +# - Linux_arm +# - Linux_arm64 +# - Linux_x64 +# - windows_x64 +# - windows_x86 +# - windows_arm64 +# helixQueueGroup: ci +# helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml +# jobParameters: +# testGroup: outerloop +# liveLibrariesBuildConfig: Release +# collectionType: pmi +# collectionName: libraries_tests +# +# - template: /eng/pipelines/common/platform-matrix.yml +# parameters: +# jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml +# buildConfig: checked +# platforms: +# - OSX_arm64 +# - Linux_arm +# - Linux_arm64 +# - Linux_x64 +# - windows_x64 +# - windows_x86 +# - windows_arm64 +# helixQueueGroup: ci +# helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml +# jobParameters: +# testGroup: outerloop +# liveLibrariesBuildConfig: Release +# collectionType: crossgen2 +# collectionName: libraries +# +# - template: /eng/pipelines/common/platform-matrix.yml +# parameters: +# jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml +# buildConfig: checked +# platforms: +# - OSX_arm64 +# - Linux_arm +# - Linux_arm64 +# - Linux_x64 +# - windows_x64 +# - windows_x86 +# - windows_arm64 +# helixQueueGroup: ci +# helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml +# jobParameters: +# testGroup: outerloop +# liveLibrariesBuildConfig: Release +# collectionType: run +# collectionName: benchmarks +# +# Collection of coreclr test run +# - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml buildConfig: checked + platformGroup: all platforms: + # It is too early to include OSX_arm64 in platform group all + # Adding it here will enable it to also run this test - OSX_arm64 - - Linux_arm - - Linux_arm64 - - Linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: pmi - collectionName: coreclr_tests - -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - OSX_arm64 - - Linux_arm - - Linux_arm64 - - Linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: pmi - collectionName: libraries_tests - -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - OSX_arm64 - - Linux_arm - - Linux_arm64 - - Linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: crossgen2 - collectionName: libraries - -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - OSX_arm64 - - Linux_arm - - Linux_arm64 - - Linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testGroup: outerloop liveLibrariesBuildConfig: Release - collectionType: run - collectionName: benchmarks + SuperPmiCollect: true \ No newline at end of file diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index 97162a7c79dae8..85979cdc265afb 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -20,7 +20,7 @@ c:\bugs\spmicollect14\payload\workitem\collectAssembliesDirectory\libraries x64 Checked - x64.checked + windows.x64.checked libraries crossgen2 Windows_NT diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py index b33983133b0aa7..fff1d9cbfb6908 100644 --- a/src/coreclr/scripts/superpmi.py +++ b/src/coreclr/scripts/superpmi.py @@ -3543,8 +3543,8 @@ def verify_base_diff_args(): lambda unused: True, "Unable to set pmi_path") - if (args.collection_command is None) and (args.pmi is False) and (args.crossgen2 is False): - print("Either a collection command or `--pmi` or `--crossgen2` must be specified") + if (args.collection_command is None) and (args.pmi is False) and (args.crossgen2 is False) and not coreclr_args.skip_collection_step: + print("Either a collection command or `--pmi` or `--crossgen2` or `--skip_collection_step` must be specified") sys.exit(1) if (args.collection_command is not None) and (len(args.assemblies) > 0): @@ -3565,7 +3565,7 @@ def verify_base_diff_args(): if args.pmi_location is not None: logging.warning("Warning: -pmi_location is set but --pmi is not.") - if args.collection_command is None and args.merge_mch_files is not True: + if args.collection_command is None and args.merge_mch_files is not True and not coreclr_args.skip_collection_step: assert args.collection_args is None assert (args.pmi is True) or (args.crossgen2 is True) assert len(args.assemblies) > 0 diff --git a/src/coreclr/tools/superpmi/mcs/commandline.cpp b/src/coreclr/tools/superpmi/mcs/commandline.cpp index d7b29b2c279bb7..86ce6023375d1f 100644 --- a/src/coreclr/tools/superpmi/mcs/commandline.cpp +++ b/src/coreclr/tools/superpmi/mcs/commandline.cpp @@ -121,6 +121,8 @@ void CommandLine::DumpHelp(const char* program) printf(" -strip range inputfile outputfile\n"); printf(" Copy method contexts from one file to another, skipping ranged items.\n"); printf(" inputfile is read and records not in range are written to outputfile.\n"); + printf(" If range is empty (e.g., from an empty .mcl file due to a clean replay),\n"); + printf(" the file is simply copied.\n"); printf(" e.g. -strip 2 a.mc b.mc\n"); printf("\n"); printf(" -toc inputfile\n"); @@ -627,12 +629,6 @@ bool CommandLine::Parse(int argc, char* argv[], /* OUT */ Options* o) DumpHelp(argv[0]); return false; } - if (o->indexCount == 0) - { - LogError("CommandLine::Parse() -strip requires a range."); - DumpHelp(argv[0]); - return false; - } return true; } if (o->actionPrintJITEEVersion) diff --git a/src/tests/Common/CLRTest.Execute.Bash.targets b/src/tests/Common/CLRTest.Execute.Bash.targets index ac0a86a8699a58..1c20d1e730c816 100644 --- a/src/tests/Common/CLRTest.Execute.Bash.targets +++ b/src/tests/Common/CLRTest.Execute.Bash.targets @@ -53,7 +53,7 @@ WARNING: When setting properties based on their current state (for example: + DependsOnTargets="$(BashScriptSnippetGen);GetIlasmRoundTripBashScript;GetSuperPMICollectionBashScript"> @@ -534,6 +534,7 @@ $(BashCLRTestEnvironmentCompatibilityCheck) $(BashCLRTestArgPrep) $(BashCLRTestExitCodePrep) $(IlasmRoundTripBashScript) +$(SuperPMICollectionBashScript) # Allow precommands to override the ExePath ExePath=$(InputAssemblyName) export TestExclusionListPath=$CORE_ROOT/TestExclusionList.txt diff --git a/src/tests/Common/CLRTest.Execute.Batch.targets b/src/tests/Common/CLRTest.Execute.Batch.targets index 2907e65e394f5a..04c9e017d7e513 100644 --- a/src/tests/Common/CLRTest.Execute.Batch.targets +++ b/src/tests/Common/CLRTest.Execute.Batch.targets @@ -52,7 +52,7 @@ WARNING: When setting properties based on their current state (for example: + DependsOnTargets="$(BatchScriptSnippetGen);GetIlasmRoundTripBatchScript;GetSuperPMICollectionBatchScript"> @@ -434,6 +434,8 @@ $(BatchCLRTestEnvironmentCompatibilityCheck) $(IlasmRoundTripBatchScript) +$(SuperPMICollectionBatchScript) + REM Allow precommands to override the ExePath set ExePath=$(InputAssemblyName) set TestExclusionListPath=%CORE_ROOT%\TestExclusionList.txt diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index f4cfed6594cf9f..cc7af64aad395c 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -187,6 +187,69 @@ IF NOT DEFINED DoLink ( + + + + + + + + + + + + + + + + + + <_PropertiesToPass Condition="'$(TargetOS)' == 'Browser' Or '$(TargetsAndroid)' == 'true'"> @@ -502,6 +503,16 @@ <_XUnitParallelMode Condition=" '$(LongRunningGCTests)' == 'true' ">none <_XUnitParallelMode Condition=" '$(GcSimulatorTests)' == 'true' ">none -parallel $(_XUnitParallelMode) -nocolor -noshadow -xml testResults.xml + false + + + + + $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults + + + + $(BUILD_SOURCESDIRECTORY)/artifacts/helixresults @@ -526,10 +537,33 @@ - + + + + + + + + + + + + + + + + + + + + + + + @@ -547,6 +581,27 @@ + + + + + + + + + + + + + + + + + + + + + @(HelixPreCommand) @(HelixPostCommand) From ec837a4aeff782264102e419f62090c0c576ec84 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Sat, 27 Aug 2022 10:14:19 -0700 Subject: [PATCH 03/23] Fix displayName YML syntax with variables --- eng/pipelines/common/templates/runtimes/run-test-job.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 5b405b16d9a7b6..6ef893512db20c 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -672,7 +672,7 @@ jobs: # If all the partitions fail, merge-mch would fail and we won't run future steps like uploading superpmi collection. # This depends on mcs.exe (from artifacts Core_Root or product binaries or PATH). - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch - displayName: ${{ format('Merge {0}-{1} SuperPMI collections', CollectionName, CollectionType) }} + displayName: 'Merge $(CollectionName)-$(CollectionType) SuperPMI collections' condition: always() # If merge step above fails, then skip "Upload as artifact" and "Upload to Azure storage" @@ -684,14 +684,14 @@ jobs: tarCompression: $(tarCompression) archiveExtension: $(archiveExtension) artifactName: 'SuperPMI_Collection_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - displayName: ${{ format('Upload artifacts SuperPMI {0}-{1} collection', CollectionName, CollectionType) }} + displayName: 'Upload artifacts SuperPMI $(CollectionName)-$(CollectionType) collection' # Ensure the Python azure-storage-blob package is installed before doing the upload. - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall displayName: Upgrade Pip to latest and install azure-storage-blob Python package - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch $(archType) -build_type $(buildConfig) -mch_files $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper) - displayName: ${{ format('Upload SuperPMI {0}-{1} collection to Azure Storage', CollectionName, CollectionType) }} + displayName: 'Upload SuperPMI $(CollectionName)-$(CollectionType) collection to Azure Storage' env: CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline From e262aa763794e5800c7c0e3b91d348f55403f68c Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Sat, 27 Aug 2022 10:14:45 -0700 Subject: [PATCH 04/23] Update coreclr tests platforms --- eng/pipelines/coreclr/superpmi-collect.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 35c0ee681e2135..c5d409b2ffa793 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -174,11 +174,14 @@ jobs: parameters: jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml buildConfig: checked - platformGroup: all platforms: - # It is too early to include OSX_arm64 in platform group all - # Adding it here will enable it to also run this test - OSX_arm64 + - Linux_arm + - Linux_arm64 + - Linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: From 777a7727c656f8031074a9b407dfe64029304451 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Sat, 27 Aug 2022 10:14:59 -0700 Subject: [PATCH 05/23] Fix msbuild syntax; add debugging output example --- src/tests/Common/helixpublishwitharcade.proj | 86 +++++++++++++++++--- 1 file changed, 75 insertions(+), 11 deletions(-) diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index 9b419dee5df370..d992bb88031df0 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -1,9 +1,65 @@ + + @@ -506,19 +562,23 @@ false - + $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults - + $(BUILD_SOURCESDIRECTORY)/artifacts/helixresults + + - + @@ -537,7 +597,7 @@ - + @@ -561,9 +621,13 @@ - + + + @@ -581,13 +645,13 @@ - + - + @@ -599,7 +663,7 @@ - + From a7ab7341d7bc12ae72531a3de78115c0b7f8c53f Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Sat, 27 Aug 2022 20:42:29 -0700 Subject: [PATCH 06/23] Fixes 1. Create new 'superpmi' machine group for internal pipeline that only has one Helix queue per architecture instead of multiple. 2. Copy superpmi.py and dependents to correlation payload superpmi_scripts sub-directory. 3. Always make output directories 4. Set MchFileTag value value in run-test-job.yml. --- .../templates/runtimes/run-test-job.yml | 3 +++ eng/pipelines/coreclr/superpmi-collect.yml | 2 +- .../coreclr/templates/helix-queues-setup.yml | 27 +++++++++++++++---- src/tests/Common/helixpublishwitharcade.proj | 17 +++++++++--- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 6ef893512db20c..f8038bf89f7999 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -154,6 +154,7 @@ jobs: # Variables used for SuperPMI collection - ${{ if eq(parameters.SuperPmiCollect, true) }}: + - MchFileTag: '${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}' - name: CollectionType value: 'run' - name: CollectionName @@ -662,11 +663,13 @@ jobs: mkdir -p $(MergedMchFileLocation) mkdir -p $(SpmiLogsLocation) displayName: Create SuperPMI directories + condition: always() - ${{ if eq(parameters.osGroup, 'windows') }}: - script: | mkdir $(MergedMchFileLocation) mkdir $(SpmiLogsLocation) displayName: Create SuperPMI directories + condition: always() # Always run merge step even if collection of some partition fails so we can store collection of the partitions that succeeded. # If all the partitions fail, merge-mch would fail and we won't run future steps like uploading superpmi collection. diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index c5d409b2ffa793..6dfd77ca85c995 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -182,7 +182,7 @@ jobs: - windows_x64 - windows_x86 - windows_arm64 - helixQueueGroup: ci + helixQueueGroup: superpmi helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testGroup: outerloop diff --git a/eng/pipelines/coreclr/templates/helix-queues-setup.yml b/eng/pipelines/coreclr/templates/helix-queues-setup.yml index 8160f934a9d046..796a36abbda258 100644 --- a/eng/pipelines/coreclr/templates/helix-queues-setup.yml +++ b/eng/pipelines/coreclr/templates/helix-queues-setup.yml @@ -11,6 +11,13 @@ parameters: dependOnEvaluatePaths: false jobParameters: {} +# parameters.jobParameters.helixQueueGroup values: +# 'pr' -- pull request +# 'ci' -- continuous integration ("merge") +# 'libraries' -- libraries tests +# 'cet' -- machines supporting CET technology +# 'superpmi' -- for TeamProject 'internal', a smaller set of queues (one per architecture, not several) for SuperPMI collection + jobs: - template: ${{ parameters.jobTemplate }} parameters: @@ -45,7 +52,9 @@ jobs: - ${{ if eq(parameters.platform, 'Linux_arm') }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - (Ubuntu.1804.Arm32.Open)Ubuntu.1804.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440 - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: + - (Ubuntu.1804.Arm32)Ubuntu.1804.Armarch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440 + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: - (Debian.10.Arm32)Ubuntu.1804.Armarch@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-arm32v7-20210304164340-6616c63 - (Debian.11.Arm32)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-arm32v7-20210304164347-5a7c380 - (Ubuntu.1804.Arm32)Ubuntu.1804.Armarch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440 @@ -57,7 +66,9 @@ jobs: - ${{ if and(eq(variables['System.TeamProject'], 'public'), notIn(parameters.jobParameters.helixQueueGroup, 'pr', 'ci', 'libraries')) }}: - (Debian.10.Arm64.Open)Ubuntu.1804.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-arm64v8-20220818195427-06f234f - (Debian.11.Arm64.Open)Ubuntu.1804.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-arm64v8-20220818195437-06f234f - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: + - (Ubuntu.1804.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8-20220824230426-06f234f + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: - (Debian.10.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-arm64v8-20220818195427-06f234f - (Debian.11.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-arm64v8-20220818195437-06f234f - (Ubuntu.1804.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8-20220824230426-06f234f @@ -93,7 +104,9 @@ jobs: - Ubuntu.1804.Amd64.Open - (Centos.8.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-8-helix-20201229003624-c1bf759 - RedHat.7.Amd64.Open - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: + - Ubuntu.1804.Amd64 + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: - (Debian.10.Amd64)Ubuntu.1804.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64-20220810215022-f344011 - (Debian.11.Amd64)Ubuntu.1804.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64-20220810215032-f344011 - Ubuntu.1804.Amd64 @@ -126,7 +139,9 @@ jobs: - (Windows.Nano.1809.Amd64.Open)windows.10.amd64.serverrs5.open@mcr.microsoft.com/dotnet-buildtools/prereqs:nanoserver-1809-helix-amd64-08e8e40-20200107182504 - Windows.7.Amd64.Open - Windows.10.Amd64.Open - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: + - Windows.10.Amd64.X86.Rt + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: - Windows.7.Amd64 - Windows.81.Amd64 - Windows.10.Amd64 @@ -140,7 +155,9 @@ jobs: - ${{ if and(eq(variables['System.TeamProject'], 'public'), notIn(parameters.jobParameters.helixQueueGroup, 'pr', 'ci', 'libraries')) }}: - Windows.7.Amd64.Open - Windows.10.Amd64.Open - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: + - Windows.10.Amd64.X86.Rt + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}: - Windows.7.Amd64 - Windows.81.Amd64 - Windows.10.Amd64 diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index d992bb88031df0..63e5a6deb0e76c 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -46,7 +46,7 @@ BundledNETCoreAppPackageVersion <_PALTestsDir> - <_SuperPmiCollect>false + <_SuperPmiCollect>true - + @@ -287,6 +287,13 @@ + + + + <_SuperPmiScriptsFiles Include="$(RepoRoot)src\coreclr\scripts\*.py" /> + + + @@ -603,6 +610,7 @@ + @@ -614,7 +622,7 @@ - + @@ -649,6 +657,7 @@ + @@ -656,7 +665,7 @@ - + From ea7294654f8a6c7c4331af727e8e560d9d395b22 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Sun, 28 Aug 2022 00:07:10 -0700 Subject: [PATCH 07/23] Set explicit Core_Root on Helix machines for superpmi.py --- src/tests/Common/helixpublishwitharcade.proj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index 63e5a6deb0e76c..892dc595b26483 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -629,7 +629,7 @@ - + + <_MergedWrapperRunScriptPrefix Condition="'$(TestWrapperTargetsWindows)' == 'true'" Include="call " /> <_MergedWrapperOutOfProcessTestMarkers Include="$(_MergedWrapperParentDirectory)/**/*.OutOfProcessTest" /> @@ -391,8 +393,8 @@ - - + + @@ -623,7 +625,8 @@ - + + @@ -666,7 +669,8 @@ - + + @@ -744,12 +748,14 @@ dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) -trait TestGroup=%(TestGroup) $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) + coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).mch.mct;coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).log %(PayloadDirectory) %(MergedTestHelixCommand) $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) + coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).mch.mct;coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).log From 21336f21b29f52f209cb24a25fd27c9df1927352 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Sun, 28 Aug 2022 15:39:58 -0700 Subject: [PATCH 09/23] Fix property syntax --- src/tests/Common/helixpublishwitharcade.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index 5ebd0bc05bd0b8..0a9189e0089eb9 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -363,7 +363,7 @@ <_MergedWrapperName>%(_MergedWrapperRunScript.FileName) <_MergedWrapperRunScriptRelative Condition="'%(_MergedWrapperRunScript.Identity)' != ''">$([System.IO.Path]::GetRelativePath($(TestBinDir), %(_MergedWrapperRunScript.FullPath))) - <_MergedWrapperRunScriptPrefix Condition="'$(TestWrapperTargetsWindows)' == 'true'" Include="call " /> + <_MergedWrapperRunScriptPrefix Condition="'$(TestWrapperTargetsWindows)' == 'true'">call <_MergedWrapperOutOfProcessTestMarkers Include="$(_MergedWrapperParentDirectory)/**/*.OutOfProcessTest" /> From 6e38eeb300d1e7c56b46142ac90a0c9005d0ebe3 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Tue, 30 Aug 2022 17:20:39 -0700 Subject: [PATCH 10/23] Fixes 1. Fix 'du' to only run on non-Windows 2. Make 'du' run always, even after failure 3. Make artifacts upload run always, even if merge-mch fails (I'm seeing MCT creation fail but not MCH merging; maybe I can more easily repro if the MCH gets uploaded?) 4. Stop uploading MCT files from Helix machines (a new MCT will get created with the merged MCH). (Optionally, we should just not create the MCT on the Helix machines.) --- .../templates/runtimes/run-test-job.yml | 66 +++++++++++++++++-- src/tests/Common/helixpublishwitharcade.proj | 4 +- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index f8038bf89f7999..75484cecae19bb 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -388,6 +388,15 @@ jobs: env: __MonoToolPrefix: aarch64-linux-gnu- + # Disk space usage before Send to Helix + - ${{ if eq(parameters.SuperPmiCollect, true) }}: + - ${{ if ne(parameters.osGroup, 'windows') }}: + - script: | + du -sh $(Build.SourcesDirectory)/* + df -h + displayName: Disk Usage before Helix + condition: always() + # Send tests to Helix - template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml parameters: @@ -651,12 +660,25 @@ jobs: continueOnError: true condition: always() + ######################################################################################################## # # Finalize SuperPMI collection: (1) merge all MCH files generated by all Helix jobs, (2) upload MCH file to Azure Storage, (3) upload log files # + ######################################################################################################## - ${{ if eq(parameters.SuperPmiCollect, true) }}: + # Disk space usage after Send to Helix + - ${{ if eq(parameters.SuperPmiCollect, true) }}: + - ${{ if ne(parameters.osGroup, 'windows') }}: + - script: | + du -sh $(Build.SourcesDirectory)/* + du -sh $(MchFilesLocation)* + ls -l $(MchFilesLocation) + df -h + displayName: Disk Usage after Helix + condition: always() + # Create required directories for merged mch collection and superpmi logs - ${{ if ne(parameters.osGroup, 'windows') }}: - script: | @@ -672,13 +694,27 @@ jobs: condition: always() # Always run merge step even if collection of some partition fails so we can store collection of the partitions that succeeded. - # If all the partitions fail, merge-mch would fail and we won't run future steps like uploading superpmi collection. + # If all the partitions fail, merge-mch will fail and we won't run future steps like uploading superpmi collection. # This depends on mcs.exe (from artifacts Core_Root or product binaries or PATH). - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch displayName: 'Merge $(CollectionName)-$(CollectionType) SuperPMI collections' condition: always() - + + # Disk space usage after Send to Helix + - ${{ if eq(parameters.SuperPmiCollect, true) }}: + - ${{ if ne(parameters.osGroup, 'windows') }}: + - script: | + du -sh $(Build.SourcesDirectory)/* + du -sh $(MchFilesLocation)* + ls -l $(MchFilesLocation) + du -sh $(MergedMchFileLocation)* + ls -l $(MergedMchFileLocation) + df -h + displayName: Disk Usage after merge + condition: always() + # If merge step above fails, then skip "Upload as artifact" and "Upload to Azure storage" + # (temporary? always upload, in case merge of MCH succeeds but creation of MCT fails) - template: /eng/pipelines/common/upload-artifact-step.yml parameters: rootFolder: $(MergedMchFileLocation) @@ -688,16 +724,17 @@ jobs: archiveExtension: $(archiveExtension) artifactName: 'SuperPMI_Collection_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' displayName: 'Upload artifacts SuperPMI $(CollectionName)-$(CollectionType) collection' - + condition: always() + # Ensure the Python azure-storage-blob package is installed before doing the upload. - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall displayName: Upgrade Pip to latest and install azure-storage-blob Python package - + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch $(archType) -build_type $(buildConfig) -mch_files $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper) displayName: 'Upload SuperPMI $(CollectionName)-$(CollectionType) collection to Azure Storage' env: CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline - + # Always upload the available logs for diagnostics - task: CopyFiles@2 displayName: Copying superpmi.log of all partitions @@ -706,10 +743,25 @@ jobs: contents: '**/$(CollectionName).$(CollectionType)*.log' targetFolder: '$(SpmiLogsLocation)' condition: always() - + - task: PublishPipelineArtifact@1 displayName: Publish SuperPMI logs inputs: targetPath: $(SpmiLogsLocation) artifactName: 'SuperPMI_Logs_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - condition: always() \ No newline at end of file + condition: always() + + # Disk space usage logs copied + - ${{ if eq(parameters.SuperPmiCollect, true) }}: + - ${{ if ne(parameters.osGroup, 'windows') }}: + - script: | + du -sh $(SpmiLogsLocation)* + ls -l $(SpmiLogsLocation) + displayName: Disk Usage after logs copied + condition: always() + + ######################################################################################################## + # + # End of SuperPMI processing + # + ######################################################################################################## \ No newline at end of file diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index 0a9189e0089eb9..a50a5601b7710c 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -748,14 +748,14 @@ dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) -trait TestGroup=%(TestGroup) $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) - coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).mch.mct;coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).log + coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).log %(PayloadDirectory) %(MergedTestHelixCommand) $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) - coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).mch.mct;coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).log + coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).log From 040e1560882cda69685027ac4fc4120914a38ef0 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Tue, 30 Aug 2022 17:30:06 -0700 Subject: [PATCH 11/23] [TEMPORARY] Upload pre-merge MCH files to artifacts --- .../common/templates/runtimes/run-test-job.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 75484cecae19bb..25bd4e93f381a1 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -693,6 +693,18 @@ jobs: displayName: Create SuperPMI directories condition: always() + # Temporary? Upload all the downloaded per-Helix MCH files, before the merge step. + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(MchFilesLocation) + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: 'SuperPMI_PreMerge_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + displayName: 'Upload pre-merge artifacts SuperPMI $(CollectionName)-$(CollectionType)' + condition: always() + # Always run merge step even if collection of some partition fails so we can store collection of the partitions that succeeded. # If all the partitions fail, merge-mch will fail and we won't run future steps like uploading superpmi collection. # This depends on mcs.exe (from artifacts Core_Root or product binaries or PATH). From a9676e26a339e0bc1f8c26dd5ba6866b80a0669c Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Tue, 30 Aug 2022 18:17:20 -0700 Subject: [PATCH 12/23] [TEMPORARY] add 'condition' option to upload-artifact-step template --- eng/pipelines/common/templates/runtimes/run-test-job.yml | 6 +++--- eng/pipelines/common/upload-artifact-step.yml | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 25bd4e93f381a1..9510c0d576597b 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -703,7 +703,7 @@ jobs: archiveExtension: $(archiveExtension) artifactName: 'SuperPMI_PreMerge_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' displayName: 'Upload pre-merge artifacts SuperPMI $(CollectionName)-$(CollectionType)' - condition: always() + condition: always() # Always run merge step even if collection of some partition fails so we can store collection of the partitions that succeeded. # If all the partitions fail, merge-mch will fail and we won't run future steps like uploading superpmi collection. @@ -736,7 +736,7 @@ jobs: archiveExtension: $(archiveExtension) artifactName: 'SuperPMI_Collection_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' displayName: 'Upload artifacts SuperPMI $(CollectionName)-$(CollectionType) collection' - condition: always() + condition: always() # Ensure the Python azure-storage-blob package is installed before doing the upload. - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall @@ -776,4 +776,4 @@ jobs: # # End of SuperPMI processing # - ######################################################################################################## \ No newline at end of file + ######################################################################################################## diff --git a/eng/pipelines/common/upload-artifact-step.yml b/eng/pipelines/common/upload-artifact-step.yml index df753b1ed0321e..249da066c7aae6 100644 --- a/eng/pipelines/common/upload-artifact-step.yml +++ b/eng/pipelines/common/upload-artifact-step.yml @@ -6,6 +6,7 @@ parameters: archiveExtension: '' artifactName: '' displayName: '' + condition: succeeded() steps: # Zip Artifact @@ -17,9 +18,11 @@ steps: archiveType: ${{ parameters.archiveType }} tarCompression: ${{ parameters.tarCompression }} includeRootFolder: ${{ parameters.includeRootFolder }} + condition: ${{ parameters.condition }} - task: PublishBuildArtifacts@1 displayName: 'Publish ${{ parameters.displayName }}' inputs: pathtoPublish: $(Build.StagingDirectory)/${{ parameters.artifactName }}${{ parameters.archiveExtension }} artifactName: ${{ parameters.artifactName }} + condition: ${{ parameters.condition }} From fbb8d8b4fc1ed8ee53a54bc9ab57c8a7ca83519c Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Tue, 30 Aug 2022 23:15:36 -0700 Subject: [PATCH 13/23] Fix buffer overflow failure in sprintf_s for a huge function signature Truncate any function signature that is too large (> 63KB). --- .../tools/superpmi/superpmi-shared/methodcontext.cpp | 11 +++++++++-- .../tools/superpmi/superpmi-shared/methodcontext.h | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp index 3d7b40ff375f09..b25c8aad527a45 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp @@ -7001,8 +7001,15 @@ int MethodContext::dumpMethodIdentityInfoToBuffer(char* buff, int len, bool igno char* obuff = buff; - // Add the Method Signature - int t = sprintf_s(buff, len, "%s -- ", CallUtils::GetMethodFullName(this, pInfo->ftn, pInfo->args, ignoreMethodName)); + // Add the Method Signature. Be careful about potentially huge method signatures; truncate if necessary. + const char* methodFullName = CallUtils::GetMethodFullName(this, pInfo->ftn, pInfo->args, ignoreMethodName); + int t = _snprintf_s(buff, len - METHOD_IDENTITY_INFO_NON_NAME_RESERVE, _TRUNCATE, "%s -- ", methodFullName); + if (t == -1) + { + // We truncated the name string, meaning we wrote exactly `len - METHOD_IDENTITY_INFO_NON_NAME_RESERVE` characters + // (including the terminating null). We advance the buffer pointer by this amount, not including that terminating null. + t = len - METHOD_IDENTITY_INFO_NON_NAME_RESERVE - 1; + } buff += t; len -= t; diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h index 3fc5247ef66c20..b14d67b6b678c9 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h @@ -42,6 +42,7 @@ extern bool g_debugRep; const char* toString(CorInfoType cit); #define METHOD_IDENTITY_INFO_SIZE 0x10000 // We assume that the METHOD_IDENTITY_INFO_SIZE will not exceed 64KB +#define METHOD_IDENTITY_INFO_NON_NAME_RESERVE 0x400 // Reserve 1KB of METHOD_IDENTITY_INFO_SIZE for everything except for the method name. // Special "jit flags" for noting some method context features From 1ec4fbf3aa7aef5c28512c47fa6bfcce176ff7df Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Tue, 30 Aug 2022 23:22:43 -0700 Subject: [PATCH 14/23] Remove disk space measurement --- .../templates/runtimes/run-test-job.yml | 42 ------------------- 1 file changed, 42 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 9510c0d576597b..8370d178ac4972 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -388,15 +388,6 @@ jobs: env: __MonoToolPrefix: aarch64-linux-gnu- - # Disk space usage before Send to Helix - - ${{ if eq(parameters.SuperPmiCollect, true) }}: - - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage before Helix - condition: always() - # Send tests to Helix - template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml parameters: @@ -668,17 +659,6 @@ jobs: - ${{ if eq(parameters.SuperPmiCollect, true) }}: - # Disk space usage after Send to Helix - - ${{ if eq(parameters.SuperPmiCollect, true) }}: - - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - du -sh $(MchFilesLocation)* - ls -l $(MchFilesLocation) - df -h - displayName: Disk Usage after Helix - condition: always() - # Create required directories for merged mch collection and superpmi logs - ${{ if ne(parameters.osGroup, 'windows') }}: - script: | @@ -712,19 +692,6 @@ jobs: displayName: 'Merge $(CollectionName)-$(CollectionType) SuperPMI collections' condition: always() - # Disk space usage after Send to Helix - - ${{ if eq(parameters.SuperPmiCollect, true) }}: - - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - du -sh $(MchFilesLocation)* - ls -l $(MchFilesLocation) - du -sh $(MergedMchFileLocation)* - ls -l $(MergedMchFileLocation) - df -h - displayName: Disk Usage after merge - condition: always() - # If merge step above fails, then skip "Upload as artifact" and "Upload to Azure storage" # (temporary? always upload, in case merge of MCH succeeds but creation of MCT fails) - template: /eng/pipelines/common/upload-artifact-step.yml @@ -763,15 +730,6 @@ jobs: artifactName: 'SuperPMI_Logs_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' condition: always() - # Disk space usage logs copied - - ${{ if eq(parameters.SuperPmiCollect, true) }}: - - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: | - du -sh $(SpmiLogsLocation)* - ls -l $(SpmiLogsLocation) - displayName: Disk Usage after logs copied - condition: always() - ######################################################################################################## # # End of SuperPMI processing From 0b2df7a85145aeecd0a2eac8298a3e3ec6e6b876 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Wed, 31 Aug 2022 10:39:10 -0700 Subject: [PATCH 15/23] Escape semicolons in CDATA block --- src/tests/Common/CLRTest.Jit.targets | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index cc7af64aad395c..6f64ffaa7ecaa6 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -191,6 +191,9 @@ IF NOT DEFINED DoLink ( *********************************************************************************************** SuperPMI collection of CoreCLR tests *********************************************************************************************** + + You shouldn't have to escape characters in a CDATA block, but it appears that you do actually need + to escape semicolons with %3B here. --> Date: Wed, 31 Aug 2022 11:09:07 -0700 Subject: [PATCH 16/23] Fix bash conditions to work on Mac as well as Linux --- src/tests/Common/CLRTest.Jit.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/Common/CLRTest.Jit.targets b/src/tests/Common/CLRTest.Jit.targets index 6f64ffaa7ecaa6..df93c54cd7e4d6 100644 --- a/src/tests/Common/CLRTest.Jit.targets +++ b/src/tests/Common/CLRTest.Jit.targets @@ -201,10 +201,10 @@ IF NOT DEFINED DoLink ( Date: Wed, 31 Aug 2022 11:39:03 -0700 Subject: [PATCH 17/23] Skip TOC creation on Helix machines We're going to merge and regenerate the overall TOC anyway, so no need to create it and download it to the AzDO machine. --- src/coreclr/scripts/superpmi.py | 9 ++++++++- src/tests/Common/helixpublishwitharcade.proj | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py index fff1d9cbfb6908..97474616f47c00 100644 --- a/src/coreclr/scripts/superpmi.py +++ b/src/coreclr/scripts/superpmi.py @@ -301,6 +301,7 @@ def add_core_root_arguments(parser, build_type_default, build_type_help): collect_parser.add_argument("--clean", action="store_true", help="Clean the collection by removing contexts that fail to replay without error.") collect_parser.add_argument("--skip_collection_step", action="store_true", help="Do not run the collection step.") collect_parser.add_argument("--skip_merge_step", action="store_true", help="Do not run the merge step.") +collect_parser.add_argument("--skip_toc_step", action="store_true", help="Do not run the TOC creation step.") collect_parser.add_argument("--skip_collect_mc_files", action="store_true", help="Do not collect .MC files") # Create a set of arguments common to all SuperPMI replay commands, namely basic replay and ASM diffs. @@ -731,7 +732,8 @@ def collect(self): else: self.__copy_to_final_mch_file__() - self.__create_toc__() + if not self.coreclr_args.skip_toc_step: + self.__create_toc__() if self.coreclr_args.clean: # There is no point to verify unless we have run the clean step. @@ -3523,6 +3525,11 @@ def verify_base_diff_args(): lambda unused: True, "Unable to set skip_merge_step.") + coreclr_args.verify(args, + "skip_toc_step", + lambda unused: True, + "Unable to set skip_toc_step.") + coreclr_args.verify(args, "clean", lambda unused: True, diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index a50a5601b7710c..ef719a53cf8d0b 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -632,7 +632,7 @@ - + - - + @@ -669,11 +667,9 @@ - - - + @@ -748,14 +744,14 @@ dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) -trait TestGroup=%(TestGroup) $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) - coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).log + coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).log %(PayloadDirectory) %(MergedTestHelixCommand) $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) - coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.0.$(TargetOS).$(TargetArchitecture).$(Configuration).log + coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).log From 723176fdc8e7fa6f76357326cb72bdd2a60fbe92 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 1 Sep 2022 12:31:51 -0700 Subject: [PATCH 21/23] Re-enable all collections --- eng/pipelines/coreclr/superpmi-collect.yml | 199 ++++++++++----------- 1 file changed, 99 insertions(+), 100 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 6dfd77ca85c995..2dc130c57ea656 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -66,106 +66,105 @@ jobs: jobParameters: testGroup: outerloop -############################################################### disable the following to test coreclr test run collection -# - template: /eng/pipelines/common/platform-matrix.yml -# parameters: -# jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml -# buildConfig: checked -# platforms: -# - OSX_arm64 -# - Linux_arm -# - Linux_arm64 -# - Linux_x64 -# - windows_x64 -# - windows_x86 -# - windows_arm64 -# helixQueueGroup: ci -# helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml -# jobParameters: -# testGroup: outerloop -# liveLibrariesBuildConfig: Release -# collectionType: pmi -# collectionName: libraries -# -# - template: /eng/pipelines/common/platform-matrix.yml -# parameters: -# jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml -# buildConfig: checked -# platforms: -# - OSX_arm64 -# - Linux_arm -# - Linux_arm64 -# - Linux_x64 -# - windows_x64 -# - windows_x86 -# - windows_arm64 -# helixQueueGroup: ci -# helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml -# jobParameters: -# testGroup: outerloop -# liveLibrariesBuildConfig: Release -# collectionType: pmi -# collectionName: coreclr_tests -# -# - template: /eng/pipelines/common/platform-matrix.yml -# parameters: -# jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml -# buildConfig: checked -# platforms: -# - OSX_arm64 -# - Linux_arm -# - Linux_arm64 -# - Linux_x64 -# - windows_x64 -# - windows_x86 -# - windows_arm64 -# helixQueueGroup: ci -# helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml -# jobParameters: -# testGroup: outerloop -# liveLibrariesBuildConfig: Release -# collectionType: pmi -# collectionName: libraries_tests -# -# - template: /eng/pipelines/common/platform-matrix.yml -# parameters: -# jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml -# buildConfig: checked -# platforms: -# - OSX_arm64 -# - Linux_arm -# - Linux_arm64 -# - Linux_x64 -# - windows_x64 -# - windows_x86 -# - windows_arm64 -# helixQueueGroup: ci -# helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml -# jobParameters: -# testGroup: outerloop -# liveLibrariesBuildConfig: Release -# collectionType: crossgen2 -# collectionName: libraries -# -# - template: /eng/pipelines/common/platform-matrix.yml -# parameters: -# jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml -# buildConfig: checked -# platforms: -# - OSX_arm64 -# - Linux_arm -# - Linux_arm64 -# - Linux_x64 -# - windows_x64 -# - windows_x86 -# - windows_arm64 -# helixQueueGroup: ci -# helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml -# jobParameters: -# testGroup: outerloop -# liveLibrariesBuildConfig: Release -# collectionType: run -# collectionName: benchmarks +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - OSX_arm64 + - Linux_arm + - Linux_arm64 + - Linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: pmi + collectionName: libraries + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - OSX_arm64 + - Linux_arm + - Linux_arm64 + - Linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: pmi + collectionName: coreclr_tests + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - OSX_arm64 + - Linux_arm + - Linux_arm64 + - Linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: pmi + collectionName: libraries_tests + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - OSX_arm64 + - Linux_arm + - Linux_arm64 + - Linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: crossgen2 + collectionName: libraries + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - OSX_arm64 + - Linux_arm + - Linux_arm64 + - Linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: run + collectionName: benchmarks # # Collection of coreclr test run From 68b86282dfef8eaee6dc7b5d786667dd35dca55f Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Thu, 1 Sep 2022 13:26:57 -0700 Subject: [PATCH 22/23] Add a condition for DownloadFilesFromResults --- src/tests/Common/helixpublishwitharcade.proj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index 99f9db5b554ccc..241a21f227dcff 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -744,14 +744,14 @@ dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) -trait TestGroup=%(TestGroup) $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) - coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).log + coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).log %(PayloadDirectory) %(MergedTestHelixCommand) $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) - coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).log + coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).mch;coreclr_tests.run.$(TargetOS).$(TargetArchitecture).$(Configuration).log From 4eff2eba536ce2be3a806cc33432e402962c44db Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Tue, 6 Sep 2022 16:26:41 -0700 Subject: [PATCH 23/23] Fix Linux path separators --- src/tests/Common/helixpublishwitharcade.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index 241a21f227dcff..885d3464347565 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -666,7 +666,7 @@ - +