[release/10.0] Backport Helix job monitor

.config/dotnet-tools.json

@@ -14,6 +14,12 @@
         "reportgenerator"
       ]
     },
+    "microsoft.dotnet.helix.jobmonitor": {
+      "version": "10.0.0-beta.26461.103",
+      "commands": [
+        "dotnet-helix-job-monitor"
+      ]
+    },
     "microsoft.dotnet.xharness.cli": {
       "version": "11.0.0-prerelease.26461.1",
       "commands": [

eng/Version.Details.props

@@ -23,6 +23,7 @@ This file should be imported by eng/Versions.props
     <MicrosoftDotNetCodeAnalysisPackageVersion>10.0.0-beta.26461.103</MicrosoftDotNetCodeAnalysisPackageVersion>
     <MicrosoftDotNetGenAPIPackageVersion>10.0.0-beta.26461.103</MicrosoftDotNetGenAPIPackageVersion>
     <MicrosoftDotNetGenFacadesPackageVersion>10.0.0-beta.26461.103</MicrosoftDotNetGenFacadesPackageVersion>
+    <MicrosoftDotNetHelixJobMonitorPackageVersion>10.0.0-beta.26461.103</MicrosoftDotNetHelixJobMonitorPackageVersion>
     <MicrosoftDotNetHelixSdkPackageVersion>10.0.0-beta.26461.103</MicrosoftDotNetHelixSdkPackageVersion>
     <MicrosoftDotNetPackageTestingPackageVersion>10.0.0-beta.26461.103</MicrosoftDotNetPackageTestingPackageVersion>
     <MicrosoftDotNetRemoteExecutorPackageVersion>10.0.0-beta.26461.103</MicrosoftDotNetRemoteExecutorPackageVersion>
@@ -138,6 +139,7 @@ This file should be imported by eng/Versions.props
     <MicrosoftDotNetCodeAnalysisVersion>$(MicrosoftDotNetCodeAnalysisPackageVersion)</MicrosoftDotNetCodeAnalysisVersion>
     <MicrosoftDotNetGenAPIVersion>$(MicrosoftDotNetGenAPIPackageVersion)</MicrosoftDotNetGenAPIVersion>
     <MicrosoftDotNetGenFacadesVersion>$(MicrosoftDotNetGenFacadesPackageVersion)</MicrosoftDotNetGenFacadesVersion>
+    <MicrosoftDotNetHelixJobMonitorVersion>$(MicrosoftDotNetHelixJobMonitorPackageVersion)</MicrosoftDotNetHelixJobMonitorVersion>
     <MicrosoftDotNetHelixSdkVersion>$(MicrosoftDotNetHelixSdkPackageVersion)</MicrosoftDotNetHelixSdkVersion>
     <MicrosoftDotNetPackageTestingVersion>$(MicrosoftDotNetPackageTestingPackageVersion)</MicrosoftDotNetPackageTestingVersion>
     <MicrosoftDotNetRemoteExecutorVersion>$(MicrosoftDotNetRemoteExecutorPackageVersion)</MicrosoftDotNetRemoteExecutorVersion>

eng/Version.Details.xml

@@ -67,6 +67,10 @@
       <Uri>https://github.com/dotnet/dotnet</Uri>
       <Sha>fa5198e61f57d7b1887976deaaff177e91c1b63d</Sha>
     </Dependency>
+    <Dependency Name="Microsoft.DotNet.Helix.JobMonitor" Version="10.0.0-beta.26461.103">
+      <Uri>https://github.com/dotnet/dotnet</Uri>
+      <Sha>fa5198e61f57d7b1887976deaaff177e91c1b63d</Sha>
+    </Dependency>
     <Dependency Name="Microsoft.DotNet.GenAPI" Version="10.0.0-beta.26461.103">
       <Uri>https://github.com/dotnet/dotnet</Uri>
       <Sha>fa5198e61f57d7b1887976deaaff177e91c1b63d</Sha>

eng/pipelines/common/templates/browser-wasm-build-tests.yml

@@ -1,6 +1,7 @@
 # Depends on other builds to get runtime packs, and nugets
 # And then runs Wasm.Build.Tests
 parameters:
+  useHelixMonitor: false
   alwaysRun: false
   isExtraPlatformsBuild: false
   isWasmOnlyBuild: false
@@ -122,6 +123,7 @@ jobs:
         postBuildSteps:
           - template: /eng/pipelines/libraries/helix.yml
             parameters:
+              useHelixMonitor: ${{ parameters.useHelixMonitor }}
               creator: dotnet-bot
               testRunNamePrefixSuffix: Mono_$(_BuildConfig)_$(_hostedOs)
               extraHelixArguments: /p:BrowserHost=$(_hostedOs)

eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml

@@ -1,4 +1,5 @@
 parameters:
+  useHelixMonitor: false
   buildConfig: ''
   osGroup: ''
   osSubgroup: ''
@@ -84,6 +85,7 @@ steps:
 
   - template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml
     parameters:
+      useHelixMonitor: ${{ parameters.useHelixMonitor }}
       displayName: Send tests to Helix
       buildConfig: $(buildConfigUpper)
       archType: ${{ parameters.archType }}

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -1,4 +1,5 @@
 parameters:
+  useHelixMonitor: false
   buildConfig: ''
   archType: ''
   osGroup: ''
@@ -273,6 +274,7 @@ jobs:
     # Send tests to Helix
     - template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml
       parameters:
+        useHelixMonitor: ${{ parameters.useHelixMonitor }}
         displayName: Send tests to Helix
         buildConfig: $(buildConfigUpper)
         archType: ${{ parameters.archType }}

eng/pipelines/common/templates/runtimes/send-to-helix-step.yml

@@ -1,4 +1,5 @@
 parameters:
+  useHelixMonitor: false
   displayName: ''
   condition: true
   archType: ''
@@ -41,7 +42,7 @@ steps:
 - template: send-to-helix-inner-step.yml
   parameters:
     osGroup: ${{ parameters.osGroup }}
-    sendParams: ${{ parameters.helixProjectArguments }} ${{ parameters.msbuildParallelism }} /bl:$(Build.SourcesDirectory)/artifacts/log/SendToHelix.binlog /p:TargetArchitecture=${{ parameters.archType }} /p:TargetOS=${{ parameters.osGroup }} /p:TargetOSSubgroup=${{ parameters.osSubgroup }} /p:Configuration=${{ parameters.buildConfig }} ${{ parameters.extraHelixArguments }}
+    sendParams: ${{ parameters.helixProjectArguments }} ${{ parameters.msbuildParallelism }} /bl:$(Build.SourcesDirectory)/artifacts/log/SendToHelix.binlog /p:TargetArchitecture=${{ parameters.archType }} /p:TargetOS=${{ parameters.osGroup }} /p:TargetOSSubgroup=${{ parameters.osSubgroup }} /p:Configuration=${{ parameters.buildConfig }} /p:EnableHelixJobMonitor=${{ parameters.useHelixMonitor }} ${{ parameters.extraHelixArguments }}
     condition: and(succeeded(), ${{ parameters.condition }})
     shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
     displayName: ${{ parameters.displayName }}

eng/pipelines/common/templates/simple-wasm-build-tests.yml

@@ -1,5 +1,6 @@
 # run Wasm.Build.Tests with a single threading runtime
 parameters:
+  useHelixMonitor: false
   alwaysRun: false
   isExtraPlatformsBuild: false
   isWasmOnlyBuild: false
@@ -45,6 +46,7 @@ jobs:
       postBuildSteps:
         - template: /eng/pipelines/libraries/helix.yml
           parameters:
+            useHelixMonitor: ${{ parameters.useHelixMonitor }}
             creator: dotnet-bot
             testRunNamePrefixSuffix: Mono_$(_BuildConfig)_$(_hostedOs)
             extraHelixArguments: /p:BrowserHost=$(_hostedOs)

eng/pipelines/common/templates/wasm-library-aot-tests.yml

@@ -1,4 +1,5 @@
 parameters:
+  useHelixMonitor: false
   alwaysRun: false
   extraBuildArgs: ''
   extraHelixArguments: ''
@@ -21,6 +22,7 @@ jobs:
 - ${{ each platform in parameters.platforms }}:
   - template: /eng/pipelines/common/templates/wasm-library-tests.yml
     parameters:
+      useHelixMonitor: ${{ parameters.useHelixMonitor }}
       platforms:
         - ${{ platform }}
       nameSuffix: ${{ parameters.nameSuffix }}

eng/pipelines/common/templates/wasm-library-tests.yml

@@ -1,4 +1,5 @@
 parameters:
+  useHelixMonitor: false
   alwaysRun: false
   extraBuildArgs: ''
   extraHelixArguments: ''
@@ -95,6 +96,7 @@ jobs:
       postBuildSteps:
         - template: /eng/pipelines/libraries/helix.yml
           parameters:
+            useHelixMonitor: ${{ parameters.useHelixMonitor }}
             creator: dotnet-bot
             testRunNamePrefixSuffix: Mono_$(_BuildConfig)
             extraHelixArguments: /p:BrowserHost=$(_hostedOs) $(_wasmRunSmokeTestsOnlyArg) ${{ parameters.extraHelixArguments }}

eng/pipelines/common/templates/wasm-runtime-tests.yml

@@ -1,4 +1,5 @@
 parameters:
+  useHelixMonitor: false
   alwaysRun: false
   isExtraPlatformsBuild: false
   isWasmOnlyBuild: false
@@ -45,6 +46,7 @@ jobs:
       postBuildSteps:
         - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml
           parameters:
+            useHelixMonitor: ${{ parameters.useHelixMonitor }}
             creator: dotnet-bot
             testRunNamePrefixSuffix: Mono_$(_BuildConfig)
             testBuildArgs: /p:InstallV8ForTests=false

eng/pipelines/common/variables.yml

@@ -45,6 +45,9 @@ variables:
 - name: isNotSpecificPlatformOnlyBuild
   value: ${{ notin(variables['Build.DefinitionName'], 'runtime-wasm', 'runtime-wasm-libtests', 'runtime-wasm-non-libtests', 'runtime-ioslike', 'runtime-ioslikesimulator', 'runtime-android', 'runtime-androidemulator', 'runtime-maccatalyst', 'runtime-linuxbionic') }}
 
+- name: enableHelixJobMonitor
+  value: false
+
 - name: debugOnPrReleaseOnRolling
   ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
     value: Release

eng/pipelines/coreclr/ci.yml

@@ -31,17 +31,26 @@ schedules:
 
 variables:
   - template: /eng/pipelines/common/variables.yml
+  - name: enableHelixJobMonitor
+    value: true
 
 extends:
   template:  /eng/pipelines/common/templates/pipeline-with-resources.yml
   parameters:
     stages:
     - stage: Build
       jobs:
+      - ${{ if eq(variables['enableHelixJobMonitor'], true) }}:
+        - template: /eng/common/core-templates/job/helix-job-monitor.yml
+          parameters:
+            helixAccessToken: $(HelixApiAccessToken)
+            timeoutInMinutes: 540
+            allowNoHelixJobs: true
 
       #
       # Debug builds
       #
+
       - template: /eng/pipelines/common/platform-matrix.yml
         parameters:
           jobTemplate: /eng/pipelines/common/global-build-job.yml
@@ -143,6 +152,7 @@ extends:
           helixQueueGroup: ci
           helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
           jobParameters:
+            useHelixMonitor: ${{ variables.enableHelixJobMonitor }}
             testGroup: outerloop
             liveLibrariesBuildConfig: Release
             unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)
@@ -166,6 +176,7 @@ extends:
           helixQueueGroup: ci
           helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
           jobParameters:
+            useHelixMonitor: ${{ variables.enableHelixJobMonitor }}
             testGroup: outerloop
             readyToRun: true
             displayNameArgs: R2R_CG2
@@ -192,3 +203,5 @@ extends:
             nameSuffix: PALTests
             postBuildSteps:
               - template: /eng/pipelines/coreclr/templates/run-paltests-step.yml
+                parameters:
+                  useHelixMonitor: ${{ variables.enableHelixJobMonitor }}

eng/pipelines/coreclr/crossgen2-composite.yml

@@ -10,13 +10,22 @@ schedules:
 
 variables:
   - template: /eng/pipelines/common/variables.yml
+  - name: enableHelixJobMonitor
+    value: true
 
 extends:
   template:  /eng/pipelines/common/templates/pipeline-with-resources.yml
   parameters:
     stages:
     - stage: Build
       jobs:
+      - ${{ if eq(variables['enableHelixJobMonitor'], true) }}:
+        - template: /eng/common/core-templates/job/helix-job-monitor.yml
+          parameters:
+            helixAccessToken: $(HelixApiAccessToken)
+            timeoutInMinutes: 540
+            allowNoHelixJobs: true
+
 
       - template: /eng/pipelines/common/platform-matrix.yml
         parameters:
@@ -73,6 +82,7 @@ extends:
           - windows_x64
           - windows_arm64
           jobParameters:
+            useHelixMonitor: ${{ variables.enableHelixJobMonitor }}
             testGroup: innerloop
             readyToRun: true
             compositeBuildMode: true

eng/pipelines/coreclr/crossgen2-gcstress.yml

@@ -10,13 +10,22 @@ schedules:
 
 variables:
   - template: /eng/pipelines/common/variables.yml
+  - name: enableHelixJobMonitor
+    value: true
 
 extends:
   template:  /eng/pipelines/common/templates/pipeline-with-resources.yml
   parameters:
     stages:
     - stage: Build
       jobs:
+      - ${{ if eq(variables['enableHelixJobMonitor'], true) }}:
+        - template: /eng/common/core-templates/job/helix-job-monitor.yml
+          parameters:
+            helixAccessToken: $(HelixApiAccessToken)
+            timeoutInMinutes: 720
+            allowNoHelixJobs: true
+
 
       - template: /eng/pipelines/common/platform-matrix.yml
         parameters:
@@ -71,6 +80,7 @@ extends:
           - windows_x64
           - windows_arm64
           jobParameters:
+            useHelixMonitor: ${{ variables.enableHelixJobMonitor }}
             testGroup: gcstress-extra
             readyToRun: true
             compositeBuildMode: true

eng/pipelines/coreclr/crossgen2-outerloop.yml

@@ -10,13 +10,22 @@ schedules:
 
 variables:
   - template: /eng/pipelines/common/variables.yml
+  - name: enableHelixJobMonitor
+    value: true
 
 extends:
   template:  /eng/pipelines/common/templates/pipeline-with-resources.yml
   parameters:
     stages:
     - stage: Build
       jobs:
+      - ${{ if eq(variables['enableHelixJobMonitor'], true) }}:
+        - template: /eng/common/core-templates/job/helix-job-monitor.yml
+          parameters:
+            helixAccessToken: $(HelixApiAccessToken)
+            timeoutInMinutes: 720
+            allowNoHelixJobs: true
+
 
       - template: /eng/pipelines/common/platform-matrix.yml
         parameters:
@@ -72,6 +81,7 @@ extends:
           - windows_x64
           - windows_arm64
           jobParameters:
+            useHelixMonitor: ${{ variables.enableHelixJobMonitor }}
             testGroup: outerloop
             readyToRun: true
             compositeBuildMode: true
@@ -95,6 +105,7 @@ extends:
           - windows_x64
           - windows_x86
           jobParameters:
+            useHelixMonitor: ${{ variables.enableHelixJobMonitor }}
             testGroup: outerloop
             readyToRun: true
             displayNameArgs: R2R

eng/pipelines/coreclr/crossgen2.yml

@@ -10,13 +10,22 @@ schedules:
 
 variables:
   - template: /eng/pipelines/common/variables.yml
+  - name: enableHelixJobMonitor
+    value: true
 
 extends:
   template:  /eng/pipelines/common/templates/pipeline-with-resources.yml
   parameters:
     stages:
     - stage: Build
       jobs:
+      - ${{ if eq(variables['enableHelixJobMonitor'], true) }}:
+        - template: /eng/common/core-templates/job/helix-job-monitor.yml
+          parameters:
+            helixAccessToken: $(HelixApiAccessToken)
+            timeoutInMinutes: 540
+            allowNoHelixJobs: true
+
 
       - template: /eng/pipelines/common/platform-matrix.yml
         parameters:
@@ -69,6 +78,7 @@ extends:
           - windows_x64
           - windows_arm64
           jobParameters:
+            useHelixMonitor: ${{ variables.enableHelixJobMonitor }}
             testGroup: innerloop
             readyToRun: true
             displayNameArgs: R2R_CG2
@@ -95,6 +105,7 @@ extends:
             postBuildSteps:
               - template: /eng/pipelines/libraries/helix.yml
                 parameters:
+                  useHelixMonitor: ${{ variables.enableHelixJobMonitor }}
                   creator: dotnet-bot
                   testRunNamePrefixSuffix: TestReadyToRun_$(_BuildConfig)
 
@@ -110,6 +121,7 @@ extends:
           - osx_x64
           - windows_x64
           jobParameters:
+            useHelixMonitor: ${{ variables.enableHelixJobMonitor }}
             testGroup: innerloop
             readyToRun: true
             hotColdSplitting: true

eng/pipelines/coreclr/gc-longrunning.yml

@@ -10,13 +10,22 @@ schedules:
 
 variables:
   - template: /eng/pipelines/common/variables.yml
+  - name: enableHelixJobMonitor
+    value: true
 
 extends:
   template:  /eng/pipelines/common/templates/pipeline-with-resources.yml
   parameters:
     stages:
     - stage: Build
       jobs:
+      - ${{ if eq(variables['enableHelixJobMonitor'], true) }}:
+        - template: /eng/common/core-templates/job/helix-job-monitor.yml
+          parameters:
+            helixAccessToken: $(HelixApiAccessToken)
+            timeoutInMinutes: 720
+            allowNoHelixJobs: true
+
 
       - template: /eng/pipelines/common/platform-matrix.yml
         parameters:
@@ -67,6 +76,7 @@ extends:
           helixQueueGroup: ci
           helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
           jobParameters:
+            useHelixMonitor: ${{ variables.enableHelixJobMonitor }}
             testGroup: gc-longrunning
             liveLibrariesBuildConfig: Release
             unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)