From 7df7b9fa429987032984e4e38d18a5026135cb0f Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Mon, 24 Feb 2025 11:19:47 -0800 Subject: [PATCH 01/18] Make python package pipeline 1ES compliant --- .../azure-pipelines/py-packaging-pipeline.yml | 50 ++-- .../stages/py-cpu-packaging-stage.yml | 116 ++++---- .../templates/py-linux-qnn.yml | 110 ++++---- .../azure-pipelines/templates/py-linux.yml | 140 +++++----- .../templates/py-win-arm64-qnn.yml | 261 +++++++++--------- .../templates/py-win-arm64ec-qnn.yml | 230 +++++++-------- .../templates/py-win-x64-qnn.yml | 5 +- 7 files changed, 472 insertions(+), 440 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml index 01d30d0e1ba86..28ddd29ec63e6 100644 --- a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml @@ -50,10 +50,10 @@ parameters: displayName: 'Linux packages cmake build type. Linux Only.' default: 'Release' values: - - Debug - - Release - - RelWithDebInfo - - MinSizeRel + - Debug + - Release + - RelWithDebInfo + - MinSizeRel # Only applies to QNN packages. - name: qnn_sdk_version @@ -63,17 +63,33 @@ parameters: trigger: none -stages: -- template: stages/py-cpu-packaging-stage.yml +resources: + repositories: + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks. + # For non-production pipelines, use "Unofficial" as defined below. + # For productions pipelines, use "Official". + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: - enable_linux_cpu: ${{ parameters.enable_linux_cpu }} - enable_windows_cpu: ${{ parameters.enable_windows_cpu }} - enable_mac_cpu: ${{ parameters.enable_mac_cpu }} - enable_linux_arm: ${{ parameters.enable_linux_arm }} - enable_windows_arm64_qnn: ${{ parameters.enable_windows_arm64_qnn }} - enable_windows_arm64ec_qnn: ${{ parameters.enable_windows_arm64ec_qnn }} - enable_windows_x64_qnn: ${{ parameters.enable_windows_x64_qnn }} - enable_linux_x64_qnn: ${{ parameters.enable_linux_x64_qnn }} - build_py_parameters: ${{ parameters.build_py_parameters }} - cmake_build_type: ${{ parameters.cmake_build_type }} - qnn_sdk_version: ${{ parameters.qnn_sdk_version }} + sdl: + sourceAnalysisPool: + name: onnxruntime-Win-CPU-2022 + os: windows + stages: + - template: stages/py-cpu-packaging-stage.yml + parameters: + enable_linux_cpu: ${{ parameters.enable_linux_cpu }} + enable_windows_cpu: ${{ parameters.enable_windows_cpu }} + enable_mac_cpu: ${{ parameters.enable_mac_cpu }} + enable_linux_arm: ${{ parameters.enable_linux_arm }} + enable_windows_arm64_qnn: ${{ parameters.enable_windows_arm64_qnn }} + enable_windows_arm64ec_qnn: ${{ parameters.enable_windows_arm64ec_qnn }} + enable_windows_x64_qnn: ${{ parameters.enable_windows_x64_qnn }} + enable_linux_x64_qnn: ${{ parameters.enable_linux_x64_qnn }} + build_py_parameters: ${{ parameters.build_py_parameters }} + cmake_build_type: ${{ parameters.cmake_build_type }} + qnn_sdk_version: ${{ parameters.qnn_sdk_version }} diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml index 4ff539df9f914..4ce42a90a0973 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml @@ -151,10 +151,11 @@ stages: Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishBuildArtifacts@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: - ArtifactName: onnxruntime + artifactName: onnxruntime + targetPath: '$(Build.ArtifactStagingDirectory)' - script: | 7z x *.whl @@ -199,7 +200,9 @@ stages: workspace: clean: all pool: - vmImage: 'macOS-13' + name: "Azure Pipelines" + image: "macOS-13" + os: macOS variables: MACOSX_DEPLOYMENT_TARGET: '13.3' strategy: @@ -251,74 +254,75 @@ stages: Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishBuildArtifacts@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: - ArtifactName: onnxruntime + artifactName: onnxruntime + targetPath: '$(Build.ArtifactStagingDirectory)' - template: ../templates/component-governance-component-detection-steps.yml parameters: condition: 'succeeded' - - ${{ if eq(parameters.enable_linux_arm, true) }}: - - stage: Python_Packaging_Linux_ARM - dependsOn: [] - jobs: - - template: ../templates/py-linux.yml - parameters: - arch: 'aarch64' - machine_pool: 'onnxruntime-linux-ARM64-CPU-2019' - extra_build_arg: ${{ parameters.build_py_parameters }} - cmake_build_type: ${{ parameters.cmake_build_type }} - - - ${{ if eq(parameters.enable_linux_cpu, true) }}: - - stage: Python_Packaging_Linux_CPU - dependsOn: [] - jobs: +- ${{ if eq(parameters.enable_linux_arm, true) }}: + - stage: Python_Packaging_Linux_ARM + dependsOn: [] + jobs: - template: ../templates/py-linux.yml parameters: - arch: 'x86_64' - machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU-Large' + arch: 'aarch64' + machine_pool: 'onnxruntime-linux-ARM64-CPU-2019' extra_build_arg: ${{ parameters.build_py_parameters }} cmake_build_type: ${{ parameters.cmake_build_type }} - - ${{ if eq(parameters.enable_windows_arm64_qnn, true) }}: - - stage: Python_Packaging_Windows_ARM64_QNN - dependsOn: [] - jobs: - - template: ../templates/py-win-arm64-qnn.yml +- ${{ if eq(parameters.enable_linux_cpu, true) }}: + - stage: Python_Packaging_Linux_CPU + dependsOn: [] + jobs: + - template: ../templates/py-linux.yml + parameters: + arch: 'x86_64' + machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU-Large' + extra_build_arg: ${{ parameters.build_py_parameters }} + cmake_build_type: ${{ parameters.cmake_build_type }} + +- ${{ if eq(parameters.enable_windows_arm64_qnn, true) }}: + - stage: Python_Packaging_Windows_ARM64_QNN + dependsOn: [] + jobs: + - template: ../templates/py-win-arm64-qnn.yml + parameters: + MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' + QNN_SDK: ${{ parameters.qnn_sdk_version }} + BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} + +- ${{ if eq(parameters.enable_windows_arm64ec_qnn, true) }}: + - stage: Python_Packaging_Windows_arm64ec_QNN + dependsOn: [] + jobs: + - template: ../templates/py-win-arm64ec-qnn.yml parameters: - MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' + MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' QNN_SDK: ${{ parameters.qnn_sdk_version }} BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - - ${{ if eq(parameters.enable_windows_arm64ec_qnn, true) }}: - - stage: Python_Packaging_Windows_arm64ec_QNN - dependsOn: [] - jobs: - - template: ../templates/py-win-arm64ec-qnn.yml - parameters: - MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - - - ${{ if eq(parameters.enable_windows_x64_qnn, true) }}: - - stage: Python_Packaging_Windows_x64_QNN - dependsOn: [] - jobs: - - template: ../templates/py-win-x64-qnn.yml - parameters: - MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - - - ${{ if eq(parameters.enable_linux_x64_qnn, true) }}: - - stage: Python_Packaging_Linux_x64_QNN - dependsOn: [] - jobs: - - template: ../templates/py-linux-qnn.yml +- ${{ if eq(parameters.enable_windows_x64_qnn, true) }}: + - stage: Python_Packaging_Windows_x64_QNN + dependsOn: [] + jobs: + - template: ../templates/py-win-x64-qnn.yml parameters: - machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU' - extra_build_arg: ${{ parameters.build_py_parameters }} - cmake_build_type: ${{ parameters.cmake_build_type }} + MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' + QNN_SDK: ${{ parameters.qnn_sdk_version }} + BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} + +- ${{ if eq(parameters.enable_linux_x64_qnn, true) }}: + - stage: Python_Packaging_Linux_x64_QNN + dependsOn: [] + jobs: + - template: ../templates/py-linux-qnn.yml + parameters: + machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU' + extra_build_arg: ${{ parameters.build_py_parameters }} + cmake_build_type: ${{ parameters.cmake_build_type }} diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml index 347a3145e8c70..20e212b0b1201 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml @@ -6,10 +6,10 @@ parameters: type: string default: 'Release' values: - - Debug - - Release - - RelWithDebInfo - - MinSizeRel + - Debug + - Release + - RelWithDebInfo + - MinSizeRel - name: device type: string @@ -33,62 +33,64 @@ jobs: timeoutInMinutes: 240 workspace: clean: all - pool: ${{ parameters.machine_pool }} + pool: + name: ${{ parameters.machine_pool }} + os: linux variables: - # The build machine pool doesn't have dotnet, so it can't run CG. - - name: skipComponentGovernanceDetection - value: true - - name: ORT_CACHE_DIR - value: $(Agent.TempDirectory)/ort_ccache - - name: TODAY - value: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] - - name: extra_build_args - ${{ if ne(parameters.extra_build_arg, '') }}: - value: -x ${{ parameters.extra_build_arg }} - ${{ if eq(parameters.extra_build_arg, '') }}: - value: '' + # The build machine pool doesn't have dotnet, so it can't run CG. + - name: skipComponentGovernanceDetection + value: true + - name: ORT_CACHE_DIR + value: $(Agent.TempDirectory)/ort_ccache + - name: TODAY + value: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] + - name: extra_build_args + ${{ if ne(parameters.extra_build_arg, '') }}: + value: -x ${{ parameters.extra_build_arg }} + ${{ if eq(parameters.extra_build_arg, '') }}: + value: '' steps: - - checkout: self - clean: true - submodules: none + - checkout: self + clean: true + submodules: none - - template: jobs/download_linux_qnn_sdk.yml - parameters: - QnnSDKVersion: ${{ parameters.QnnSdk }} + - template: jobs/download_linux_qnn_sdk.yml + parameters: + QnnSDKVersion: ${{ parameters.QnnSdk }} - - template: set-nightly-build-option-variable-step.yml + - template: set-nightly-build-option-variable-step.yml - - template: get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile - Context: tools/ci_build/github/linux/docker/inference/x86_64/python/cpu - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecpubuildpythonx86_64_qnn + - template: get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile + Context: tools/ci_build/github/linux/docker/inference/x86_64/python/cpu + DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" + Repository: onnxruntimecpubuildpythonx86_64_qnn - - template: linux-build-step-with-cache.yml - parameters: - WithCache: ${{parameters.with_cache}} - Today: $(TODAY) - AdditionalKey: Linux_py_qnn_Wheels_x64 - CacheDir: $(ORT_CACHE_DIR) - ChangeEveryCommit: true - BuildStep: - - task: Bash@3 - displayName: 'Build Python Wheel' - inputs: - targetType: filePath - filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh - arguments: -i onnxruntimecpubuildpythonx86_64_qnn -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args) - env: - ADDITIONAL_DOCKER_PARAMETER: "--volume $(QnnSDKRootDir):/qnn_sdk" + - template: linux-build-step-with-cache.yml + parameters: + WithCache: ${{parameters.with_cache}} + Today: $(TODAY) + AdditionalKey: Linux_py_qnn_Wheels_x64 + CacheDir: $(ORT_CACHE_DIR) + ChangeEveryCommit: true + BuildStep: + - task: Bash@3 + displayName: 'Build Python Wheel' + inputs: + targetType: filePath + filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh + arguments: -i onnxruntimecpubuildpythonx86_64_qnn -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args) + env: + ADDITIONAL_DOCKER_PARAMETER: "--volume $(QnnSDKRootDir):/qnn_sdk" - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: Linux ONNXRuntime QNN python wheel' - inputs: - PathtoPublish: '$(Build.BinariesDirectory)/dist' - ArtifactName: onnxruntime-linux-qnn-x64 + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: Linux ONNXRuntime QNN python wheel' + inputs: + targetPath: '$(Build.BinariesDirectory)/dist' + artifactName: onnxruntime-linux-qnn-x64 - - template: component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' + - template: component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml index e591b719ecfa9..eac823e9e7b78 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml @@ -9,10 +9,10 @@ parameters: type: string default: 'Release' values: - - Debug - - Release - - RelWithDebInfo - - MinSizeRel + - Debug + - Release + - RelWithDebInfo + - MinSizeRel - name: device type: string @@ -39,71 +39,75 @@ jobs: timeoutInMinutes: 240 workspace: clean: all - pool: ${{ parameters.machine_pool }} + pool: + name: ${{ parameters.machine_pool }} + os: 'linux' + ${{ if eq(parameters.arch, 'aarch64') }}: + hostArchitecture: Arm64 variables: - # The build machine pool doesn't have dotnet, so it can't run CG. - - name: skipComponentGovernanceDetection - value: true - - name: ORT_CACHE_DIR - value: $(Agent.TempDirectory)/ort_ccache - - name: TODAY - value: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] - - name: extra_build_args - ${{ if ne(parameters.extra_build_arg, '') }}: - value: '-x ${{ parameters.extra_build_arg }}' - ${{ if eq(parameters.extra_build_arg, '') }}: - value: '' - - name: python_exe_path - ${{ if ne(parameters.python_exe_path, '') }}: - value: '-p ${{ parameters.python_exe_path }}' - ${{ if eq(parameters.python_exe_path, '') }}: - value: '' + # The build machine pool doesn't have dotnet, so it can't run CG. + - name: skipComponentGovernanceDetection + value: true + - name: ORT_CACHE_DIR + value: $(Agent.TempDirectory)/ort_ccache + - name: TODAY + value: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] + - name: extra_build_args + ${{ if ne(parameters.extra_build_arg, '') }}: + value: '-x ${{ parameters.extra_build_arg }}' + ${{ if eq(parameters.extra_build_arg, '') }}: + value: '' + - name: python_exe_path + ${{ if ne(parameters.python_exe_path, '') }}: + value: '-p ${{ parameters.python_exe_path }}' + ${{ if eq(parameters.python_exe_path, '') }}: + value: '' steps: - - checkout: self - clean: true - submodules: none - - - template: set-nightly-build-option-variable-step.yml - - - template: get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cpu/Dockerfile - Context: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cpu - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecpubuildpython${{ parameters.arch }} - - - template: linux-build-step-with-cache.yml - parameters: - WithCache: ${{parameters.with_cache}} - Today: $(TODAY) - AdditionalKey: Linux_py_Wheels_${{ parameters.arch }} - CacheDir: $(ORT_CACHE_DIR) - ChangeEveryCommit: true - BuildStep: - - task: Bash@3 - displayName: 'Build Python Wheel' - inputs: - targetType: filePath - filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh - arguments: -i onnxruntimecpubuildpython${{ parameters.arch }} -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args) $(python_exe_path) - ${{ if eq(parameters.with_cache, 'true') }}: - env: - ADDITIONAL_DOCKER_PARAMETER: "--volume $(ORT_CACHE_DIR):/cache -e CCACHE_DIR=/cache -e ORT_BUILD_WITH_CACHE=1" - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - PathtoPublish: '$(Build.BinariesDirectory)/dist' - ArtifactName: onnxruntime-${{ parameters.ep }} - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Test Binaries' - inputs: - artifactName: 'drop-linux-cpu-${{ parameters.arch }}-${{ parameters.ep }}' - targetPath: '$(Build.BinariesDirectory)/${{ parameters.cmake_build_type }}' - - - template: component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' + - checkout: self + clean: true + submodules: none + + - template: set-nightly-build-option-variable-step.yml + + - template: get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cpu/Dockerfile + Context: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cpu + DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" + Repository: onnxruntimecpubuildpython${{ parameters.arch }} + + - template: linux-build-step-with-cache.yml + parameters: + WithCache: ${{parameters.with_cache}} + Today: $(TODAY) + AdditionalKey: Linux_py_Wheels_${{ parameters.arch }} + CacheDir: $(ORT_CACHE_DIR) + ChangeEveryCommit: true + BuildStep: + - task: Bash@3 + displayName: 'Build Python Wheel' + inputs: + targetType: filePath + filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh + arguments: -i onnxruntimecpubuildpython${{ parameters.arch }} -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args) $(python_exe_path) + ${{ if eq(parameters.with_cache, 'true') }}: + env: + ADDITIONAL_DOCKER_PARAMETER: "--volume $(ORT_CACHE_DIR):/cache -e CCACHE_DIR=/cache -e ORT_BUILD_WITH_CACHE=1" + + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + targetPath: '$(Build.BinariesDirectory)/dist' + artifactName: onnxruntime-${{ parameters.ep }} + + - task: 1ES.PublishPipelineArtifact@0 + displayName: 'Publish Test Binaries' + inputs: + artifactName: 'drop-linux-cpu-${{ parameters.arch }}-${{ parameters.ep }}' + targetPath: '$(Build.BinariesDirectory)/${{ parameters.cmake_build_type }}' + + - template: component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml index 4c9d0dccaf48d..18b8917711a14 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml @@ -26,6 +26,8 @@ jobs: clean: all pool: name: ${{ parameters.MACHINE_POOL }} + os: windows + hostArchitecture: Arm64 strategy: matrix: Python311_arm64: @@ -41,132 +43,133 @@ jobs: GRADLE_OPTS: '-Dorg.gradle.daemon=false' VSGenerator: 'Visual Studio 17 2022' steps: - - checkout: self - clean: true - submodules: recursive - - - template: telemetry-steps.yml - - - script: | - MKDIR $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64 - XCOPY /s /y /h /e /c /q "$(LocalPythonDir)\*.*" $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64\ - COPY NUL $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64.complete - DIR $(Agent.ToolsDirectory)\Python - DIR $(Agent.ToolsDirectory)\Python\$(PythonVersion) - DIR $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64 - displayName: Copy python $(PythonVersion) version to agent tools directory - - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PythonVersion) - addToPath: true - architecture: 'arm64' - - - task: PipAuthenticate@1 - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: 'Lotus' - - - task: onebranch.pipeline.tsaoptions@1 - displayName: 'OneBranch TSAOptions' - inputs: - tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json' - appendSourceBranchName: false - - - task: PythonScript@0 - inputs: - scriptSource: inline - script: | - import subprocess - subprocess.call(['pip', 'install', '-q', 'setuptools', 'wheel']) - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Install python modules' - - - template: set-nightly-build-option-variable-step.yml - - - template: jobs/download_win_qnn_sdk.yml - parameters: - QnnSDKVersion: ${{ parameters.QNN_SDK }} - - - task: PythonScript@0 - displayName: 'Generate cmake config' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: > - --config RelWithDebInfo - --build_dir $(Build.BinariesDirectory) - --skip_submodule_sync - --cmake_generator "$(VSGenerator)" - --build_shared_lib - --use_qnn - --qnn_home $(QnnSDKRootDir) - --enable_pybind - --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --update - $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} - workingDirectory: '$(Build.BinariesDirectory)' - - - task: VSBuild@1 - displayName: 'Build' - inputs: - solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' - platform: 'arm64' - configuration: RelWithDebInfo - msbuildArchitecture: 'arm64' - maximumCpuCount: true - logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - createLogFile: true - - # Esrp signing - - template: win-esrp-dll.yml - parameters: - FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi' - DisplayName: 'ESRP - Sign Native dlls' - DoEsrp: true - Pattern: '*.pyd' - - - task: PythonScript@0 - displayName: 'Build wheel' - inputs: - scriptPath: '$(Build.SourcesDirectory)\setup.py' - arguments: 'bdist_wheel $(NightlyBuildOption) --wheel_name_suffix=qnn' - workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' - - - task: CopyFiles@2 - displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' - inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' - Contents: '*.whl' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - ArtifactName: onnxruntime_qnn_arm64 - - - script: | - 7z x *.whl - workingDirectory: '$(Build.ArtifactStagingDirectory)' - displayName: 'unzip the package' - - - task: CredScan@3 - displayName: 'Run CredScan' - inputs: - debugMode: false - continueOnError: true - - - task: BinSkim@4 - displayName: 'Run BinSkim' - inputs: - AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll' - - - task: TSAUpload@2 - displayName: 'TSA upload' - condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) - inputs: - GdnPublishTsaOnboard: false - GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' - - - template: component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' + - checkout: self + clean: true + submodules: recursive + + - template: telemetry-steps.yml + + - script: | + MKDIR $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64 + XCOPY /s /y /h /e /c /q "$(LocalPythonDir)\*.*" $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64\ + COPY NUL $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64.complete + DIR $(Agent.ToolsDirectory)\Python + DIR $(Agent.ToolsDirectory)\Python\$(PythonVersion) + DIR $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64 + displayName: Copy python $(PythonVersion) version to agent tools directory + + - task: UsePythonVersion@0 + inputs: + versionSpec: $(PythonVersion) + addToPath: true + architecture: 'arm64' + + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'Lotus' + + - task: onebranch.pipeline.tsaoptions@1 + displayName: 'OneBranch TSAOptions' + inputs: + tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json' + appendSourceBranchName: false + + - task: PythonScript@0 + inputs: + scriptSource: inline + script: | + import subprocess + subprocess.call(['pip', 'install', '-q', 'setuptools', 'wheel']) + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' + + - template: set-nightly-build-option-variable-step.yml + + - template: jobs/download_win_qnn_sdk.yml + parameters: + QnnSDKVersion: ${{ parameters.QNN_SDK }} + + - task: PythonScript@0 + displayName: 'Generate cmake config' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: > + --config RelWithDebInfo + --build_dir $(Build.BinariesDirectory) + --skip_submodule_sync + --cmake_generator "$(VSGenerator)" + --build_shared_lib + --use_qnn + --qnn_home $(QnnSDKRootDir) + --enable_pybind + --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --update + $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} + workingDirectory: '$(Build.BinariesDirectory)' + + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' + platform: 'arm64' + configuration: RelWithDebInfo + msbuildArchitecture: 'arm64' + maximumCpuCount: true + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + createLogFile: true + + # Esrp signing + - template: win-esrp-dll.yml + parameters: + FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi' + DisplayName: 'ESRP - Sign Native dlls' + DoEsrp: true + Pattern: '*.pyd' + + - task: PythonScript@0 + displayName: 'Build wheel' + inputs: + scriptPath: '$(Build.SourcesDirectory)\setup.py' + arguments: 'bdist_wheel $(NightlyBuildOption) --wheel_name_suffix=qnn' + workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' + + - task: CopyFiles@2 + displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' + Contents: '*.whl' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + artifactName: onnxruntime_qnn_arm64 + targetPath: '$(Build.ArtifactStagingDirectory)' + + - script: | + 7z x *.whl + workingDirectory: '$(Build.ArtifactStagingDirectory)' + displayName: 'unzip the package' + + - task: CredScan@3 + displayName: 'Run CredScan' + inputs: + debugMode: false + continueOnError: true + + - task: BinSkim@4 + displayName: 'Run BinSkim' + inputs: + AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll' + + - task: TSAUpload@2 + displayName: 'TSA upload' + condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) + inputs: + GdnPublishTsaOnboard: false + GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' + + - template: component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml index ed29f1e67515e..ba2576987c8ac 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml @@ -26,6 +26,8 @@ jobs: clean: all pool: name: ${{ parameters.MACHINE_POOL }} + hostArchitecture: Arm64 + os: windows strategy: matrix: Python310_x64: @@ -40,117 +42,117 @@ jobs: GRADLE_OPTS: '-Dorg.gradle.daemon=false' VSGenerator: 'Visual Studio 17 2022' steps: - - checkout: self - clean: true - submodules: recursive - - - template: telemetry-steps.yml - - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PythonVersion) - addToPath: true - architecture: 'x64' - - - task: PipAuthenticate@1 - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: 'Lotus' - - - task: onebranch.pipeline.tsaoptions@1 - displayName: 'OneBranch TSAOptions' - inputs: - tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json' - appendSourceBranchName: fals - - - script: python -m pip install -r $(Build.SourcesDirectory)\tools\ci_build\github\linux\python\requirements.txt - - - - template: set-nightly-build-option-variable-step.yml - - - template: jobs/download_win_qnn_sdk.yml - parameters: - QnnSDKVersion: ${{ parameters.QNN_SDK }} - - - task: PythonScript@0 - displayName: 'Generate cmake config' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: > - --config RelWithDebInfo - --build_dir $(Build.BinariesDirectory) - --skip_submodule_sync - --cmake_generator "$(VSGenerator)" - --build_shared_lib - --use_qnn - --qnn_home $(QnnSDKRootDir) - --enable_pybind - --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --update --arm64ec - $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} - workingDirectory: '$(Build.BinariesDirectory)' - - - task: VSBuild@1 - displayName: 'Build' - inputs: - solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' - platform: 'arm64ec' - configuration: RelWithDebInfo - msbuildArchitecture: 'x64' - maximumCpuCount: true - logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - createLogFile: true - - # Esrp signing - - template: win-esrp-dll.yml - parameters: - FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi' - DisplayName: 'ESRP - Sign Native dlls' - DoEsrp: true - Pattern: '*.pyd' - - - task: PythonScript@0 - displayName: 'Build wheel' - inputs: - scriptPath: '$(Build.SourcesDirectory)\setup.py' - arguments: 'bdist_wheel $(NightlyBuildOption) --wheel_name_suffix=qnn' - workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' - - - task: CopyFiles@2 - displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' - inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' - Contents: '*.whl' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - ArtifactName: onnxruntime_qnn_arm64ec - - - script: | - 7z x *.whl - workingDirectory: '$(Build.ArtifactStagingDirectory)' - displayName: 'unzip the package' - - - task: CredScan@3 - displayName: 'Run CredScan' - inputs: - debugMode: false - continueOnError: true - - - task: BinSkim@4 - displayName: 'Run BinSkim' - inputs: - AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll' - - - task: TSAUpload@2 - displayName: 'TSA upload' - condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) - inputs: - GdnPublishTsaOnboard: false - GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' - - - template: component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' + - checkout: self + clean: true + submodules: recursive + + - template: telemetry-steps.yml + + - task: UsePythonVersion@0 + inputs: + versionSpec: $(PythonVersion) + addToPath: true + architecture: 'x64' + + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'Lotus' + + - task: onebranch.pipeline.tsaoptions@1 + displayName: 'OneBranch TSAOptions' + inputs: + tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json' + appendSourceBranchName: fals + + - script: python -m pip install -r $(Build.SourcesDirectory)\tools\ci_build\github\linux\python\requirements.txt + + + - template: set-nightly-build-option-variable-step.yml + + - template: jobs/download_win_qnn_sdk.yml + parameters: + QnnSDKVersion: ${{ parameters.QNN_SDK }} + + - task: PythonScript@0 + displayName: 'Generate cmake config' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: > + --config RelWithDebInfo + --build_dir $(Build.BinariesDirectory) + --skip_submodule_sync + --cmake_generator "$(VSGenerator)" + --build_shared_lib + --use_qnn + --qnn_home $(QnnSDKRootDir) + --enable_pybind + --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --update --arm64ec + $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} + workingDirectory: '$(Build.BinariesDirectory)' + + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' + platform: 'arm64ec' + configuration: RelWithDebInfo + msbuildArchitecture: 'x64' + maximumCpuCount: true + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + createLogFile: true + + # Esrp signing + - template: win-esrp-dll.yml + parameters: + FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi' + DisplayName: 'ESRP - Sign Native dlls' + DoEsrp: true + Pattern: '*.pyd' + + - task: PythonScript@0 + displayName: 'Build wheel' + inputs: + scriptPath: '$(Build.SourcesDirectory)\setup.py' + arguments: 'bdist_wheel $(NightlyBuildOption) --wheel_name_suffix=qnn' + workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' + + - task: CopyFiles@2 + displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' + Contents: '*.whl' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + artifactName: onnxruntime_qnn_arm64ec + targetPath: '$(Build.ArtifactStagingDirectory)' + - script: | + 7z x *.whl + workingDirectory: '$(Build.ArtifactStagingDirectory)' + displayName: 'unzip the package' + + - task: CredScan@3 + displayName: 'Run CredScan' + inputs: + debugMode: false + continueOnError: true + + - task: BinSkim@4 + displayName: 'Run BinSkim' + inputs: + AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll' + + - task: TSAUpload@2 + displayName: 'TSA upload' + condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) + inputs: + GdnPublishTsaOnboard: false + GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' + + - template: component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml index 13069846da342..f430582981a75 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml @@ -116,10 +116,11 @@ jobs: Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishBuildArtifacts@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: - ArtifactName: onnxruntime_qnn_x64 + artifactName: onnxruntime_qnn_x64 + targetPath: '$(Build.ArtifactStagingDirectory)' - script: | 7z x *.whl From b488312ad2bb95abebe291894bb0609dc35e4241 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Mon, 24 Feb 2025 11:22:46 -0800 Subject: [PATCH 02/18] change 1ES.PublishPipelineArtifact@0 to 1ES.PublishPipelineArtifact@1 --- tools/ci_build/github/azure-pipelines/templates/py-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml index eac823e9e7b78..d0535629a2ca4 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml @@ -102,7 +102,7 @@ jobs: targetPath: '$(Build.BinariesDirectory)/dist' artifactName: onnxruntime-${{ parameters.ep }} - - task: 1ES.PublishPipelineArtifact@0 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Test Binaries' inputs: artifactName: 'drop-linux-cpu-${{ parameters.arch }}-${{ parameters.ep }}' From 9c75e286cc82bf43bdcc107520551d072a46b0cd Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Mon, 24 Feb 2025 11:26:47 -0800 Subject: [PATCH 03/18] Make python CUDA package pipeline 1ES compliant --- .../py-cuda-publishing-pipeline.yml | 23 +++++++++++++++---- .../stages/py-cuda-publishing-stage.yml | 6 +++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml index e4499705efdf5..2ef0dc2adfc05 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml @@ -19,8 +19,23 @@ variables: value: ORT-Nightly ${{ else }}: value: onnxruntime-cuda-12 - -stages: -- template: stages/py-cuda-publishing-stage.yml +resources: + repositories: + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks. + # For non-production pipelines, use "Unofficial" as defined below. + # For productions pipelines, use "Official". + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: - artifact_feed: $(ArtifactFeed) \ No newline at end of file + sdl: + sourceAnalysisPool: + name: onnxruntime-Win-CPU-2022 + os: windows + stages: + - template: stages/py-cuda-publishing-stage.yml + parameters: + artifact_feed: $(ArtifactFeed) \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml index 1ff43667f4788..fbfbc69bce0a8 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cuda-publishing-stage.yml @@ -7,13 +7,15 @@ stages: - stage: Python_Publishing_GPU jobs: - job: Python_Publishing_GPU - pool: 'onnxruntime-Ubuntu2204-AMD-CPU' + pool: + name: 'onnxruntime-Ubuntu2204-AMD-CPU' + os: linux steps: - checkout: none - download: build displayName: 'Download Pipeline Artifact' artifact: 'whl' - + - task: UsePythonVersion@0 inputs: versionSpec: '3.13' From 293ce37c6bde9ddef2c691e67bcdc5faaca0048f Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Mon, 24 Feb 2025 11:28:19 -0800 Subject: [PATCH 04/18] combine resources --- .../azure-pipelines/py-cuda-publishing-pipeline.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml index 2ef0dc2adfc05..230c391c00ebd 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml @@ -7,7 +7,11 @@ resources: include: - main branch: main - + repositories: + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release parameters: - name: isReleaseBuild type: boolean @@ -19,12 +23,7 @@ variables: value: ORT-Nightly ${{ else }}: value: onnxruntime-cuda-12 -resources: - repositories: - - repository: 1esPipelines - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release + extends: # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks. # For non-production pipelines, use "Unofficial" as defined below. From ca0aef2b56dbd609a9fda8d0567b68edf5375c18 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 26 Feb 2025 10:32:46 -0800 Subject: [PATCH 05/18] _qnn_Wheels --- .../github/azure-pipelines/stages/py-cpu-packaging-stage.yml | 4 ++-- .../github/azure-pipelines/templates/py-win-arm64-qnn.yml | 2 +- .../github/azure-pipelines/templates/py-win-arm64ec-qnn.yml | 1 - .../github/azure-pipelines/templates/py-win-x64-qnn.yml | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml index 4ce42a90a0973..da0e6b368c2ee 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml @@ -154,7 +154,7 @@ stages: - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: - artifactName: onnxruntime + artifactName: onnxruntime-win-$(PythonVersion) targetPath: '$(Build.ArtifactStagingDirectory)' - script: | @@ -257,7 +257,7 @@ stages: - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: - artifactName: onnxruntime + artifactName: onnxruntime-macos-$(PythonVersion) targetPath: '$(Build.ArtifactStagingDirectory)' - template: ../templates/component-governance-component-detection-steps.yml diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml index 18b8917711a14..0e3bcd68a774d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml @@ -144,7 +144,7 @@ jobs: - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: - artifactName: onnxruntime_qnn_arm64 + artifactName: onnxruntime_qnn_arm64_$(PythonVersion) targetPath: '$(Build.ArtifactStagingDirectory)' - script: | diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml index ba2576987c8ac..0bf52271a5e3f 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml @@ -26,7 +26,6 @@ jobs: clean: all pool: name: ${{ parameters.MACHINE_POOL }} - hostArchitecture: Arm64 os: windows strategy: matrix: diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml index f430582981a75..0234d36db93fb 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml @@ -119,7 +119,7 @@ jobs: - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: - artifactName: onnxruntime_qnn_x64 + artifactName: onnxruntime_qnn_x64_$(PythonVersion) targetPath: '$(Build.ArtifactStagingDirectory)' - script: | From 8b11ece40a964996676e06a048588ab8dd258adc Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 26 Feb 2025 10:34:55 -0800 Subject: [PATCH 06/18] _qnn_Wheels --- .../github/azure-pipelines/templates/py-win-arm64ec-qnn.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml index 0bf52271a5e3f..a6c04a665e70f 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml @@ -127,7 +127,7 @@ jobs: - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: - artifactName: onnxruntime_qnn_arm64ec + artifactName: onnxruntime_qnn_arm64ec_$(PythonVersion) targetPath: '$(Build.ArtifactStagingDirectory)' - script: | 7z x *.whl From 60f74902507590dcf9905bbadb2870c0990b2052 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 26 Feb 2025 11:05:07 -0800 Subject: [PATCH 07/18] _qnn_Wheels --- .../azure-pipelines/stages/nuget-cuda-packaging-stage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml index 8fabb80a73869..d73cc8c9ec939 100644 --- a/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml @@ -32,7 +32,7 @@ stages: - job: NuGet_Packaging_GPU workspace: clean: all - pool: 'Onnxruntime-Win-CPU-2022' + pool: 'onnxruntime-Win-CPU-2022' variables: breakCodesignValidationInjection: ${{ parameters.DoEsrp }} ReleaseVersionSuffix: $[stageDependencies.Setup.Set_Variables.outputs['Set_Release_Version_Suffix.ReleaseVersionSuffix']] From b12a3853e5fb8acaef3c3efe7ab73368235f213a Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 26 Feb 2025 11:11:40 -0800 Subject: [PATCH 08/18] _qnn_Wheels --- .../azure-pipelines/stages/nuget-cuda-packaging-stage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml index d73cc8c9ec939..8fabb80a73869 100644 --- a/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-packaging-stage.yml @@ -32,7 +32,7 @@ stages: - job: NuGet_Packaging_GPU workspace: clean: all - pool: 'onnxruntime-Win-CPU-2022' + pool: 'Onnxruntime-Win-CPU-2022' variables: breakCodesignValidationInjection: ${{ parameters.DoEsrp }} ReleaseVersionSuffix: $[stageDependencies.Setup.Set_Variables.outputs['Set_Release_Version_Suffix.ReleaseVersionSuffix']] From e5e4a811e0d7786b3371004d03dbed0e3411a285 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 26 Feb 2025 12:38:20 -0800 Subject: [PATCH 09/18] onnxruntime-Win-CPU-2022 --- tools/ci_build/github/azure-pipelines/templates/py-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml index d0535629a2ca4..8db7681732d8f 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml @@ -100,7 +100,7 @@ jobs: displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: targetPath: '$(Build.BinariesDirectory)/dist' - artifactName: onnxruntime-${{ parameters.ep }} + artifactName: onnxruntime-${{ parameters.ep }}-${{ parameters.arch }} - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Test Binaries' From 790f3c7fb7c70bf857bd2b8a0bc3753bde39a475 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Thu, 27 Feb 2025 14:51:22 -0800 Subject: [PATCH 10/18] 1es --- .../templates/py-linux-qnn.yml | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml index 20e212b0b1201..7d2b53b2b4746 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml @@ -27,6 +27,10 @@ parameters: displayName: QNN SDK version type: string default: 2.31.0.250130 +- name: is1ES + displayName: 'Whether the pipeline is running in 1ES' + type: boolean + default: false jobs: - job: Linux_py_qnn_Wheels_x64 @@ -84,12 +88,19 @@ jobs: arguments: -i onnxruntimecpubuildpythonx86_64_qnn -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args) env: ADDITIONAL_DOCKER_PARAMETER: "--volume $(QnnSDKRootDir):/qnn_sdk" + - ${{ if eq(parameters.is1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: Linux ONNXRuntime QNN python wheel' + inputs: + targetPath: '$(Build.BinariesDirectory)/dist' + artifactName: onnxruntime-linux-qnn-x64 - - task: 1ES.PublishPipelineArtifact@1 - displayName: 'Publish Artifact: Linux ONNXRuntime QNN python wheel' - inputs: - targetPath: '$(Build.BinariesDirectory)/dist' - artifactName: onnxruntime-linux-qnn-x64 + - ${{ if eq(parameters.is1ES, false) }}: + - task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: Linux ONNXRuntime QNN python wheel' + inputs: + targetPath: '$(Build.BinariesDirectory)/dist' + artifactName: onnxruntime-linux-qnn-x64 - template: component-governance-component-detection-steps.yml parameters: From 7592d1605432cebae6ef978bd170b5c24c95d324 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Thu, 27 Feb 2025 14:57:39 -0800 Subject: [PATCH 11/18] 1es --- .../stages/py-cpu-packaging-stage.yml | 6 +++++ .../templates/py-linux-qnn.yml | 1 + .../azure-pipelines/templates/py-linux.yml | 22 ++++++++++++++----- .../templates/py-win-arm64-qnn.yml | 22 ++++++++++++++----- .../templates/py-win-arm64ec-qnn.yml | 22 ++++++++++++++----- .../templates/py-win-x64-qnn.yml | 22 ++++++++++++++----- 6 files changed, 75 insertions(+), 20 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml index da0e6b368c2ee..c196bab734ed1 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml @@ -275,6 +275,7 @@ stages: machine_pool: 'onnxruntime-linux-ARM64-CPU-2019' extra_build_arg: ${{ parameters.build_py_parameters }} cmake_build_type: ${{ parameters.cmake_build_type }} + is1ES: true - ${{ if eq(parameters.enable_linux_cpu, true) }}: - stage: Python_Packaging_Linux_CPU @@ -286,6 +287,7 @@ stages: machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU-Large' extra_build_arg: ${{ parameters.build_py_parameters }} cmake_build_type: ${{ parameters.cmake_build_type }} + is1ES: true - ${{ if eq(parameters.enable_windows_arm64_qnn, true) }}: - stage: Python_Packaging_Windows_ARM64_QNN @@ -296,6 +298,7 @@ stages: MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' QNN_SDK: ${{ parameters.qnn_sdk_version }} BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} + is1ES: true - ${{ if eq(parameters.enable_windows_arm64ec_qnn, true) }}: - stage: Python_Packaging_Windows_arm64ec_QNN @@ -306,6 +309,7 @@ stages: MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' QNN_SDK: ${{ parameters.qnn_sdk_version }} BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} + is1ES: true - ${{ if eq(parameters.enable_windows_x64_qnn, true) }}: - stage: Python_Packaging_Windows_x64_QNN @@ -316,6 +320,7 @@ stages: MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' QNN_SDK: ${{ parameters.qnn_sdk_version }} BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} + is1ES: true - ${{ if eq(parameters.enable_linux_x64_qnn, true) }}: - stage: Python_Packaging_Linux_x64_QNN @@ -326,3 +331,4 @@ stages: machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU' extra_build_arg: ${{ parameters.build_py_parameters }} cmake_build_type: ${{ parameters.cmake_build_type }} + is1ES: true diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml index 7d2b53b2b4746..8126cda449daa 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml @@ -27,6 +27,7 @@ parameters: displayName: QNN SDK version type: string default: 2.31.0.250130 + - name: is1ES displayName: 'Whether the pipeline is running in 1ES' type: boolean diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml index 8db7681732d8f..abe95b6341295 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml @@ -34,6 +34,11 @@ parameters: type: string default: '' +- name: is1ES + displayName: 'Whether the pipeline is running in 1ES' + type: boolean + default: false + jobs: - job: Linux_py_Wheels_${{ parameters.arch }}_${{parameters.ep}} timeoutInMinutes: 240 @@ -96,11 +101,18 @@ jobs: env: ADDITIONAL_DOCKER_PARAMETER: "--volume $(ORT_CACHE_DIR):/cache -e CCACHE_DIR=/cache -e ORT_BUILD_WITH_CACHE=1" - - task: 1ES.PublishPipelineArtifact@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - targetPath: '$(Build.BinariesDirectory)/dist' - artifactName: onnxruntime-${{ parameters.ep }}-${{ parameters.arch }} + - ${{ if eq(parameters.is1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + targetPath: '$(Build.BinariesDirectory)/dist' + artifactName: onnxruntime-${{ parameters.ep }}-${{ parameters.arch }} + - ${{ if eq(parameters.is1ES, false) }}: + - task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + targetPath: '$(Build.BinariesDirectory)/dist' + artifactName: onnxruntime-${{ parameters.ep }}-${{ parameters.arch }} - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Test Binaries' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml index 0e3bcd68a774d..10ea7f6203bb1 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml @@ -19,6 +19,11 @@ parameters: type: string default: '' +- name: is1ES + displayName: 'Whether the pipeline is running in 1ES' + type: boolean + default: false + jobs: - job: Win_py_arm64_qnn_Wheels timeoutInMinutes: 210 @@ -141,11 +146,18 @@ jobs: Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: 1ES.PublishPipelineArtifact@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - artifactName: onnxruntime_qnn_arm64_$(PythonVersion) - targetPath: '$(Build.ArtifactStagingDirectory)' + - ${{ if eq(parameters.is1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + artifactName: onnxruntime_qnn_arm64_$(PythonVersion) + targetPath: '$(Build.ArtifactStagingDirectory)' + - ${{ if eq(parameters.is1ES, false) }}: + - task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + input: + artifactName: onnxruntime_qnn_arm64_$(PythonVersion) + targetPath: '$(Build.ArtifactStagingDirectory)' - script: | 7z x *.whl diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml index a6c04a665e70f..24321d2a3e1ec 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml @@ -19,6 +19,11 @@ parameters: type: string default: '' +- name: is1ES + displayName: 'Whether the pipeline is running in 1ES' + type: boolean + default: false + jobs: - job: Win_py_x64_qnn_Wheels timeoutInMinutes: 210 @@ -124,11 +129,18 @@ jobs: Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: 1ES.PublishPipelineArtifact@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - artifactName: onnxruntime_qnn_arm64ec_$(PythonVersion) - targetPath: '$(Build.ArtifactStagingDirectory)' + - ${{ if eq(parameters.is1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + artifactName: onnxruntime_qnn_arm64ec_$(PythonVersion) + targetPath: '$(Build.ArtifactStagingDirectory)' + - ${{ if eq(parameters.is1ES, false) }}: + - task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + artifactName: onnxruntime_qnn_arm64ec_$(PythonVersion) + targetPath: '$(Build.ArtifactStagingDirectory)' - script: | 7z x *.whl workingDirectory: '$(Build.ArtifactStagingDirectory)' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml index 0234d36db93fb..175b343e55d57 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml @@ -19,6 +19,11 @@ parameters: type: string default: '' +- name: is1ES + displayName: 'Whether the pipeline is running in 1ES' + type: boolean + default: false + jobs: - job: Win_py_x64_qnn_Wheels timeoutInMinutes: 210 @@ -116,11 +121,18 @@ jobs: Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: 1ES.PublishPipelineArtifact@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - artifactName: onnxruntime_qnn_x64_$(PythonVersion) - targetPath: '$(Build.ArtifactStagingDirectory)' + - ${{ if eq(parameters.is1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + artifactName: onnxruntime_qnn_x64_$(PythonVersion) + targetPath: '$(Build.ArtifactStagingDirectory)' + - ${{ if eq(parameters.is1ES, false) }}: + - task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + artifactName: onnxruntime_qnn_x64_$(PythonVersion) + targetPath: '$(Build.ArtifactStagingDirectory)' - script: | 7z x *.whl From e58d5307644f298bd5a517a46721cea11c65c397 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Thu, 27 Feb 2025 15:02:44 -0800 Subject: [PATCH 12/18] 1es --- .../azure-pipelines/templates/py-linux.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml index abe95b6341295..d7e7e8be7b7d7 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml @@ -107,18 +107,24 @@ jobs: inputs: targetPath: '$(Build.BinariesDirectory)/dist' artifactName: onnxruntime-${{ parameters.ep }}-${{ parameters.arch }} + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Test Binaries' + inputs: + artifactName: 'drop-linux-cpu-${{ parameters.arch }}-${{ parameters.ep }}' + targetPath: '$(Build.BinariesDirectory)/${{ parameters.cmake_build_type }}' - ${{ if eq(parameters.is1ES, false) }}: - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: targetPath: '$(Build.BinariesDirectory)/dist' artifactName: onnxruntime-${{ parameters.ep }}-${{ parameters.arch }} + - task: PublishPipelineArtifact@1 + displayName: 'Publish Test Binaries' + inputs: + artifactName: 'drop-linux-cpu-${{ parameters.arch }}-${{ parameters.ep }}' + targetPath: '$(Build.BinariesDirectory)/${{ parameters.cmake_build_type }}' + - - task: 1ES.PublishPipelineArtifact@1 - displayName: 'Publish Test Binaries' - inputs: - artifactName: 'drop-linux-cpu-${{ parameters.arch }}-${{ parameters.ep }}' - targetPath: '$(Build.BinariesDirectory)/${{ parameters.cmake_build_type }}' - template: component-governance-component-detection-steps.yml parameters: From b5cfcb7b916fb5abc0db73caa3600c3718ff7872 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Mon, 3 Mar 2025 11:34:38 -0800 Subject: [PATCH 13/18] onnxruntime-${{ parameters.arch }}-${{ parameters.ep }} --- tools/ci_build/github/azure-pipelines/templates/py-linux.yml | 4 ++-- .../azure-pipelines/templates/py-packaging-linux-test-cpu.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml index d7e7e8be7b7d7..8d0c4334f4874 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml @@ -106,7 +106,7 @@ jobs: displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: targetPath: '$(Build.BinariesDirectory)/dist' - artifactName: onnxruntime-${{ parameters.ep }}-${{ parameters.arch }} + artifactName: onnxruntime-${{ parameters.arch }}-${{ parameters.ep }} - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Test Binaries' inputs: @@ -117,7 +117,7 @@ jobs: displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: targetPath: '$(Build.BinariesDirectory)/dist' - artifactName: onnxruntime-${{ parameters.ep }}-${{ parameters.arch }} + artifactName: onnxruntime-${{ parameters.arch }}-${{ parameters.ep }} - task: PublishPipelineArtifact@1 displayName: 'Publish Test Binaries' inputs: diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml index c475feaef0018..cbbaca95f4e01 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml @@ -50,7 +50,7 @@ jobs: artifact: 'drop-linux-cpu-${{ parameters.arch }}-${{parameters.ep}}' - download: current # pipeline resource identifier. - artifact: 'onnxruntime${{ parameters.python_wheel_suffix }}-${{ parameters.ep }}' + artifact: 'onnxruntime-${{ parameters.arch }}-${{ parameters.ep }}' - bash: | set -e -x @@ -64,7 +64,7 @@ jobs: artifact: 'drop-linux-cpu-${{ parameters.arch }}-${{parameters.ep}}' - download: build # pipeline resource identifier. - artifact: 'onnxruntime${{ parameters.python_wheel_suffix }}-${{ parameters.ep }}' + artifact: 'onnxruntime-${{ parameters.arch }}-${{ parameters.ep }}' - bash: | set -e -x From c07b7915282af9357f7df83e1ce7113335863820 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Mon, 3 Mar 2025 12:14:29 -0800 Subject: [PATCH 14/18] name: 'Azure Pipelines' --- .../azure-pipelines/templates/c-api-cpu.yml | 65 ++++++++++--------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index a6fe5ac27749b..8a2943431fc12 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -123,7 +123,10 @@ stages: workspace: clean: all pool: - vmImage: 'macOS-13' + name: 'Azure Pipelines' + image: 'macOS-13' + os: 'macOS' + timeoutInMinutes: 300 steps: - template: set-version-number-variables-step.yml @@ -289,8 +292,8 @@ stages: artifact: 'onnxruntime-java' - template: component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' + parameters: + condition: 'succeeded' - stage: NuGet_Packaging_CPU dependsOn: @@ -312,8 +315,8 @@ stages: OrtPackageId: ${{ parameters.OrtNugetPackageId }} breakCodesignValidationInjection: ${{ parameters.DoEsrp }} ReleaseVersionSuffix: $[stageDependencies.Setup.Set_Variables.outputs['Set_Release_Version_Suffix.ReleaseVersionSuffix']] - BuildDate : $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Date.BuildDate']] - BuildTime : $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Time.BuildTime']] + BuildDate: $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Date.BuildDate']] + BuildTime: $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Time.BuildTime']] steps: - checkout: self @@ -384,7 +387,7 @@ stages: BuildId: ${{ parameters.BuildId }} - script: | - dir + dir workingDirectory: '$(Build.BinariesDirectory)/nuget-artifact' displayName: 'List artifacts' @@ -398,7 +401,7 @@ stages: filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_nuget_files.ps1 - script: | - mklink /D /J models C:\local\models + mklink /D /J models C:\local\models workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Create models link' - task: NuGetToolInstaller@0 @@ -505,8 +508,8 @@ stages: workingDirectory: '$(Build.SourcesDirectory)\csharp' - template: component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' + parameters: + condition: 'succeeded' - stage: Nodejs_Packaging @@ -537,11 +540,11 @@ stages: submodules: true - script: | - echo.>>.gitattributes - echo /js/** text=auto eol=lf>>.gitattributes - rd /s /q js - git checkout -- js/** - git checkout -- .gitattributes + echo.>>.gitattributes + echo /js/** text=auto eol=lf>>.gitattributes + rd /s /q js + git checkout -- js/** + git checkout -- .gitattributes workingDirectory: '$(Build.SourcesDirectory)' displayName: 'Testing: force EOL to lf on windows for /js/**' @@ -650,20 +653,20 @@ stages: filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_nuget_files.ps1 - script: | - dir + dir workingDirectory: '$(Build.BinariesDirectory)/nuget-artifact' displayName: 'List artifacts' - script: | - npm ci + npm ci workingDirectory: '$(Build.SourcesDirectory)/js' displayName: 'Install NPM packages /js' - script: | - npm ci + npm ci workingDirectory: '$(Build.SourcesDirectory)/js/common' displayName: 'Install NPM packages /js/common' - script: | - npm ci + npm ci workingDirectory: '$(Build.SourcesDirectory)/js/node' displayName: 'Install NPM packages /js/node' @@ -774,45 +777,45 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)' - template: component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' + parameters: + condition: 'succeeded' - template: ../nuget/templates/test_win.yml parameters: - AgentPool : 'onnxruntime-Win-CPU-2022' - Skipx86Tests : false - NugetPackageName : 'Microsoft.ML.OnnxRuntime' + AgentPool: 'onnxruntime-Win-CPU-2022' + Skipx86Tests: false + NugetPackageName: 'Microsoft.ML.OnnxRuntime' ArtifactSuffix: 'CPU' SpecificArtifact: ${{ parameters.SpecificArtifact }} BuildId: ${{ parameters.BuildId }} - template: ../nuget/templates/test_linux.yml parameters: - AgentPool : onnxruntime-Ubuntu2204-AMD-CPU - NugetPackageName : 'Microsoft.ML.OnnxRuntime' + AgentPool: onnxruntime-Ubuntu2204-AMD-CPU + NugetPackageName: 'Microsoft.ML.OnnxRuntime' ArtifactSuffix: 'CPU' SpecificArtifact: ${{ parameters.SpecificArtifact }} BuildId: ${{ parameters.BuildId }} - template: ../nuget/templates/test_macos.yml parameters: - AgentPool : macOS-13 + AgentPool: macOS-13 ArtifactSuffix: 'CPU' - template: ../nodejs/templates/test_win.yml parameters: - AgentPool : 'onnxruntime-Win-CPU-2022' - StageSuffix : 'Win_CPU_x64' + AgentPool: 'onnxruntime-Win-CPU-2022' + StageSuffix: 'Win_CPU_x64' - template: ../nodejs/templates/test_linux.yml parameters: - AgentPool : 'Azure-Pipelines-EO-Ubuntu-2004-aiinfra' - StageSuffix : 'Linux_CPU_x64' + AgentPool: 'Azure-Pipelines-EO-Ubuntu-2004-aiinfra' + StageSuffix: 'Linux_CPU_x64' - template: ../nodejs/templates/test_macos.yml parameters: - StageSuffix : 'macOS_CPU_x64' + StageSuffix: 'macOS_CPU_x64' - template: final-jar-testing.yml parameters: From 8545608e1ee60cf6d8458e3300dd489e71955f1f Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Mon, 3 Mar 2025 12:23:42 -0800 Subject: [PATCH 15/18] undo --- .../azure-pipelines/templates/c-api-cpu.yml | 65 +++++++++---------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index 8a2943431fc12..a6fe5ac27749b 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -123,10 +123,7 @@ stages: workspace: clean: all pool: - name: 'Azure Pipelines' - image: 'macOS-13' - os: 'macOS' - + vmImage: 'macOS-13' timeoutInMinutes: 300 steps: - template: set-version-number-variables-step.yml @@ -292,8 +289,8 @@ stages: artifact: 'onnxruntime-java' - template: component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' + parameters : + condition : 'succeeded' - stage: NuGet_Packaging_CPU dependsOn: @@ -315,8 +312,8 @@ stages: OrtPackageId: ${{ parameters.OrtNugetPackageId }} breakCodesignValidationInjection: ${{ parameters.DoEsrp }} ReleaseVersionSuffix: $[stageDependencies.Setup.Set_Variables.outputs['Set_Release_Version_Suffix.ReleaseVersionSuffix']] - BuildDate: $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Date.BuildDate']] - BuildTime: $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Time.BuildTime']] + BuildDate : $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Date.BuildDate']] + BuildTime : $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Time.BuildTime']] steps: - checkout: self @@ -387,7 +384,7 @@ stages: BuildId: ${{ parameters.BuildId }} - script: | - dir + dir workingDirectory: '$(Build.BinariesDirectory)/nuget-artifact' displayName: 'List artifacts' @@ -401,7 +398,7 @@ stages: filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_nuget_files.ps1 - script: | - mklink /D /J models C:\local\models + mklink /D /J models C:\local\models workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Create models link' - task: NuGetToolInstaller@0 @@ -508,8 +505,8 @@ stages: workingDirectory: '$(Build.SourcesDirectory)\csharp' - template: component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' + parameters : + condition : 'succeeded' - stage: Nodejs_Packaging @@ -540,11 +537,11 @@ stages: submodules: true - script: | - echo.>>.gitattributes - echo /js/** text=auto eol=lf>>.gitattributes - rd /s /q js - git checkout -- js/** - git checkout -- .gitattributes + echo.>>.gitattributes + echo /js/** text=auto eol=lf>>.gitattributes + rd /s /q js + git checkout -- js/** + git checkout -- .gitattributes workingDirectory: '$(Build.SourcesDirectory)' displayName: 'Testing: force EOL to lf on windows for /js/**' @@ -653,20 +650,20 @@ stages: filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_nuget_files.ps1 - script: | - dir + dir workingDirectory: '$(Build.BinariesDirectory)/nuget-artifact' displayName: 'List artifacts' - script: | - npm ci + npm ci workingDirectory: '$(Build.SourcesDirectory)/js' displayName: 'Install NPM packages /js' - script: | - npm ci + npm ci workingDirectory: '$(Build.SourcesDirectory)/js/common' displayName: 'Install NPM packages /js/common' - script: | - npm ci + npm ci workingDirectory: '$(Build.SourcesDirectory)/js/node' displayName: 'Install NPM packages /js/node' @@ -777,45 +774,45 @@ stages: targetPath: '$(Build.ArtifactStagingDirectory)' - template: component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' + parameters : + condition : 'succeeded' - template: ../nuget/templates/test_win.yml parameters: - AgentPool: 'onnxruntime-Win-CPU-2022' - Skipx86Tests: false - NugetPackageName: 'Microsoft.ML.OnnxRuntime' + AgentPool : 'onnxruntime-Win-CPU-2022' + Skipx86Tests : false + NugetPackageName : 'Microsoft.ML.OnnxRuntime' ArtifactSuffix: 'CPU' SpecificArtifact: ${{ parameters.SpecificArtifact }} BuildId: ${{ parameters.BuildId }} - template: ../nuget/templates/test_linux.yml parameters: - AgentPool: onnxruntime-Ubuntu2204-AMD-CPU - NugetPackageName: 'Microsoft.ML.OnnxRuntime' + AgentPool : onnxruntime-Ubuntu2204-AMD-CPU + NugetPackageName : 'Microsoft.ML.OnnxRuntime' ArtifactSuffix: 'CPU' SpecificArtifact: ${{ parameters.SpecificArtifact }} BuildId: ${{ parameters.BuildId }} - template: ../nuget/templates/test_macos.yml parameters: - AgentPool: macOS-13 + AgentPool : macOS-13 ArtifactSuffix: 'CPU' - template: ../nodejs/templates/test_win.yml parameters: - AgentPool: 'onnxruntime-Win-CPU-2022' - StageSuffix: 'Win_CPU_x64' + AgentPool : 'onnxruntime-Win-CPU-2022' + StageSuffix : 'Win_CPU_x64' - template: ../nodejs/templates/test_linux.yml parameters: - AgentPool: 'Azure-Pipelines-EO-Ubuntu-2004-aiinfra' - StageSuffix: 'Linux_CPU_x64' + AgentPool : 'Azure-Pipelines-EO-Ubuntu-2004-aiinfra' + StageSuffix : 'Linux_CPU_x64' - template: ../nodejs/templates/test_macos.yml parameters: - StageSuffix: 'macOS_CPU_x64' + StageSuffix : 'macOS_CPU_x64' - template: final-jar-testing.yml parameters: From 113104a2b26b663da5055fd9867f73088db64f01 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Mon, 3 Mar 2025 14:01:41 -0800 Subject: [PATCH 16/18] displayName: 'Move the artifacts to the binaries directory' --- .../templates/py-packaging-linux-test-cpu.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml index cbbaca95f4e01..eef97341b8d53 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml @@ -19,10 +19,10 @@ parameters: type: string default: 'Release' values: - - Debug - - Release - - RelWithDebInfo - - MinSizeRel + - Debug + - Release + - RelWithDebInfo + - MinSizeRel - name: timeout type: number @@ -55,9 +55,10 @@ jobs: - bash: | set -e -x mv "$(Pipeline.Workspace)/drop-linux-cpu-${{ parameters.arch }}-${{parameters.ep}}" $(Build.BinariesDirectory)/${{parameters.cmake_build_type}} - mv "$(Pipeline.Workspace)/onnxruntime${{ parameters.python_wheel_suffix }}-${{parameters.ep}}" "$(Build.BinariesDirectory)/whl" + mv "$(Pipeline.Workspace)/onnxruntime-${{ parameters.arch }}-${{ parameters.ep }}" "$(Build.BinariesDirectory)/whl" cp -r "$(Build.BinariesDirectory)/whl" $(Build.BinariesDirectory)/tmp find "$(Build.BinariesDirectory)/tmp" -name '*.whl' -exec bash -c 'unzip -d "${1%.*}" "$1"' _ {} \; + displayName: 'Move the artifacts to the binaries directory' # The private ADO project - ${{ if eq(variables['System.CollectionId'], 'bc038106-a83b-4dab-9dd3-5a41bc58f34c') }}: - download: build # pipeline resource identifier. @@ -70,9 +71,10 @@ jobs: set -e -x ls $(Pipeline.Workspace)/build mv "$(Pipeline.Workspace)/build/drop-linux-cpu-${{ parameters.arch }}-${{parameters.ep}}" $(Build.BinariesDirectory)/${{parameters.cmake_build_type}} - mv "$(Pipeline.Workspace)/build/onnxruntime${{ parameters.python_wheel_suffix }}-${{parameters.ep}}" "$(Build.BinariesDirectory)/whl" + mv "$(Pipeline.Workspace)/build/onnxruntime-${{ parameters.arch }}-${{ parameters.ep }}" "$(Build.BinariesDirectory)/whl" cp -r "$(Build.BinariesDirectory)/whl" $(Build.BinariesDirectory)/tmp find "$(Build.BinariesDirectory)/tmp" -name '*.whl' -exec bash -c 'unzip -d "${1%.*}" "$1"' _ {} \; + displayName: 'Move the artifacts to the binaries directory' # The BinSkim task uses a dotnet program which doesn't support ARM CPUs yet - ${{ if eq(parameters.arch, 'x86_64') }}: From bb5c691d855b3353fe5935a4a1dc15296a55ba0a Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Tue, 4 Mar 2025 10:40:03 -0800 Subject: [PATCH 17/18] Update macOS --- .../azure-pipelines/templates/react-native-ci.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml b/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml index 7991916a47ca4..52dbb76632e0c 100644 --- a/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml @@ -62,10 +62,14 @@ stages: dependsOn: '${{parameters.InitialStageDependsOn}}' jobs: - job: ReactNative_CI_iOS - pool: - name: 'Azure Pipelines' - image: 'macOS-13' - os: 'macOS' + ${{ if eq(parameters.is1ES, false) }}: + pool: + vmImage: 'macOS-13' + ${{ if eq(parameters.is1ES, true) }}: + pool: + name: 'Azure Pipelines' + image: 'macOS-13' + os: 'macOS' timeoutInMinutes: 120 From d9ba389b5d24eba96550bf1baec9414e223c04fc Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Tue, 4 Mar 2025 19:40:20 -0800 Subject: [PATCH 18/18] Update python package stages --- .../azure-pipelines/py-package-test-pipeline.yml | 2 ++ .../templates/py-package-smoking-test.yml | 11 +++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml index a0e49692220f9..7a78c6ba0fcdf 100644 --- a/tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml @@ -31,10 +31,12 @@ stages: machine_pool: vmImage: 'macOS-13' itemPattern: '*/*mac*x86_64.whl' + arch: 'x86_64' - template: templates/py-package-smoking-test.yml parameters: job_name: Test_LINUX_x86_64_Wheels itemPattern: '*/*manylinux*x86_64.whl' + arch: 'x86_64' machine_pool: name: 'onnxruntime-Ubuntu2204-AMD-CPU' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-package-smoking-test.yml b/tools/ci_build/github/azure-pipelines/templates/py-package-smoking-test.yml index 3a3da0f8f5afa..2e21f59752f16 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-package-smoking-test.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-package-smoking-test.yml @@ -9,9 +9,13 @@ parameters: - name: machine_pool type: object -- name: python_arch +- name: ep type: string - default: 'x64' + default: 'cpu' + +- name: arch + type: string + default: 'x86_64' jobs: - job: ${{ parameters.job_name }} @@ -37,10 +41,9 @@ jobs: displayName: 'Use Python' inputs: versionSpec: $(PythonVersion) - architecture: ${{ parameters.python_arch }} - download: build # pipeline resource identifier. - artifact: 'onnxruntime' + artifact: 'onnxruntime-${{ parameters.arch }}-${{ parameters.ep }}' - task: Bash@3 inputs: