From 025af433c08cd8c61a0b50e7c4c92ebc5c35328b Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Tue, 26 Aug 2025 14:45:14 -0700 Subject: [PATCH 1/3] Refactor extract_nuget_files.ps1 for clarity and robustness --- .../github/windows/extract_nuget_files.ps1 | 162 ++++++++++-------- 1 file changed, 95 insertions(+), 67 deletions(-) diff --git a/tools/ci_build/github/windows/extract_nuget_files.ps1 b/tools/ci_build/github/windows/extract_nuget_files.ps1 index ff8f63a85b97a..b7f76746ea731 100644 --- a/tools/ci_build/github/windows/extract_nuget_files.ps1 +++ b/tools/ci_build/github/windows/extract_nuget_files.ps1 @@ -1,105 +1,133 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -# This file is used by Zip-Nuget Packaging NoContribOps Pipeline,Zip-Nuget-Java Packaging Pipeline +# This file is used by Zip-Nuget-Java Packaging Pipeline -# Re-construct a build directory that contains binaries from all the different platforms we're including -# in the native ORT nuget package +# Define the directory for NuGet artifacts. $nuget_artifacts_dir = "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts" -New-Item -Path $nuget_artifacts_dir -ItemType directory +# Create the directory if it doesn't exist. +New-Item -Path $nuget_artifacts_dir -ItemType directory -ErrorAction SilentlyContinue ## .zip files -# unzip directly -# exclude the iOS xcframework as we need to leave that zipped up to preserve symlinks -Get-ChildItem -Path $Env:BUILD_BINARIESDIRECTORY\nuget-artifact\* -Include *.zip -Exclude onnxruntime_ios_xcframework.*.zip | +# Unzip files directly, excluding the iOS xcframework to preserve its symlinks. +Get-ChildItem -Path "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact\*" -Include *.zip -Exclude onnxruntime_ios_xcframework.*.zip | Foreach-Object { - $cmd = "7z.exe x $($_.FullName) -y -o$nuget_artifacts_dir" - Write-Output $cmd - Invoke-Expression -Command $cmd + $arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir" + Write-Output "Executing: 7z.exe $arguments" + # Directly call 7z.exe using the call operator '&' + & 7z.exe $arguments + # Check the exit code of the last command. A non-zero code indicates an error. + if ($LASTEXITCODE -ne 0) { + throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE" + } } ## .tgz files -# first extract the tar file from the tgz -Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tgz | +# First, extract the .tar file from the .tgz archive. +Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.tgz | Foreach-Object { - $cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" - Write-Output $cmd - Invoke-Expression -Command $cmd + $arguments = "x", "$($_.FullName)", "-y", "-o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" + Write-Output "Executing: 7z.exe $arguments" + & 7z.exe $arguments + if ($LASTEXITCODE -ne 0) { + throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE" + } } -# now extract the actual folder structure from the tar file to the build dir -Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tar | +# Now, extract the contents from the .tar file into the final directory. +Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.tar | Foreach-Object { - $cmd = "7z.exe x $($_.FullName) -y -o$nuget_artifacts_dir" - Write-Output $cmd - Invoke-Expression -Command $cmd + $arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir" + Write-Output "Executing: 7z.exe $arguments" + & 7z.exe $arguments + if ($LASTEXITCODE -ne 0) { + throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE" + } } -# process iOS xcframework -$xcframeworks = Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter onnxruntime_ios_xcframework.*.zip +# Process iOS xcframework +$xcframeworks = Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter onnxruntime_ios_xcframework.*.zip if ($xcframeworks.Count -eq 1) { - $xcframework = $xcframeworks[0] - $target_dir = "$nuget_artifacts_dir\onnxruntime-ios-xcframework" - # remove version info from filename and use required filename format - $target_file = "$target_dir\onnxruntime.xcframework.zip" - New-Item -Path $target_dir -ItemType directory + $xcframework = $xcframeworks[0] + $target_dir = "$nuget_artifacts_dir\onnxruntime-ios-xcframework" + # Use the required filename format, removing version info. + $target_file = "$target_dir\onnxruntime.xcframework.zip" + New-Item -Path $target_dir -ItemType directory -ErrorAction SilentlyContinue - Write-Output "Copy-Item $($xcframework.FullName) $target_file" - Copy-Item $xcframework.FullName $target_file + Write-Output "Copying $($xcframework.FullName) to $target_file" + Copy-Item $xcframework.FullName $target_file } elseif ($xcframeworks.Count -gt 1) { - Write-Error "Expected at most one onnxruntime_ios_xcframework*.zip file but got: [$xcframeworks]" + Write-Error "Expected at most one onnxruntime_ios_xcframework*.zip file but got: [$xcframeworks]" } - -# copy android AAR. -# for full build of onnxruntime Android AAR, there should only be one .aar file -# called onnxruntime-android-x.y.z.aar or onnxruntime-training-android-x.y.z.aar but sanity check that -$aars = Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.aar +# Copy Android AAR file. +# There should only be one .aar file for a full build. +$aars = Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.aar if ($aars.Count -eq 1) { - $aar = $aars[0] - $aar_prefix = "onnxruntime" - if ($aar -like "onnxruntime-training*") { - $aar_prefix = "onnxruntime-training" - } - $target_dir = "$nuget_artifacts_dir\$aar_prefix-android-aar" - $target_file = "$target_dir\onnxruntime.aar" # remove '-mobile' and version info from filename - New-Item -Path $target_dir -ItemType directory + $aar = $aars[0] + $aar_prefix = "onnxruntime" + if ($aar.Name -like "onnxruntime-training*") { + $aar_prefix = "onnxruntime-training" + } + $target_dir = "$nuget_artifacts_dir\$aar_prefix-android-aar" + # Remove version info from the filename for consistency. + $target_file = "$target_dir\onnxruntime.aar" + New-Item -Path $target_dir -ItemType directory -ErrorAction SilentlyContinue - Write-Output "Copy-Item $($aar.FullName) $target_file" - Copy-Item $aar.FullName $target_file + Write-Output "Copying $($aar.FullName) to $target_file" + Copy-Item $aar.FullName $target_file } elseif ($aars.Count -gt 1) { - Write-Error "Expected at most one Android .aar file but got: [$aars]" + Write-Error "Expected at most one Android .aar file but got: [$aars]" } -# Check whether this is a training pipeline -$is_training_pipeline = $false -if (Test-Path -Path $nuget_artifacts_dir\onnxruntime-training-win-x64-*) { - $is_training_pipeline = $true - Write-Output "onnxruntime-training-win-x64-* dir exists. This is a training pipeline." +# Check if this is a training pipeline by looking for a specific directory. +$is_training_pipeline = Test-Path -Path "$nuget_artifacts_dir\onnxruntime-training-win-x64-*" +if ($is_training_pipeline) { + Write-Output "onnxruntime-training-win-x64-* dir exists. This is a training pipeline." } -# Copy onnxruntime and protoc binaries to the binaries dir as these are required -# by Microsoft.ML.OnnxRuntime.Tests.NetCoreApp +# Copy onnxruntime and protoc binaries required by tests. +$destinationDir = "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo" if ($is_training_pipeline) { - Copy-Item -Path $nuget_artifacts_dir\onnxruntime-training-win-x64-*\lib\* -Destination $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo + Copy-Item -Path "$nuget_artifacts_dir\onnxruntime-training-win-x64-*\lib\*" -Destination $destinationDir -Recurse } else { - Copy-Item -Path $nuget_artifacts_dir\onnxruntime-win-x64-*\lib\* -Destination $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo + Copy-Item -Path "$nuget_artifacts_dir\onnxruntime-win-x64-*\lib\*" -Destination $destinationDir -Recurse } -"Get-ChildItem -Directory -Path $nuget_artifacts_dir\onnxruntime-*" -$ort_dirs = Get-ChildItem -Directory -Path $nuget_artifacts_dir\onnxruntime-* -foreach ($ort_dir in $ort_dirs) -{ - # remove the last '-xxx' segment from the dir name. typically that's the architecture. - $dirname = Split-Path -Path $ort_dir -Leaf - $dirname = $dirname.SubString(0,$dirname.LastIndexOf('-')) - Write-Output "Renaming $ort_dir to $dirname" - Rename-Item -Path $ort_dir -NewName $nuget_artifacts_dir\$dirname +# Rename directories to remove the architecture-specific suffix. +Write-Output "Renaming onnxruntime directories..." +Get-ChildItem -Directory -Path "$nuget_artifacts_dir\onnxruntime-*" | ForEach-Object { + $dirname = $_.Name + # Find the last hyphen and remove the suffix. + $lastHyphenIndex = $dirname.LastIndexOf('-') + if ($lastHyphenIndex -gt -1) { + $newName = $dirname.Substring(0, $lastHyphenIndex) + $newPath = Join-Path -Path $_.Parent.FullName -ChildPath $newName + Write-Output "Renaming '$($_.FullName)' to '$newPath'" + Rename-Item -Path $_.FullName -NewName $newName + } } -# List artifacts -"Post copy artifacts" -Get-ChildItem -Recurse $nuget_artifacts_dir\ +# List the final artifacts. +Write-Output "Post-copy artifacts:" +Get-ChildItem -Recurse $nuget_artifacts_dir + +# Check if all .so files are symlinks, which is required for the package structure. +Write-Output "Checking for .so file symlinks..." +$so_files = Get-ChildItem -Recurse -Path $nuget_artifacts_dir -Filter *.so +# Find any .so files that are NOT symlinks. +$non_symlink_so_files = $so_files | Where-Object { -not ($_.Attributes -band [System.IO.FileAttributes]::ReparsePoint) } + +if ($non_symlink_so_files) { + Write-Error "Found .so files that are not symlinks:" + foreach ($file in $non_symlink_so_files) { + Write-Error "- $($file.FullName)" + } + throw "One or more .so files are not symlinks. This can cause issues in the NuGet package." +} +else { + Write-Output "All found .so files are correctly configured as symlinks." +} From b8a2d3e0d73bdbc38741db74d3039688f84894ec Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 27 Aug 2025 09:08:40 -0700 Subject: [PATCH 2/3] Add -snld20 flag for symbolic link support in extraction --- tools/ci_build/github/windows/extract_nuget_files.ps1 | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/windows/extract_nuget_files.ps1 b/tools/ci_build/github/windows/extract_nuget_files.ps1 index b7f76746ea731..e7d2781df85e1 100644 --- a/tools/ci_build/github/windows/extract_nuget_files.ps1 +++ b/tools/ci_build/github/windows/extract_nuget_files.ps1 @@ -12,7 +12,8 @@ New-Item -Path $nuget_artifacts_dir -ItemType directory -ErrorAction SilentlyCon # Unzip files directly, excluding the iOS xcframework to preserve its symlinks. Get-ChildItem -Path "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact\*" -Include *.zip -Exclude onnxruntime_ios_xcframework.*.zip | Foreach-Object { - $arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir" + # The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01). + $arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir", "-snld20" Write-Output "Executing: 7z.exe $arguments" # Directly call 7z.exe using the call operator '&' & 7z.exe $arguments @@ -26,7 +27,8 @@ Foreach-Object { # First, extract the .tar file from the .tgz archive. Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.tgz | Foreach-Object { - $arguments = "x", "$($_.FullName)", "-y", "-o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" + # The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01). + $arguments = "x", "$($_.FullName)", "-y", "-o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact", "-snld20" Write-Output "Executing: 7z.exe $arguments" & 7z.exe $arguments if ($LASTEXITCODE -ne 0) { @@ -37,7 +39,8 @@ Foreach-Object { # Now, extract the contents from the .tar file into the final directory. Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.tar | Foreach-Object { - $arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir" + # The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01). + $arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir", "-snld20" Write-Output "Executing: 7z.exe $arguments" & 7z.exe $arguments if ($LASTEXITCODE -ne 0) { From 77cabb4c2b632250d41ad55936d0bf7a6f80e4dc Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 28 Aug 2025 07:35:28 -0700 Subject: [PATCH 3/3] update --- .../github/windows/extract_nuget_files.ps1 | 19 +--- .../windows/extract_nuget_files_gpu.ps1 | 86 +++++++++++++------ 2 files changed, 61 insertions(+), 44 deletions(-) diff --git a/tools/ci_build/github/windows/extract_nuget_files.ps1 b/tools/ci_build/github/windows/extract_nuget_files.ps1 index e7d2781df85e1..20d6c1f2b63a5 100644 --- a/tools/ci_build/github/windows/extract_nuget_files.ps1 +++ b/tools/ci_build/github/windows/extract_nuget_files.ps1 @@ -116,21 +116,4 @@ Get-ChildItem -Directory -Path "$nuget_artifacts_dir\onnxruntime-*" | ForEach-Ob # List the final artifacts. Write-Output "Post-copy artifacts:" -Get-ChildItem -Recurse $nuget_artifacts_dir - -# Check if all .so files are symlinks, which is required for the package structure. -Write-Output "Checking for .so file symlinks..." -$so_files = Get-ChildItem -Recurse -Path $nuget_artifacts_dir -Filter *.so -# Find any .so files that are NOT symlinks. -$non_symlink_so_files = $so_files | Where-Object { -not ($_.Attributes -band [System.IO.FileAttributes]::ReparsePoint) } - -if ($non_symlink_so_files) { - Write-Error "Found .so files that are not symlinks:" - foreach ($file in $non_symlink_so_files) { - Write-Error "- $($file.FullName)" - } - throw "One or more .so files are not symlinks. This can cause issues in the NuGet package." -} -else { - Write-Output "All found .so files are correctly configured as symlinks." -} +Get-ChildItem -Recurse $nuget_artifacts_dir \ No newline at end of file diff --git a/tools/ci_build/github/windows/extract_nuget_files_gpu.ps1 b/tools/ci_build/github/windows/extract_nuget_files_gpu.ps1 index 01a8eebe75df2..29946dcb73f8a 100644 --- a/tools/ci_build/github/windows/extract_nuget_files_gpu.ps1 +++ b/tools/ci_build/github/windows/extract_nuget_files_gpu.ps1 @@ -2,47 +2,81 @@ # Licensed under the MIT License. # This file is used by Zip-Nuget-Java Packaging Pipeline -New-Item -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts -ItemType directory +# Define the directory for NuGet artifacts. +$nuget_artifacts_dir = "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts" +# Create the directory if it doesn't exist. +New-Item -Path $nuget_artifacts_dir -ItemType directory -ErrorAction SilentlyContinue -Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.zip | +## .zip files +Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.zip | Foreach-Object { - $cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts" - Write-Output $cmd - Invoke-Expression -Command $cmd + # The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01). + $arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir", "-snld20" + Write-Output "Executing: 7z.exe $arguments" + & 7z.exe $arguments + if ($LASTEXITCODE -ne 0) { + throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE" + } } -Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tgz | +## .tgz files +Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.tgz | Foreach-Object { - $cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" # *.tar will be created after *.tgz is extracted - Write-Output $cmd - Invoke-Expression -Command $cmd + # The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01). + # *.tar will be created after *.tgz is extracted + $arguments = "x", "$($_.FullName)", "-y", "-o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact", "-snld20" + Write-Output "Executing: 7z.exe $arguments" + & 7z.exe $arguments + if ($LASTEXITCODE -ne 0) { + throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE" + } } -Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tar | +## .tar files +Get-ChildItem "$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" -Filter *.tar | Foreach-Object { - $cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts" - Write-Output $cmd - Invoke-Expression -Command $cmd + # The -snld20 flag is used to bypass security checks for creating symbolic links (added in 7-Zip 25.01). + $arguments = "x", "$($_.FullName)", "-y", "-o$nuget_artifacts_dir", "-snld20" + Write-Output "Executing: 7z.exe $arguments" + & 7z.exe $arguments + if ($LASTEXITCODE -ne 0) { + throw "Error extracting '$($_.FullName)'. Exit code: $LASTEXITCODE" + } } +# Create directory for protobuf build dependencies. +New-Item -Path "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build\RelWithDebInfo" -ItemType directory -ErrorAction SilentlyContinue -New-Item -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build\RelWithDebInfo -ItemType directory - -Copy-Item -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-x64-cuda-*\lib\* -Destination $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo +# Copy CUDA libraries. +Copy-Item -Path "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-x64-cuda-*\lib\*" -Destination "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo" +# Install protoc via dotnet. $protocInstallDir = "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build" dotnet new console dotnet add package Google.Protobuf.Tools --version 3.21.12 --package-directory $protocInstallDir +if ($LASTEXITCODE -ne 0) { + throw "Error adding Google.Protobuf.Tools package. Exit code: $LASTEXITCODE" +} + +# Find and copy the protoc executable. $protocDir = Get-ChildItem -Path $protocInstallDir -Recurse -Filter "protoc.exe" | Select-Object -ExpandProperty DirectoryName -First 1 -Write-Output $protocDir -Copy-Item -Path $protocDir -Destination $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build\RelWithDebInfo - -$ort_dirs = Get-ChildItem -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-* -Directory -foreach ($ort_dir in $ort_dirs) -{ - $dirname = Split-Path -Path $ort_dir -Leaf - $dirname = $dirname.SubString(0,$dirname.LastIndexOf('-')) - Write-Output "Renaming $ort_dir to $dirname" - Rename-Item -Path $ort_dir -NewName $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\$dirname +if ($protocDir) { + Write-Output "Found protoc directory: $protocDir" + Copy-Item -Path $protocDir -Destination "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\_deps\protobuf-build\RelWithDebInfo" +} +else { + Write-Error "Could not find protoc.exe in $protocInstallDir" } +# Rename onnxruntime directories to a generic format. +$ort_dirs = Get-ChildItem -Path "$Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-*" -Directory +foreach ($ort_dir in $ort_dirs) { + $dirname = Split-Path -Path $ort_dir -Leaf + $lastHyphenIndex = $dirname.LastIndexOf('-') + if ($lastHyphenIndex -gt -1) { + $newName = $dirname.Substring(0, $lastHyphenIndex) + $newPath = Join-Path -Path $ort_dir.Parent.FullName -ChildPath $newName + Write-Output "Renaming '$($ort_dir.FullName)' to '$newPath'" + Rename-Item -Path $ort_dir.FullName -NewName $newName + } +}