diff --git a/.pipelines/PackageBuildPRCheck.yml b/.pipelines/PackageBuildPRCheck.yml deleted file mode 100644 index 3e76a409e43..00000000000 --- a/.pipelines/PackageBuildPRCheck.yml +++ /dev/null @@ -1,62 +0,0 @@ -trigger: none -pr: - - fasttrack/* - -parameters: - - name: buildConfiguration - type: object - default: - - name: "AMD64" - agentPool: "mariner-core-x64-1es-mariner2-gpt-test" - agentVMName: "Mariner-Build-Mariner2-1ES-Managed-Image2-GPT-Test" - - name: "ARM64" - agentPool: "mariner-core-arm64-1es-mariner2" - agentVMName: "Mariner-Build-Mariner2-1ES-ARM64-Managed-Image" - -resources: - repositories: - - repository: templates - type: git - name: OneBranch.Pipelines/GovernedTemplates - ref: refs/heads/main - -extends: - template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates - parameters: - featureFlags: - runOnHost: true - globalSdl: - credscan: - suppressionsFile: .config/CredScanSuppressions.json - stages: - - ${{ each configuration in parameters.buildConfiguration }}: - - stage: Toolchain_${{ configuration.name }} - jobs: - - job: Build - pool: - type: linux - isCustom: true - name: ${{ configuration.agentPool }} - vmImage: ${{ configuration.agentVMName }} - steps: - - template: .pipelines/templates/ToolchainBuild.yml@self - - - stage: RPMs_${{ configuration.name }} - dependsOn: Toolchain_${{ configuration.name }} - jobs: - - job: Build - pool: - type: linux - isCustom: true - name: ${{ configuration.agentPool }} - vmImage: ${{ configuration.agentVMName }} - strategy: - matrix: - regular: - runCheck: 'false' - ptest: - runCheck: 'true' - steps: - - template: .pipelines/templates/PackageBuild.yml@self - parameters: - isCheckBuild: '$(runCheck)' diff --git a/.pipelines/livepatching/BuildLivepatch.sh b/.pipelines/livepatching/BuildLivepatch.sh index 74946e1f55c..a7cf987063b 100755 --- a/.pipelines/livepatching/BuildLivepatch.sh +++ b/.pipelines/livepatching/BuildLivepatch.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + set -e ROOT_DIR="$(git rev-parse --show-toplevel)" diff --git a/.pipelines/livepatching/BuildLivepatchSigned.sh b/.pipelines/livepatching/BuildLivepatchSigned.sh index 1a6bf8ea15f..5b5468dd2cd 100755 --- a/.pipelines/livepatching/BuildLivepatchSigned.sh +++ b/.pipelines/livepatching/BuildLivepatchSigned.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + set -e ROOT_DIR="$(git rev-parse --show-toplevel)" diff --git a/.pipelines/prchecks/PackageBuildPRCheck.yml b/.pipelines/prchecks/PackageBuildPRCheck.yml new file mode 100644 index 00000000000..4c7a1a3cf44 --- /dev/null +++ b/.pipelines/prchecks/PackageBuildPRCheck.yml @@ -0,0 +1,93 @@ +# Since we're boosting our builds by using a private, pre-compiled raw toolchain +# the pipeline requires defining the following variables outside of the YAML: +# - rawToolchainCacheURL_AMD64 +# - rawToolchainCacheURL_ARM64 +# - agentPool_AMD64 +# - agentPool_ARM64 +# - agentVMName_AMD64 +# - agentVMName_ARM64 + +trigger: none + +parameters: + - name: buildConfiguration + type: object + default: + - name: "AMD64" + agentPool: "$(agentPool_AMD64)" + agentVMName: "$(agentVMName_AMD64)" + rawToolchainCacheURL: "$(rawToolchainCacheURL_AMD64)" + rawToolchainExpectedHash: "f56df34b90915c93f772d3961bf5e9eeb8c1233db43dd92070214e4ce6b72894" + - name: "ARM64" + agentPool: "$(agentPool_ARM64)" + agentVMName: "$(agentVMName_ARM64)" + rawToolchainCacheURL: "$(rawToolchainCacheURL_ARM64)" + rawToolchainExpectedHash: "65de43b3bdcfdaac71df1f11fd1f830a8109b1eb9d7cb6cbc2e2d0e929d0ef76" + +resources: + repositories: + - repository: templates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + +extends: + template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates + parameters: + featureFlags: + runOnHost: true + globalSdl: + credscan: + suppressionsFile: .config/CredScanSuppressions.json + stages: + - ${{ each configuration in parameters.buildConfiguration }}: + - stage: Toolchain_${{ configuration.name }} + jobs: + - job: Build + pool: + type: linux + isCustom: true + name: ${{ configuration.agentPool }} + vmImage: ${{ configuration.agentVMName }} + variables: + ob_outputDirectory: $(Build.ArtifactStagingDirectory) + steps: + - template: .pipelines/templates/RawToolchainDownload.yml@self + parameters: + rawToolchainCacheURL: ${{ configuration.rawToolchainCacheURL }} + rawToolchainExpectedHash: ${{ configuration.rawToolchainExpectedHash }} + + - template: .pipelines/templates/ToolchainBuild.yml@self + parameters: + buildArtifactsFolder: $(ob_outputDirectory) + + # 1. Automatic publishing seems to be broken in the OneBranch templates when the build runs directly on the agent host. + # Once that's fixed, we should remove this step. + # 2. The value for 'artifact' cannot be changed, as this is the only value OneBranch accepts. + # We cannot also use pre-defined variables like "drop_$(System.StageName)_$(Agent.JobName)", to automatically track name changes. + - task: PublishPipelineArtifact@1 + inputs: + artifact: drop_Toolchain_${{ configuration.name }}_Build + targetPath: $(ob_outputDirectory) + condition: always() + displayName: 'Publish toolchain artifacts' + + - stage: RPMs_${{ configuration.name }} + dependsOn: Toolchain_${{ configuration.name }} + jobs: + - job: Build + pool: + type: linux + isCustom: true + name: ${{ configuration.agentPool }} + vmImage: ${{ configuration.agentVMName }} + strategy: + matrix: + regular: + runCheck: 'false' + ptest: + runCheck: 'true' + steps: + - template: .pipelines/templates/PackageBuild.yml@self + parameters: + isCheckBuild: '$(runCheck)' diff --git a/.pipelines/templates/RawToolchainDownload.yml b/.pipelines/templates/RawToolchainDownload.yml new file mode 100644 index 00000000000..71f9b3f1c68 --- /dev/null +++ b/.pipelines/templates/RawToolchainDownload.yml @@ -0,0 +1,37 @@ +parameters: + - name: coreRepoRoot + type: string + default: "$(Build.SourcesDirectory)" + + - name: rawToolchainCacheURL + type: string + + - name: rawToolchainExpectedHash + type: string + +steps: + - bash: | + set -e + + raw_toolchain_file_path="${{ parameters.coreRepoRoot }}/build/toolchain/toolchain_from_container.tar.gz" + + echo "-- Downloading cached raw toolchain." + + mkdir -p "$(dirname "$raw_toolchain_file_path")" + if ! wget --quiet --timeout=30 --continue "$RAW_TOOLCHAIN_URL" -O "$raw_toolchain_file_path"; then + echo "-- ERROR: failed to download raw toolchain cache." >&2 + exit 1 + fi + + # Verifying toolchain's SHA-256 hash. + cache_sha256=$(sha256sum "$raw_toolchain_file_path" | cut -d' ' -f1) + if [[ "$cache_sha256" != "${{ parameters.rawToolchainExpectedHash }}" ]]; then + echo "-- ERROR: raw toolchain hash verification failed. Expected (${{ parameters.rawToolchainExpectedHash }}). Got ($cache_sha256)." >&2 + exit 1 + fi + echo "-- Raw toolchain hash OK." + + touch "$raw_toolchain_file_path" + env: + RAW_TOOLCHAIN_URL: ${{ parameters.rawToolchainCacheURL }} + displayName: "Populate raw toolchain" diff --git a/.pipelines/templates/ToolchainBuild.yml b/.pipelines/templates/ToolchainBuild.yml index 6b760451c51..6306f23ba17 100644 --- a/.pipelines/templates/ToolchainBuild.yml +++ b/.pipelines/templates/ToolchainBuild.yml @@ -1,4 +1,38 @@ +parameters: + - name: buildArtifactsFolder + type: string + default: "$(Build.ArtifactStagingDirectory)" + + - name: coreRepoRoot + type: string + default: "$(Build.SourcesDirectory)" + steps: - - script: | - echo "The used architecture is: $(uname -m)" - displayName: 'Use parameter' + - bash: sudo make -C "${{ parameters.coreRepoRoot }}/toolkit" "-j$(nproc)" toolchain QUICK_REBUILD=y + displayName: "Build toolchain" + + - bash: | + failed_rpms_log="${{ parameters.coreRepoRoot }}/build/logs/toolchain/failures.txt" + + if [[ -f "$failed_rpms_log" ]]; then + echo "List of RPMs that failed to build:" >&2 + cat "$failed_rpms_log" >&2 + else + echo "Build failed - no specific RPM" >&2 + fi + condition: failed() + displayName: "Print failed RPMs" + + - bash: | + published_artifacts_dir="${{ parameters.buildArtifactsFolder }}/ARTIFACTS" + mkdir -p "$published_artifacts_dir" + cp "${{ parameters.coreRepoRoot }}"/build/toolchain/toolchain_built_{,s}rpms_all.tar.gz "$published_artifacts_dir" + condition: succeeded() + displayName: "Copy artifacts for publishing" + + - bash: | + published_logs_dir="${{ parameters.buildArtifactsFolder }}/LOGS" + mkdir -p "$published_logs_dir" + tar -C "${{ parameters.coreRepoRoot }}/build/logs/toolchain" -czf "$published_logs_dir/toolchain.logs.tar.gz" . + condition: always() + displayName: "Copy logs for publishing"