diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy index f1bcc786b72e..0fc71b430ca0 100644 --- a/ci/jenkins/generated/arm_jenkinsfile.groovy +++ b/ci/jenkins/generated/arm_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-12-06T20:56:42.365592 +// Generated at 2022-12-09T15:39:24.387114 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -205,8 +205,7 @@ def docker_init(image) { if (image.contains("amazonaws.com")) { // If this string is in the image name it's from ECR and needs to be pulled // with the right credentials - // ecr_pull(image) - sh "echo Pulling from AWS is not implemented && exit 1" + ecr_pull(image) } else { sh( script: """ @@ -219,6 +218,47 @@ def docker_init(image) { } } +def ecr_pull(full_name) { + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 5 docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', diff --git a/ci/jenkins/generated/cortexm_jenkinsfile.groovy b/ci/jenkins/generated/cortexm_jenkinsfile.groovy index 4b5ba2e104f4..25846f5b4b5e 100644 --- a/ci/jenkins/generated/cortexm_jenkinsfile.groovy +++ b/ci/jenkins/generated/cortexm_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-12-06T20:56:42.204393 +// Generated at 2022-12-09T15:39:24.437899 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -205,8 +205,7 @@ def docker_init(image) { if (image.contains("amazonaws.com")) { // If this string is in the image name it's from ECR and needs to be pulled // with the right credentials - // ecr_pull(image) - sh "echo Pulling from AWS is not implemented && exit 1" + ecr_pull(image) } else { sh( script: """ @@ -219,6 +218,47 @@ def docker_init(image) { } } +def ecr_pull(full_name) { + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 5 docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy index 378b20db51b0..f9ede00399a2 100644 --- a/ci/jenkins/generated/cpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-12-06T20:56:42.393957 +// Generated at 2022-12-09T15:39:24.540570 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -205,8 +205,7 @@ def docker_init(image) { if (image.contains("amazonaws.com")) { // If this string is in the image name it's from ECR and needs to be pulled // with the right credentials - // ecr_pull(image) - sh "echo Pulling from AWS is not implemented && exit 1" + ecr_pull(image) } else { sh( script: """ @@ -219,6 +218,47 @@ def docker_init(image) { } } +def ecr_pull(full_name) { + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 5 docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', diff --git a/ci/jenkins/generated/docker_jenkinsfile.groovy b/ci/jenkins/generated/docker_jenkinsfile.groovy index 050ef2983e43..9e1946c194e6 100644 --- a/ci/jenkins/generated/docker_jenkinsfile.groovy +++ b/ci/jenkins/generated/docker_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-12-07T07:10:24.637792 +// Generated at 2022-12-09T15:39:24.508775 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -205,8 +205,7 @@ def docker_init(image) { if (image.contains("amazonaws.com")) { // If this string is in the image name it's from ECR and needs to be pulled // with the right credentials - // ecr_pull(image) - sh "echo Pulling from AWS is not implemented && exit 1" + ecr_pull(image) } else { sh( script: """ @@ -219,6 +218,47 @@ def docker_init(image) { } } +def ecr_pull(full_name) { + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 5 docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', @@ -544,47 +584,6 @@ def ecr_push(full_name) { return ecr_name } -def ecr_pull(full_name) { - aws_account_id = sh( - returnStdout: true, - script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', - label: 'Get AWS ID' - ).trim() - - try { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2', - "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { - sh( - script: ''' - set -eux - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO - ''', - label: 'Log in to ECR' - ) - sh( - script: """ - set -eux - . ci/scripts/retry.sh - retry 5 docker pull ${full_name} - """, - label: 'Pull image from ECR' - ) - } - } finally { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2', - "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { - sh( - script: 'docker logout $AWS_ECR_REPO', - label: 'Clean up login credentials' - ) - } - } -} - def build_image(image_name) { hash = sh( returnStdout: true, diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy index 48a6619cbab1..bebc0c4c22a5 100644 --- a/ci/jenkins/generated/gpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-12-07T07:10:24.840515 +// Generated at 2022-12-09T15:39:24.455336 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -205,8 +205,7 @@ def docker_init(image) { if (image.contains("amazonaws.com")) { // If this string is in the image name it's from ECR and needs to be pulled // with the right credentials - // ecr_pull(image) - sh "echo Pulling from AWS is not implemented && exit 1" + ecr_pull(image) } else { sh( script: """ @@ -219,6 +218,47 @@ def docker_init(image) { } } +def ecr_pull(full_name) { + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 5 docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', diff --git a/ci/jenkins/generated/hexagon_jenkinsfile.groovy b/ci/jenkins/generated/hexagon_jenkinsfile.groovy index e5397eee3a9c..c2f39a0d084b 100644 --- a/ci/jenkins/generated/hexagon_jenkinsfile.groovy +++ b/ci/jenkins/generated/hexagon_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-12-06T20:56:42.338377 +// Generated at 2022-12-09T15:39:24.369191 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -205,8 +205,7 @@ def docker_init(image) { if (image.contains("amazonaws.com")) { // If this string is in the image name it's from ECR and needs to be pulled // with the right credentials - // ecr_pull(image) - sh "echo Pulling from AWS is not implemented && exit 1" + ecr_pull(image) } else { sh( script: """ @@ -219,6 +218,47 @@ def docker_init(image) { } } +def ecr_pull(full_name) { + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 5 docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', diff --git a/ci/jenkins/generated/i386_jenkinsfile.groovy b/ci/jenkins/generated/i386_jenkinsfile.groovy index 876670acebba..ae66fbe3e48c 100644 --- a/ci/jenkins/generated/i386_jenkinsfile.groovy +++ b/ci/jenkins/generated/i386_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-12-06T20:56:42.288840 +// Generated at 2022-12-09T15:39:24.421467 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -205,8 +205,7 @@ def docker_init(image) { if (image.contains("amazonaws.com")) { // If this string is in the image name it's from ECR and needs to be pulled // with the right credentials - // ecr_pull(image) - sh "echo Pulling from AWS is not implemented && exit 1" + ecr_pull(image) } else { sh( script: """ @@ -219,6 +218,47 @@ def docker_init(image) { } } +def ecr_pull(full_name) { + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 5 docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', diff --git a/ci/jenkins/generated/lint_jenkinsfile.groovy b/ci/jenkins/generated/lint_jenkinsfile.groovy index 3aaea4436fcb..f8dccc863590 100644 --- a/ci/jenkins/generated/lint_jenkinsfile.groovy +++ b/ci/jenkins/generated/lint_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-12-06T20:56:42.313954 +// Generated at 2022-12-09T15:39:24.476946 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -205,8 +205,7 @@ def docker_init(image) { if (image.contains("amazonaws.com")) { // If this string is in the image name it's from ECR and needs to be pulled // with the right credentials - // ecr_pull(image) - sh "echo Pulling from AWS is not implemented && exit 1" + ecr_pull(image) } else { sh( script: """ @@ -219,6 +218,47 @@ def docker_init(image) { } } +def ecr_pull(full_name) { + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 5 docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', diff --git a/ci/jenkins/generated/minimal_jenkinsfile.groovy b/ci/jenkins/generated/minimal_jenkinsfile.groovy index f8a59ef5734d..6c4abb0bd5af 100644 --- a/ci/jenkins/generated/minimal_jenkinsfile.groovy +++ b/ci/jenkins/generated/minimal_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-12-06T20:56:42.235080 +// Generated at 2022-12-09T15:39:24.492813 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -205,8 +205,7 @@ def docker_init(image) { if (image.contains("amazonaws.com")) { // If this string is in the image name it's from ECR and needs to be pulled // with the right credentials - // ecr_pull(image) - sh "echo Pulling from AWS is not implemented && exit 1" + ecr_pull(image) } else { sh( script: """ @@ -219,6 +218,47 @@ def docker_init(image) { } } +def ecr_pull(full_name) { + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 5 docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', diff --git a/ci/jenkins/generated/riscv_jenkinsfile.groovy b/ci/jenkins/generated/riscv_jenkinsfile.groovy index eb62c3731f79..7b9bbe7ad399 100644 --- a/ci/jenkins/generated/riscv_jenkinsfile.groovy +++ b/ci/jenkins/generated/riscv_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-12-06T20:56:42.442689 +// Generated at 2022-12-09T15:39:24.405262 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -205,8 +205,7 @@ def docker_init(image) { if (image.contains("amazonaws.com")) { // If this string is in the image name it's from ECR and needs to be pulled // with the right credentials - // ecr_pull(image) - sh "echo Pulling from AWS is not implemented && exit 1" + ecr_pull(image) } else { sh( script: """ @@ -219,6 +218,47 @@ def docker_init(image) { } } +def ecr_pull(full_name) { + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 5 docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', diff --git a/ci/jenkins/generated/wasm_jenkinsfile.groovy b/ci/jenkins/generated/wasm_jenkinsfile.groovy index d43c7f9d24e4..8c8ee0388699 100644 --- a/ci/jenkins/generated/wasm_jenkinsfile.groovy +++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-12-06T20:56:42.420989 +// Generated at 2022-12-09T15:39:24.526394 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -205,8 +205,7 @@ def docker_init(image) { if (image.contains("amazonaws.com")) { // If this string is in the image name it's from ECR and needs to be pulled // with the right credentials - // ecr_pull(image) - sh "echo Pulling from AWS is not implemented && exit 1" + ecr_pull(image) } else { sh( script: """ @@ -219,6 +218,47 @@ def docker_init(image) { } } +def ecr_pull(full_name) { + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 5 docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', diff --git a/ci/jenkins/templates/docker_jenkinsfile.groovy.j2 b/ci/jenkins/templates/docker_jenkinsfile.groovy.j2 index db3e6159b82a..07ae49811337 100644 --- a/ci/jenkins/templates/docker_jenkinsfile.groovy.j2 +++ b/ci/jenkins/templates/docker_jenkinsfile.groovy.j2 @@ -61,47 +61,6 @@ def ecr_push(full_name) { return ecr_name } -def ecr_pull(full_name) { - aws_account_id = sh( - returnStdout: true, - script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', - label: 'Get AWS ID' - ).trim() - - try { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION={{ aws_default_region }}', - "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) { - sh( - script: ''' - set -eux - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO - ''', - label: 'Log in to ECR' - ) - sh( - script: """ - set -eux - . ci/scripts/retry.sh - retry 5 docker pull ${full_name} - """, - label: 'Pull image from ECR' - ) - } - } finally { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION={{ aws_default_region }}', - "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) { - sh( - script: 'docker logout $AWS_ECR_REPO', - label: 'Clean up login credentials' - ) - } - } -} - def build_image(image_name) { hash = sh( returnStdout: true, diff --git a/ci/jenkins/templates/utils/Prepare.groovy.j2 b/ci/jenkins/templates/utils/Prepare.groovy.j2 index b295bb430853..d5aebdc07008 100644 --- a/ci/jenkins/templates/utils/Prepare.groovy.j2 +++ b/ci/jenkins/templates/utils/Prepare.groovy.j2 @@ -75,8 +75,7 @@ def docker_init(image) { if (image.contains("amazonaws.com")) { // If this string is in the image name it's from ECR and needs to be pulled // with the right credentials - // ecr_pull(image) - sh "echo Pulling from AWS is not implemented && exit 1" + ecr_pull(image) } else { sh( script: """ @@ -89,6 +88,47 @@ def docker_init(image) { } } +def ecr_pull(full_name) { + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION={{ aws_default_region }}', + "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + . ${jenkins_scripts_root}/retry.sh + retry 5 docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION={{ aws_default_region }}', + "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + def should_skip_slow_tests(pr_number) { withCredentials([string( credentialsId: 'tvm-bot-jenkins-reader', diff --git a/ci/scripts/jenkins/determine_docker_images.py b/ci/scripts/jenkins/determine_docker_images.py index 78da9a354629..41003958dd61 100755 --- a/ci/scripts/jenkins/determine_docker_images.py +++ b/ci/scripts/jenkins/determine_docker_images.py @@ -32,6 +32,7 @@ PAGE_SIZE = 25 TEST_DATA = None IMAGE_TAGS_FILE = REPO_ROOT / "ci" / "jenkins" / "docker-images.ini" +TVM_CI_ECR = "477529581014.dkr.ecr.us-west-2.amazonaws.com" def docker_api(url: str, use_pagination: bool = False) -> Dict[str, Any]: @@ -111,7 +112,10 @@ def image_exists(spec: str) -> bool: name_dir.mkdir(exist_ok=True) images_to_use = {} for filename, spec in images.items(): - if image_exists(spec): + if spec.startswith(TVM_CI_ECR): + logging.info(f"{spec} is from ECR") + images_to_use[filename] = spec + elif image_exists(spec): logging.info(f"{spec} found in tlcpack") images_to_use[filename] = spec else: