diff --git a/Jenkinsfile b/Jenkinsfile index 424f97494d76..024b920ac676 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-05-17T17:26:21.660243 +// Generated at 2022-05-19T11:41:58.421857 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -482,53 +482,9 @@ def make(docker_type, path, make_flag) { } } -// Specifications to Jenkins "stash" command for use with various pack_ and unpack_ functions. -tvm_runtime = 'build/libtvm_runtime.so, build/config.cmake' // use libtvm_runtime.so. -tvm_lib = 'build/libtvm.so, ' + tvm_runtime // use libtvm.so to run the full compiler. -// LLVM upstream lib -tvm_multilib = 'build/libtvm.so, ' + - 'build/libvta_fsim.so, ' + - tvm_runtime - -tvm_multilib_tsim = 'build/libvta_tsim.so, ' + - tvm_multilib - -microtvm_tar_gz = 'build/microtvm_template_projects.tar.gz' - -// pack libraries for later use -def pack_lib(name, libs) { - sh (script: """ - echo "Packing ${libs} into ${name}" - echo ${libs} | sed -e 's/,/ /g' | xargs md5sum - """, label: 'Stash libraries and show md5') - stash includes: libs, name: name -} +// Filenames for stashing between build and test steps +s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" -// unpack libraries saved before -def unpack_lib(name, libs) { - unstash name - sh (script: """ - echo "Unpacked ${libs} from ${name}" - echo ${libs} | sed -e 's/,/ /g' | xargs md5sum - """, label: 'Unstash libraries and show md5') -} - -// compress microtvm template projects and pack the tar. -def pack_microtvm_template_projects(name) { - sh( - script: 'cd build && tar -czvf microtvm_template_projects.tar.gz microtvm_template_projects/', - label: 'Compress microtvm_template_projects' - ) - pack_lib(name + '-microtvm-libs', microtvm_tar_gz) -} - -def unpack_microtvm_template_projects(name) { - unpack_lib(name + '-microtvm-libs', microtvm_tar_gz) - sh( - script: 'cd build && tar -xzvf microtvm_template_projects.tar.gz', - label: 'Unpack microtvm_template_projects' - ) -} def ci_setup(image) { sh ( @@ -565,24 +521,63 @@ def cpp_unittest(image) { ) } + +def add_microtvm_permissions() { + sh( + script: 'find build/microtvm_template_projects -type f | xargs chmod +x', + label: 'Add execute permissions for microTVM files', + ) +} + + def build() { stage('Build') { environment { SKIP_SLOW_TESTS = "${skip_slow_tests}" } - parallel 'BUILD: GPU': { + parallel( + 'BUILD: GPU': { if (!skip_ci) { node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") { init_git() sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" make("${ci_gpu} --no-gpu", 'build', '-j2') - pack_lib('gpu', tvm_multilib) - pack_microtvm_template_projects('gpu') + sh( + script: """ + set -eux + md5sum build/libtvm.so + aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/gpu/build/libtvm.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/gpu/build/libvta_fsim.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/gpu/build/libtvm_runtime.so + md5sum build/config.cmake + aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/gpu/build/config.cmake + aws s3 cp --no-progress build/microtvm_template_projects s3://${s3_prefix}/gpu/build/microtvm_template_projects --recursive + """, + label: 'Upload artifacts to S3', + ) + + // compiler test sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build2" make("${ci_gpu} --no-gpu", 'build2', '-j2') - pack_lib('gpu2', tvm_multilib) + sh( + script: """ + set -eux + md5sum build/libtvm.so + aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/gpu2/build/libtvm.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/gpu2/build/libvta_fsim.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/gpu2/build/libtvm_runtime.so + md5sum build/config.cmake + aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/gpu2/build/config.cmake + """, + label: 'Upload artifacts to S3', + ) + } } } @@ -597,7 +592,23 @@ stage('Build') { label: 'Create CPU cmake config', ) make(ci_cpu, 'build', '-j2') - pack_lib('cpu', tvm_multilib_tsim) + sh( + script: """ + set -eux + md5sum build/libvta_tsim.so + aws s3 cp --no-progress build/libvta_tsim.so s3://${s3_prefix}/cpu/build/libvta_tsim.so + md5sum build/libtvm.so + aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/cpu/build/libtvm.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/cpu/build/libvta_fsim.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/cpu/build/libtvm_runtime.so + md5sum build/config.cmake + aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/cpu/build/config.cmake + """, + label: 'Upload artifacts to S3', + ) + timeout(time: max_time, unit: 'MINUTES') { ci_setup(ci_cpu) // sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh" @@ -644,7 +655,23 @@ stage('Build') { label: 'Create i386 cmake config', ) make(ci_i386, 'build', '-j2') - pack_lib('i386', tvm_multilib_tsim) + sh( + script: """ + set -eux + md5sum build/libvta_tsim.so + aws s3 cp --no-progress build/libvta_tsim.so s3://${s3_prefix}/i386/build/libvta_tsim.so + md5sum build/libtvm.so + aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/i386/build/libtvm.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/i386/build/libvta_fsim.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/i386/build/libtvm_runtime.so + md5sum build/config.cmake + aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/i386/build/config.cmake + """, + label: 'Upload artifacts to S3', + ) + } } } else { @@ -661,7 +688,21 @@ stage('Build') { label: 'Create ARM cmake config', ) make(ci_arm, 'build', '-j4') - pack_lib('arm', tvm_multilib) + sh( + script: """ + set -eux + md5sum build/libtvm.so + aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/arm/build/libtvm.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/arm/build/libvta_fsim.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/arm/build/libtvm_runtime.so + md5sum build/config.cmake + aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/arm/build/config.cmake + """, + label: 'Upload artifacts to S3', + ) + } } } else { @@ -678,8 +719,20 @@ stage('Build') { label: 'Create QEMU cmake config', ) make(ci_qemu, 'build', '-j2') - pack_lib('qemu', tvm_lib) - pack_microtvm_template_projects('qemu') + sh( + script: """ + set -eux + md5sum build/libtvm.so + aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/qemu/build/libtvm.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/qemu/build/libtvm_runtime.so + md5sum build/config.cmake + aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/qemu/build/config.cmake + aws s3 cp --no-progress build/microtvm_template_projects s3://${s3_prefix}/qemu/build/microtvm_template_projects --recursive + """, + label: 'Upload artifacts to S3', + ) + } } } else { @@ -696,13 +749,26 @@ stage('Build') { label: 'Create Hexagon cmake config', ) make(ci_hexagon, 'build', '-j2') - pack_lib('hexagon', tvm_lib) + sh( + script: """ + set -eux + md5sum build/libtvm.so + aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/hexagon/build/libtvm.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/hexagon/build/libtvm_runtime.so + md5sum build/config.cmake + aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/hexagon/build/config.cmake + """, + label: 'Upload artifacts to S3', + ) + } } } else { Utils.markStageSkippedForConditional('BUILD: Hexagon') } - } + }, + ) } } @@ -726,10 +792,38 @@ stage('Test') { 'PLATFORM=gpu', 'TVM_NUM_SHARDS=2', 'TVM_SHARD_INDEX=0'], { - unpack_lib('gpu2', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + cpp_unittest(ci_gpu) - unpack_lib('gpu', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_gpu) cpp_unittest(ci_gpu) sh ( @@ -762,7 +856,21 @@ stage('Test') { 'PLATFORM=gpu', 'TVM_NUM_SHARDS=2', 'TVM_SHARD_INDEX=1'], { - unpack_lib('gpu', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh", @@ -798,7 +906,23 @@ stage('Test') { 'PLATFORM=cpu', 'TVM_NUM_SHARDS=2', 'TVM_SHARD_INDEX=0'], { - unpack_lib('cpu', tvm_multilib_tsim) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + md5sum build/libvta_tsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", @@ -826,7 +950,23 @@ stage('Test') { 'PLATFORM=cpu', 'TVM_NUM_SHARDS=2', 'TVM_SHARD_INDEX=1'], { - unpack_lib('cpu', tvm_multilib_tsim) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + md5sum build/libvta_tsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", @@ -851,7 +991,23 @@ stage('Test') { try { init_git() withEnv(['PLATFORM=cpu'], { - unpack_lib('cpu', tvm_multilib_tsim) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + md5sum build/libvta_tsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_cpu) cpp_unittest(ci_cpu) python_unittest(ci_cpu) @@ -882,7 +1038,21 @@ stage('Test') { 'PLATFORM=i386', 'TVM_NUM_SHARDS=3', 'TVM_SHARD_INDEX=0'], { - unpack_lib('i386', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_i386) cpp_unittest(ci_i386) python_unittest(ci_i386) @@ -913,7 +1083,21 @@ stage('Test') { 'PLATFORM=i386', 'TVM_NUM_SHARDS=3', 'TVM_SHARD_INDEX=1'], { - unpack_lib('i386', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_i386) python_unittest(ci_i386) sh ( @@ -943,7 +1127,21 @@ stage('Test') { 'PLATFORM=i386', 'TVM_NUM_SHARDS=3', 'TVM_SHARD_INDEX=2'], { - unpack_lib('i386', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_i386) python_unittest(ci_i386) sh ( @@ -973,7 +1171,19 @@ stage('Test') { 'PLATFORM=hexagon', 'TVM_NUM_SHARDS=4', 'TVM_SHARD_INDEX=0'], { - unpack_lib('hexagon', tvm_lib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_hexagon) cpp_unittest(ci_hexagon) sh ( @@ -1006,7 +1216,19 @@ stage('Test') { 'PLATFORM=hexagon', 'TVM_NUM_SHARDS=4', 'TVM_SHARD_INDEX=1'], { - unpack_lib('hexagon', tvm_lib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_hexagon) sh ( script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh", @@ -1038,7 +1260,19 @@ stage('Test') { 'PLATFORM=hexagon', 'TVM_NUM_SHARDS=4', 'TVM_SHARD_INDEX=2'], { - unpack_lib('hexagon', tvm_lib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_hexagon) sh ( script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh", @@ -1070,7 +1304,19 @@ stage('Test') { 'PLATFORM=hexagon', 'TVM_NUM_SHARDS=4', 'TVM_SHARD_INDEX=3'], { - unpack_lib('hexagon', tvm_lib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_hexagon) sh ( script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh", @@ -1099,8 +1345,21 @@ stage('Test') { try { init_git() withEnv(['PLATFORM=qemu'], { - unpack_lib('qemu', tvm_lib) - unpack_microtvm_template_projects('qemu') + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/config.cmake build/config.cmake + md5sum build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/microtvm_template_projects build/microtvm_template_projects --recursive + """, + label: 'Download artifacts from S3', + ) + + add_microtvm_permissions() ci_setup(ci_qemu) cpp_unittest(ci_qemu) sh ( @@ -1130,7 +1389,21 @@ stage('Test') { try { init_git() withEnv(['PLATFORM=arm'], { - unpack_lib('arm', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_arm) cpp_unittest(ci_arm) sh ( @@ -1163,7 +1436,21 @@ stage('Test') { 'PLATFORM=arm', 'TVM_NUM_SHARDS=2', 'TVM_SHARD_INDEX=0'], { - unpack_lib('arm', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_arm) python_unittest(ci_arm) sh ( @@ -1192,7 +1479,21 @@ stage('Test') { 'PLATFORM=arm', 'TVM_NUM_SHARDS=2', 'TVM_SHARD_INDEX=1'], { - unpack_lib('arm', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_arm) python_unittest(ci_arm) sh ( @@ -1221,7 +1522,21 @@ stage('Test') { 'PLATFORM=gpu', 'TVM_NUM_SHARDS=2', 'TVM_SHARD_INDEX=0'], { - unpack_lib('gpu', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", @@ -1249,7 +1564,21 @@ stage('Test') { 'PLATFORM=gpu', 'TVM_NUM_SHARDS=2', 'TVM_SHARD_INDEX=1'], { - unpack_lib('gpu', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", @@ -1277,7 +1606,21 @@ stage('Test') { 'PLATFORM=gpu', 'TVM_NUM_SHARDS=3', 'TVM_SHARD_INDEX=0'], { - unpack_lib('gpu', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", @@ -1305,7 +1648,21 @@ stage('Test') { 'PLATFORM=gpu', 'TVM_NUM_SHARDS=3', 'TVM_SHARD_INDEX=1'], { - unpack_lib('gpu', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", @@ -1333,7 +1690,21 @@ stage('Test') { 'PLATFORM=gpu', 'TVM_NUM_SHARDS=3', 'TVM_SHARD_INDEX=2'], { - unpack_lib('gpu', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", @@ -1358,7 +1729,21 @@ stage('Test') { try { init_git() withEnv(['PLATFORM=cpu'], { - unpack_lib('cpu', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_frontend_cpu.sh", @@ -1383,7 +1768,21 @@ stage('Test') { try { init_git() withEnv(['PLATFORM=arm'], { - unpack_lib('arm', tvm_multilib) + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + ci_setup(ci_arm) sh ( script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_frontend_cpu.sh", @@ -1405,8 +1804,23 @@ stage('Test') { node('GPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/docs-python-gpu") { init_git() - unpack_lib('gpu', tvm_multilib) - unpack_microtvm_template_projects('gpu') + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/microtvm_template_projects build/microtvm_template_projects --recursive + """, + label: 'Download artifacts from S3', + ) + + add_microtvm_permissions() timeout(time: 180, unit: 'MINUTES') { ci_setup(ci_gpu) sh ( @@ -1414,7 +1828,15 @@ stage('Test') { label: 'Build docs', ) } - pack_lib('docs', 'docs.tgz') + sh( + script: """ + set -eux + md5sum docs.tgz + aws s3 cp --no-progress docs.tgz s3://${s3_prefix}/docs/docs.tgz + """, + label: 'Upload artifacts to S3', + ) + archiveArtifacts(artifacts: 'docs.tgz', fingerprint: true) } } @@ -1489,7 +1911,15 @@ stage('Deploy') { if (env.BRANCH_NAME == 'main' && env.DOCS_DEPLOY_ENABLED == 'yes') { node('CPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/deploy-docs") { - unpack_lib('docs', 'docs.tgz') + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/docs/docs.tgz docs.tgz + md5sum docs.tgz + """, + label: 'Download artifacts from S3', + ) + deploy_docs() } } diff --git a/jenkins/Jenkinsfile.j2 b/jenkins/Jenkinsfile.j2 index f250ff12feed..8742d0724485 100644 --- a/jenkins/Jenkinsfile.j2 +++ b/jenkins/Jenkinsfile.j2 @@ -399,53 +399,14 @@ def make(docker_type, path, make_flag) { } } -// Specifications to Jenkins "stash" command for use with various pack_ and unpack_ functions. -tvm_runtime = 'build/libtvm_runtime.so, build/config.cmake' // use libtvm_runtime.so. -tvm_lib = 'build/libtvm.so, ' + tvm_runtime // use libtvm.so to run the full compiler. -// LLVM upstream lib -tvm_multilib = 'build/libtvm.so, ' + - 'build/libvta_fsim.so, ' + - tvm_runtime - -tvm_multilib_tsim = 'build/libvta_tsim.so, ' + - tvm_multilib - -microtvm_tar_gz = 'build/microtvm_template_projects.tar.gz' - -// pack libraries for later use -def pack_lib(name, libs) { - sh (script: """ - echo "Packing ${libs} into ${name}" - echo ${libs} | sed -e 's/,/ /g' | xargs md5sum - """, label: 'Stash libraries and show md5') - stash includes: libs, name: name -} - -// unpack libraries saved before -def unpack_lib(name, libs) { - unstash name - sh (script: """ - echo "Unpacked ${libs} from ${name}" - echo ${libs} | sed -e 's/,/ /g' | xargs md5sum - """, label: 'Unstash libraries and show md5') -} +// Filenames for stashing between build and test steps +{% set tvm_runtime = ['build/libtvm_runtime.so', 'build/config.cmake'] %} +{% set tvm_lib = ['build/libtvm.so'] + tvm_runtime %} +{% set tvm_multilib = ['build/libtvm.so', 'build/libvta_fsim.so'] + tvm_runtime %} +{% set tvm_multilib_tsim = ['build/libvta_tsim.so'] + tvm_multilib %} +{% set microtvm_template_projects = ['build/microtvm_template_projects',] %} +s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" -// compress microtvm template projects and pack the tar. -def pack_microtvm_template_projects(name) { - sh( - script: 'cd build && tar -czvf microtvm_template_projects.tar.gz microtvm_template_projects/', - label: 'Compress microtvm_template_projects' - ) - pack_lib(name + '-microtvm-libs', microtvm_tar_gz) -} - -def unpack_microtvm_template_projects(name) { - unpack_lib(name + '-microtvm-libs', microtvm_tar_gz) - sh( - script: 'cd build && tar -xzvf microtvm_template_projects.tar.gz', - label: 'Unpack microtvm_template_projects' - ) -} def ci_setup(image) { sh ( @@ -482,24 +443,36 @@ def cpp_unittest(image) { ) } + +def add_microtvm_permissions() { + {% for folder in microtvm_template_projects %} + sh( + script: 'find {{ folder }} -type f | xargs chmod +x', + label: 'Add execute permissions for microTVM files', + ) + {% endfor %} +} + + def build() { stage('Build') { environment { SKIP_SLOW_TESTS = "${skip_slow_tests}" } - parallel 'BUILD: GPU': { + parallel( + 'BUILD: GPU': { if (!skip_ci) { node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-gpu') }}) { init_git() sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" make("${ci_gpu} --no-gpu", 'build', '-j2') - pack_lib('gpu', tvm_multilib) - pack_microtvm_template_projects('gpu') + {{ m.upload_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }} + // compiler test sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build2" make("${ci_gpu} --no-gpu", 'build2', '-j2') - pack_lib('gpu2', tvm_multilib) + {{ m.upload_artifacts(tag='gpu2', filenames=tvm_multilib) }} } } } @@ -514,7 +487,7 @@ stage('Build') { label: 'Create CPU cmake config', ) make(ci_cpu, 'build', '-j2') - pack_lib('cpu', tvm_multilib_tsim) + {{ m.upload_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} timeout(time: max_time, unit: 'MINUTES') { ci_setup(ci_cpu) // sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh" @@ -561,7 +534,7 @@ stage('Build') { label: 'Create i386 cmake config', ) make(ci_i386, 'build', '-j2') - pack_lib('i386', tvm_multilib_tsim) + {{ m.upload_artifacts(tag='i386', filenames=tvm_multilib_tsim) }} } } } else { @@ -578,7 +551,7 @@ stage('Build') { label: 'Create ARM cmake config', ) make(ci_arm, 'build', '-j4') - pack_lib('arm', tvm_multilib) + {{ m.upload_artifacts(tag='arm', filenames=tvm_multilib) }} } } } else { @@ -595,8 +568,7 @@ stage('Build') { label: 'Create QEMU cmake config', ) make(ci_qemu, 'build', '-j2') - pack_lib('qemu', tvm_lib) - pack_microtvm_template_projects('qemu') + {{ m.upload_artifacts(tag='qemu', filenames=tvm_lib, folders=microtvm_template_projects) }} } } } else { @@ -613,13 +585,14 @@ stage('Build') { label: 'Create Hexagon cmake config', ) make(ci_hexagon, 'build', '-j2') - pack_lib('hexagon', tvm_lib) + {{ m.upload_artifacts(tag='hexagon', filenames=tvm_lib) }} } } } else { Utils.markStageSkippedForConditional('BUILD: Hexagon') } - } + }, + ) } } @@ -640,14 +613,14 @@ stage('Test') { platform="gpu", ) %} {% if shard_index == 1 %} - unpack_lib('gpu2', tvm_multilib) + {{ m.download_artifacts(tag='gpu2', filenames=tvm_multilib) }} cpp_unittest(ci_gpu) - unpack_lib('gpu', tvm_multilib) + {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} ci_setup(ci_gpu) cpp_unittest(ci_gpu) {% else %} - unpack_lib('gpu', tvm_multilib) + {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} ci_setup(ci_gpu) {% endif %} {% if shard_index == 2 or num_shards < 2 %} @@ -672,7 +645,7 @@ stage('Test') { ws="tvm/integration-python-cpu", platform="cpu", ) %} - unpack_lib('cpu', tvm_multilib_tsim) + {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} ci_setup(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", @@ -685,7 +658,7 @@ stage('Test') { ws="tvm/ut-python-cpu", platform="cpu", ) %} - unpack_lib('cpu', tvm_multilib_tsim) + {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} ci_setup(ci_cpu) cpp_unittest(ci_cpu) python_unittest(ci_cpu) @@ -702,7 +675,7 @@ stage('Test') { ws="tvm/integration-python-i386", platform="i386", ) %} - unpack_lib('i386', tvm_multilib) + {{ m.download_artifacts(tag='i386', filenames=tvm_multilib) }} ci_setup(ci_i386) {% if shard_index == 1 %} cpp_unittest(ci_i386) @@ -721,7 +694,7 @@ stage('Test') { platform="hexagon", num_shards=4, ) %} - unpack_lib('hexagon', tvm_lib) + {{ m.download_artifacts(tag='hexagon', filenames=tvm_lib) }} ci_setup(ci_hexagon) {% if shard_index == 1 %} cpp_unittest(ci_hexagon) @@ -741,8 +714,8 @@ stage('Test') { ws="tvm/test-qemu", platform="qemu", ) %} - unpack_lib('qemu', tvm_lib) - unpack_microtvm_template_projects('qemu') + {{ m.download_artifacts(tag='qemu', filenames=tvm_lib, folders=microtvm_template_projects) }} + add_microtvm_permissions() ci_setup(ci_qemu) cpp_unittest(ci_qemu) sh ( @@ -760,7 +733,7 @@ stage('Test') { ws="tvm/ut-python-arm", platform="arm", ) %} - unpack_lib('arm', tvm_multilib) + {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }} ci_setup(ci_arm) cpp_unittest(ci_arm) sh ( @@ -778,7 +751,7 @@ stage('Test') { node="ARM", ws="tvm/ut-python-arm", platform="arm", ) %} - unpack_lib('arm', tvm_multilib) + {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }} ci_setup(ci_arm) python_unittest(ci_arm) sh ( @@ -793,7 +766,7 @@ stage('Test') { ws="tvm/topi-python-gpu", platform="gpu", ) %} - unpack_lib('gpu', tvm_multilib) + {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", @@ -806,7 +779,7 @@ stage('Test') { ws="tvm/frontend-python-gpu", platform="gpu", ) %} - unpack_lib('gpu', tvm_multilib) + {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }} ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", @@ -819,7 +792,7 @@ stage('Test') { ws="tvm/frontend-python-cpu", platform="cpu", ) %} - unpack_lib('cpu', tvm_multilib) + {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib) }} ci_setup(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_frontend_cpu.sh", @@ -832,7 +805,7 @@ stage('Test') { ws="tvm/frontend-python-arm", platform="arm", ) %} - unpack_lib('arm', tvm_multilib) + {{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }} ci_setup(ci_arm) sh ( script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_frontend_cpu.sh", @@ -844,8 +817,8 @@ stage('Test') { node('GPU') { ws({{ m.per_exec_ws('tvm/docs-python-gpu') }}) { init_git() - unpack_lib('gpu', tvm_multilib) - unpack_microtvm_template_projects('gpu') + {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }} + add_microtvm_permissions() timeout(time: 180, unit: 'MINUTES') { ci_setup(ci_gpu) sh ( @@ -853,7 +826,7 @@ stage('Test') { label: 'Build docs', ) } - pack_lib('docs', 'docs.tgz') + {{ m.upload_artifacts(tag='docs', filenames=["docs.tgz"]) }} archiveArtifacts(artifacts: 'docs.tgz', fingerprint: true) } } @@ -928,7 +901,7 @@ stage('Deploy') { if (env.BRANCH_NAME == 'main' && env.DOCS_DEPLOY_ENABLED == 'yes') { node('CPU') { ws({{ m.per_exec_ws('tvm/deploy-docs') }}) { - unpack_lib('docs', 'docs.tgz') + {{ m.download_artifacts(tag='docs', filenames=["docs.tgz"]) }} deploy_docs() } } diff --git a/jenkins/macros.j2 b/jenkins/macros.j2 index de33a203f603..2ce005a128ef 100644 --- a/jenkins/macros.j2 +++ b/jenkins/macros.j2 @@ -89,3 +89,35 @@ } }, {% endmacro %} + +{% macro upload_artifacts(tag, filenames, folders=[]) %} +sh( + script: """ + set -eux + {% for filename in filenames %} + md5sum {{ filename }} + aws s3 cp --no-progress {{ filename }} s3://${s3_prefix}/{{ tag }}/{{ filename }} + {% endfor %} + {% for folder in (folders or []) %} + aws s3 cp --no-progress {{ folder }} s3://${s3_prefix}/{{ tag }}/{{ folder }} --recursive + {% endfor %} + """, + label: 'Upload artifacts to S3', + ) +{% endmacro %} + +{% macro download_artifacts(tag, filenames, folders=None) %} +sh( + script: """ + set -eux + {% for filename in filenames %} + aws s3 cp --no-progress s3://${s3_prefix}/{{ tag }}/{{ filename }} {{ filename }} + md5sum {{ filename }} + {% endfor %} + {% for folder in (folders or []) %} + aws s3 cp --no-progress s3://${s3_prefix}/{{ tag }}/{{ folder }} {{ folder }} --recursive + {% endfor %} + """, + label: 'Download artifacts from S3', + ) +{% endmacro %}