From 03ae88edc20598811bfeaef3516e2f80b1de8496 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Wed, 8 May 2024 15:26:17 +0800 Subject: [PATCH 01/13] download smb files --- .../py-packaging-training-cuda-stage-steps.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml index f6b36733ebdd8..8b211e0638b43 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml @@ -203,6 +203,21 @@ stages: condition: succeededOrFailed() workingDirectory: $(Build.SourcesDirectory) + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '/mnist' + artifactName: 'mnist' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '/bert_data' + artifactName: 'bert-data' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '/bert_data' + artifactName: 'hf_models_cache' + - task: CmdLine@2 displayName: 'test ortmodule' inputs: From df86a1140f4011d5076184db13d67895302f8762 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Wed, 8 May 2024 15:33:05 +0800 Subject: [PATCH 02/13] update --- ...py-packaging-training-cuda-stage-steps.yml | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml index 8b211e0638b43..99070b9c050a3 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml @@ -155,6 +155,36 @@ stages: - template: set-python-manylinux-variables-step.yml + - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/mnist" -d "/mnist" + displayName: 'Mount MNIST' + condition: succeededOrFailed() + workingDirectory: $(Build.SourcesDirectory) + + - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/bert-data" -d "/bert_data" + displayName: 'Mount bert-data' + condition: succeededOrFailed() + workingDirectory: $(Build.SourcesDirectory) + + - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/hf-models-cache" -d "/hf_models_cache" + displayName: 'Mount hf-models-cache' + condition: succeededOrFailed() + workingDirectory: $(Build.SourcesDirectory) + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '/mnist' + artifactName: 'mnist' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '/bert_data' + artifactName: 'bert-data' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '/bert_data' + artifactName: 'hf_models_cache' + - template: flex-downloadPipelineArtifact.yml parameters: ArtifactName: "onnxruntime_gpu_${{ variables['buildConfig'] }}_${{ parameters.python_version }}" @@ -188,36 +218,6 @@ stages: --build-arg LD_LIBRARY_PATH_ARG=/usr/local/lib64 Repository: $(Repository) - - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/mnist" -d "/mnist" - displayName: 'Mount MNIST' - condition: succeededOrFailed() - workingDirectory: $(Build.SourcesDirectory) - - - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/bert-data" -d "/bert_data" - displayName: 'Mount bert-data' - condition: succeededOrFailed() - workingDirectory: $(Build.SourcesDirectory) - - - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/hf-models-cache" -d "/hf_models_cache" - displayName: 'Mount hf-models-cache' - condition: succeededOrFailed() - workingDirectory: $(Build.SourcesDirectory) - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '/mnist' - artifactName: 'mnist' - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '/bert_data' - artifactName: 'bert-data' - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '/bert_data' - artifactName: 'hf_models_cache' - - task: CmdLine@2 displayName: 'test ortmodule' inputs: From 2ab920c684dccb1d3f3b25c0a0e3ed9a1113700a Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Wed, 8 May 2024 15:40:47 +0800 Subject: [PATCH 03/13] update --- .../templates/py-packaging-training-cuda-stage-steps.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml index 99070b9c050a3..ce57f40377e75 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml @@ -153,22 +153,22 @@ stages: clean: true submodules: none - - template: set-python-manylinux-variables-step.yml + # - template: set-python-manylinux-variables-step.yml - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/mnist" -d "/mnist" displayName: 'Mount MNIST' condition: succeededOrFailed() - workingDirectory: $(Build.SourcesDirectory) + workingDirectory: $(Build.SourcesDirectory)/onnxruntime - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/bert-data" -d "/bert_data" displayName: 'Mount bert-data' condition: succeededOrFailed() - workingDirectory: $(Build.SourcesDirectory) + workingDirectory: $(Build.SourcesDirectory)/onnxruntime - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/hf-models-cache" -d "/hf_models_cache" displayName: 'Mount hf-models-cache' condition: succeededOrFailed() - workingDirectory: $(Build.SourcesDirectory) + workingDirectory: $(Build.SourcesDirectory)/onnxruntime - task: PublishPipelineArtifact@1 inputs: From d2d021f4343ca6d8afa7a008b60044b4deb47dcf Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Wed, 8 May 2024 18:32:40 +0800 Subject: [PATCH 04/13] typo --- .../templates/py-packaging-training-cuda-stage-steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml index ce57f40377e75..e7ae63ee57d4a 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml @@ -182,7 +182,7 @@ stages: - task: PublishPipelineArtifact@1 inputs: - targetPath: '/bert_data' + targetPath: '/hf_models_cache' artifactName: 'hf_models_cache' - template: flex-downloadPipelineArtifact.yml From cac17865f0d33cde0cafbfe389a44d9367f81918 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Thu, 9 May 2024 11:01:35 +0800 Subject: [PATCH 05/13] donwload data from blob --- .../jobs/download_training_test_data.yml | 8 ++++ ...py-packaging-training-cuda-stage-steps.yml | 38 ++----------------- 2 files changed, 12 insertions(+), 34 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml new file mode 100644 index 0000000000000..54c9d7a863bba --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml @@ -0,0 +1,8 @@ +steps: + - script: | + azcopy cp --recursive https://orttrainingtestdatascus.blob.core.windows.net/MNIST/ $(Agent.TempDirectory)/MNIST + displayName: 'Download Training Test Data MNIST' + + - script: | + ls -al $(Agent.TempDirectory)/MNIST + displayName: 'Print contents of Training Test Data MNIST' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml index e7ae63ee57d4a..dfb33209e98fc 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml @@ -153,37 +153,7 @@ stages: clean: true submodules: none - # - template: set-python-manylinux-variables-step.yml - - - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/mnist" -d "/mnist" - displayName: 'Mount MNIST' - condition: succeededOrFailed() - workingDirectory: $(Build.SourcesDirectory)/onnxruntime - - - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/bert-data" -d "/bert_data" - displayName: 'Mount bert-data' - condition: succeededOrFailed() - workingDirectory: $(Build.SourcesDirectory)/onnxruntime - - - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/hf-models-cache" -d "/hf_models_cache" - displayName: 'Mount hf-models-cache' - condition: succeededOrFailed() - workingDirectory: $(Build.SourcesDirectory)/onnxruntime - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '/mnist' - artifactName: 'mnist' - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '/bert_data' - artifactName: 'bert-data' - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '/hf_models_cache' - artifactName: 'hf_models_cache' + - template: set-python-manylinux-variables-step.yml - template: flex-downloadPipelineArtifact.yml parameters: @@ -218,6 +188,8 @@ stages: --build-arg LD_LIBRARY_PATH_ARG=/usr/local/lib64 Repository: $(Repository) + - template: jobs/download_training_test_data.yml + - task: CmdLine@2 displayName: 'test ortmodule' inputs: @@ -230,9 +202,7 @@ stages: --gpus all \ -e NVIDIA_VISIBLE_DEVICES=all \ --volume $(Build.ArtifactStagingDirectory):/build \ - --volume /mnist:/mnist \ - --volume /bert_data:/bert_data \ - --volume /hf_models_cache:/hf_models_cache \ + --volume $(Agent.TempDirectory)/MNIST:/mnist \ $(Repository) \ bash -c " $(PythonManylinuxDir)/bin/python3 -m pip install /build/Release/dist/$basefilename && $(PythonManylinuxDir)/bin/python3 -m onnxruntime.training.ortmodule.torch_cpp_extensions.install " ; workingDirectory: $(Build.SourcesDirectory) From 5045140b2d8c805603ba8ea148e7178c512c85a5 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Thu, 9 May 2024 12:25:59 +0800 Subject: [PATCH 06/13] typo --- .../templates/jobs/download_training_test_data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml index 54c9d7a863bba..3e89331858c24 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml @@ -1,6 +1,6 @@ steps: - script: | - azcopy cp --recursive https://orttrainingtestdatascus.blob.core.windows.net/MNIST/ $(Agent.TempDirectory)/MNIST + azcopy cp --recursive https://orttrainingtestdatascus.blob.core.windows.net/testdata/MNIST/ $(Agent.TempDirectory)/MNIST displayName: 'Download Training Test Data MNIST' - script: | From ca8f644b5c2b9e99d69564a4ab01d4c62494cc9b Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Thu, 9 May 2024 13:56:41 +0800 Subject: [PATCH 07/13] update --- .../templates/jobs/download_training_test_data.yml | 2 +- .../templates/py-packaging-training-cuda-stage-steps.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml index 3e89331858c24..a6e7e463b051f 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml @@ -1,6 +1,6 @@ steps: - script: | - azcopy cp --recursive https://orttrainingtestdatascus.blob.core.windows.net/testdata/MNIST/ $(Agent.TempDirectory)/MNIST + azcopy cp --recursive https://.blob.core.windows.net/orttrainingtestdatascus/MNIST/ $(Agent.TempDirectory)/MNIST displayName: 'Download Training Test Data MNIST' - script: | diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml index dfb33209e98fc..0890883bbe1e2 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage-steps.yml @@ -153,6 +153,8 @@ stages: clean: true submodules: none + - template: jobs/download_training_test_data.yml + - template: set-python-manylinux-variables-step.yml - template: flex-downloadPipelineArtifact.yml @@ -188,8 +190,6 @@ stages: --build-arg LD_LIBRARY_PATH_ARG=/usr/local/lib64 Repository: $(Repository) - - template: jobs/download_training_test_data.yml - - task: CmdLine@2 displayName: 'test ortmodule' inputs: From 70f93c90520afb1800b3666be9cc529de5368297 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Thu, 9 May 2024 14:09:32 +0800 Subject: [PATCH 08/13] typo --- .../templates/jobs/download_training_test_data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml index a6e7e463b051f..7e20308ae8e54 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml @@ -1,6 +1,6 @@ steps: - script: | - azcopy cp --recursive https://.blob.core.windows.net/orttrainingtestdatascus/MNIST/ $(Agent.TempDirectory)/MNIST + azcopy cp --recursive https://lotusscus.blob.core.windows.net/orttrainingtestdatascus/MNIST/ $(Agent.TempDirectory)/MNIST displayName: 'Download Training Test Data MNIST' - script: | From dcf4cea4af9d8acb945cfd59e8ea0162cca4b653 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Thu, 9 May 2024 14:17:01 +0800 Subject: [PATCH 09/13] update --- .../templates/jobs/download_training_test_data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml index 7e20308ae8e54..4e378b6a0cd67 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml @@ -1,6 +1,6 @@ steps: - script: | - azcopy cp --recursive https://lotusscus.blob.core.windows.net/orttrainingtestdatascus/MNIST/ $(Agent.TempDirectory)/MNIST + azcopy cp --recursive https://lotusscus.blob.core.windows.net/orttrainingtestdatascus/MNIST/ $(Agent.TempDirectory) displayName: 'Download Training Test Data MNIST' - script: | From f77245c374b2ec9aa3d70ed2f4f189e3fcb86973 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Thu, 9 May 2024 15:19:01 +0800 Subject: [PATCH 10/13] update all orttrainingtestdatascus --- ...ortmodule-distributed-test-ci-pipeline.yml | 6 +-- .../jobs/download_training_test_data.yml | 4 +- ...orttraining-linux-gpu-test-ci-pipeline.yml | 16 +----- .../azure_scale_set_vm_mount_test_data.sh | 53 ------------------- 4 files changed, 6 insertions(+), 73 deletions(-) delete mode 100755 tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml index 654bc0921556a..65e8c4d689d38 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml @@ -58,9 +58,7 @@ stages: -e DisplayName: 'Build' - - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/mnist" -d "/mnist" - displayName: 'Mount MNIST' - condition: succeededOrFailed() + - templates: templates/jobs/download_training_test_data.yml # Entry point for all ORTModule distributed tests # Refer to orttraining/orttraining/test/python/how_to_add_ortmodule_distributed_ci_pipeline_tests.md for guidelines on how to add new tests to this pipeline. @@ -71,7 +69,7 @@ stages: --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ - --volume /mnist:/mnist \ + --volume $(Agent.TempDirectory)/mnist:/mnist \ onnxruntime_ortmodule_distributed_tests_image \ bash -c "rm -rf /build/RelWithDebInfo/onnxruntime/ && python3 -m pip install /build/RelWithDebInfo/dist/onnxruntime*.whl && python3 -m onnxruntime.training.ortmodule.torch_cpp_extensions.install && /build/RelWithDebInfo/launch_test.py --cmd_line_with_args 'python orttraining_ortmodule_distributed_tests.py --mnist /mnist' --cwd /build/RelWithDebInfo" \ displayName: 'Run orttraining_ortmodule_distributed_tests.py' diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml index 4e378b6a0cd67..8f6434f7ac40d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/download_training_test_data.yml @@ -1,8 +1,8 @@ steps: - script: | - azcopy cp --recursive https://lotusscus.blob.core.windows.net/orttrainingtestdatascus/MNIST/ $(Agent.TempDirectory) + azcopy cp --recursive https://lotusscus.blob.core.windows.net/orttrainingtestdatascus/mnist/ $(Agent.TempDirectory) displayName: 'Download Training Test Data MNIST' - script: | - ls -al $(Agent.TempDirectory)/MNIST + ls -al $(Agent.TempDirectory)/mnist displayName: 'Print contents of Training Test Data MNIST' diff --git a/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-test-ci-pipeline.yml index 5dc156e301357..837a8da84e25d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-test-ci-pipeline.yml @@ -6,17 +6,7 @@ parameters: steps: -- bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/mnist" -d "/mnist" - displayName: 'Mount MNIST' - condition: succeededOrFailed() - -- bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/bert-data" -d "/bert_data" - displayName: 'Mount bert-data' - condition: succeededOrFailed() - -- bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/hf-models-cache" -d "/hf_models_cache" - displayName: 'Mount hf-models-cache' - condition: succeededOrFailed() + - templates: templates/jobs/download_training_test_data.yml # Entry point for all ORTModule tests # The onnxruntime folder is deleted in the build directory @@ -29,9 +19,7 @@ steps: --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory)/${{ parameters.BuildConfig }}:/build \ - --volume /mnist:/mnist \ - --volume /bert_data:/bert_data \ - --volume /hf_models_cache:/hf_models_cache \ + --volume $(Agent.TempDirectory)/mnist:/mnist \ ${{ parameters.DockerImageTag }} \ bash -c "rm -rf /build/onnxruntime/ && python3 -m pip install /build/dist/onnxruntime*.whl && python3 -m onnxruntime.training.ortmodule.torch_cpp_extensions.install && /build/launch_test.py --cmd_line_with_args 'python orttraining_ortmodule_tests.py --mnist /mnist --bert_data /bert_data/hf_data/glue_data/CoLA/original/raw' --cwd /build" \ displayName: 'Run orttraining_ortmodule_tests.py' diff --git a/tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh b/tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh deleted file mode 100755 index 8a09f86f005cb..0000000000000 --- a/tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh +++ /dev/null @@ -1,53 +0,0 @@ -#! /usr/bin/env bash - -function credentialize () { - if [ ! -d "/etc/smbcredentials" ]; then - sudo mkdir /etc/smbcredentials - fi - - if [ -f "/etc/smbcredentials/orttrainingtestdatascus.cred" ]; then - sudo rm /etc/smbcredentials/orttrainingtestdatascus.cred - fi - - # to create orttrainingtestdatascus.cred, I have to do: 'sudo bash -c ...' - sudo bash -c 'echo "username=orttrainingtestdatascus" >> /etc/smbcredentials/orttrainingtestdatascus.cred' - - # $1 get removed (do defend injection attack?) if I do 'sudo bash -c...' - # to enable 'sudo echo...' I need to 'sudo chmod 777...' first. - sudo chmod 777 /etc/smbcredentials/orttrainingtestdatascus.cred - sudo echo "password=$1" >> /etc/smbcredentials/orttrainingtestdatascus.cred - - sudo chmod 600 /etc/smbcredentials/orttrainingtestdatascus.cred -} - -function mount_data () { - echo "Mounting source $1 at destination $2" - - if [ -d $2 ]; then - sudo umount $2 - if [ $? != 0 ]; then - echo "umount failed" - fi - fi - - if [ -d $2 ]; then - sudo rmdir $2 - fi - - sudo mkdir -p $2 - - sudo bash -c 'echo "$1 $2 cifs nofail,vers=3.0,credentials=/etc/smbcredentials/orttrainingtestdatascus.cred,dir_mode=0777,file_mode=0777,serverino" >> /etc/fstab' -- $1 $2 - sudo mount -t cifs $1 $2 -o vers=3.0,credentials=/etc/smbcredentials/orttrainingtestdatascus.cred,dir_mode=0777,file_mode=0777,serverino -} - -while getopts "p:s:d:" opt; do - case $opt in - p) storage_account_password=$OPTARG;; - s) data_source=$OPTARG;; - d) data_destination=$OPTARG;; - esac -done - -credentialize $storage_account_password - -mount_data $data_source $data_destination From 9bc0d220d663a7711025856d220c0d0da1691f51 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Thu, 9 May 2024 21:43:37 +0800 Subject: [PATCH 11/13] typo --- ...raining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml | 2 +- .../templates/orttraining-linux-gpu-test-ci-pipeline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml index 65e8c4d689d38..01304f2b9349b 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml @@ -58,7 +58,7 @@ stages: -e DisplayName: 'Build' - - templates: templates/jobs/download_training_test_data.yml + - template: templates/jobs/download_training_test_data.yml # Entry point for all ORTModule distributed tests # Refer to orttraining/orttraining/test/python/how_to_add_ortmodule_distributed_ci_pipeline_tests.md for guidelines on how to add new tests to this pipeline. diff --git a/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-test-ci-pipeline.yml index 837a8da84e25d..10aff15522509 100644 --- a/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-test-ci-pipeline.yml @@ -6,7 +6,7 @@ parameters: steps: - - templates: templates/jobs/download_training_test_data.yml +- template: templates/jobs/download_training_test_data.yml # Entry point for all ORTModule tests # The onnxruntime folder is deleted in the build directory From c1f1fa1405f87654a1b64d524e1b8e467147308f Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Thu, 9 May 2024 21:46:15 +0800 Subject: [PATCH 12/13] typo --- .../templates/orttraining-linux-gpu-test-ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-test-ci-pipeline.yml index 10aff15522509..f832315c1f0df 100644 --- a/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/templates/orttraining-linux-gpu-test-ci-pipeline.yml @@ -6,7 +6,7 @@ parameters: steps: -- template: templates/jobs/download_training_test_data.yml +- template: jobs/download_training_test_data.yml # Entry point for all ORTModule tests # The onnxruntime folder is deleted in the build directory From 5bda4b87493a9be0680b80a9e08f789bd9673aec Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Thu, 9 May 2024 23:11:30 +0800 Subject: [PATCH 13/13] move download before build --- ...ining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml index 01304f2b9349b..2c6b6183a9aa0 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml @@ -41,6 +41,8 @@ stages: clean: true submodules: recursive + - template: templates/jobs/download_training_test_data.yml + - template: templates/run-docker-build-steps.yml parameters: RunDockerBuildArgs: | @@ -58,8 +60,6 @@ stages: -e DisplayName: 'Build' - - template: templates/jobs/download_training_test_data.yml - # Entry point for all ORTModule distributed tests # Refer to orttraining/orttraining/test/python/how_to_add_ortmodule_distributed_ci_pipeline_tests.md for guidelines on how to add new tests to this pipeline. - script: |