From aa423a21839e201a98c19adb7b879db08e381db9 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Tue, 2 Aug 2022 15:59:38 -0700 Subject: [PATCH 01/10] use torch-provided rocm in AMD runner --- .github/workflows/amd.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/amd.yml b/.github/workflows/amd.yml index 018624db461a..dbea5800045b 100644 --- a/.github/workflows/amd.yml +++ b/.github/workflows/amd.yml @@ -33,6 +33,9 @@ jobs: python --version which hipcc hipcc --version + pip install --upgrade pip + pip uninstall --yes torch torchvision + pip install torch torchvision --extra-index-url https://download.pytorch.org/whl/rocm5.1.1 python -c "import torch; print('torch:', torch.__version__, torch)" python -c "import torch; print('CUDA available:', torch.cuda.is_available())" sudo apt-get update From 88e04867b26773594832fdf3ffa817ed26d328e7 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Tue, 2 Aug 2022 16:22:16 -0700 Subject: [PATCH 02/10] fix for module level pytest skip --- .github/workflows/amd.yml | 4 ++-- tests/unit/ops/adagrad/test_cpu_adagrad.py | 2 +- tests/unit/ops/adam/test_cpu_adam.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/amd.yml b/.github/workflows/amd.yml index dbea5800045b..3f4f2b55ec0c 100644 --- a/.github/workflows/amd.yml +++ b/.github/workflows/amd.yml @@ -66,5 +66,5 @@ jobs: run: | if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi cd tests - TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -x -n 4 unit/ - TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -x -m 'sequential' unit/ + TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -n 4 unit/ + TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -m 'sequential' unit/ diff --git a/tests/unit/ops/adagrad/test_cpu_adagrad.py b/tests/unit/ops/adagrad/test_cpu_adagrad.py index 66e246ed23fc..6f530d0309fa 100755 --- a/tests/unit/ops/adagrad/test_cpu_adagrad.py +++ b/tests/unit/ops/adagrad/test_cpu_adagrad.py @@ -7,7 +7,7 @@ from deepspeed.ops.op_builder import CPUAdagradBuilder if not deepspeed.ops.__compatible_ops__[CPUAdagradBuilder.NAME]: - pytest.skip("cpu-adagrad is not compatible") + pytest.skip("cpu-adagrad is not compatible", allow_module_level=True) def check_equal(first, second, atol=1e-2, verbose=False): diff --git a/tests/unit/ops/adam/test_cpu_adam.py b/tests/unit/ops/adam/test_cpu_adam.py index 7357c086d08d..54389ce5fcf4 100755 --- a/tests/unit/ops/adam/test_cpu_adam.py +++ b/tests/unit/ops/adam/test_cpu_adam.py @@ -8,7 +8,7 @@ from deepspeed.ops.op_builder import CPUAdamBuilder if not deepspeed.ops.__compatible_ops__[CPUAdamBuilder.NAME]: - pytest.skip("cpu-adam is not compatible") + pytest.skip("cpu-adam is not compatible", allow_module_level=True) pytest.cpu_vendor = get_cpu_info()["vendor_id_raw"].lower() From 55c9ac743de45b3ad9511fd6592e313fc082b325 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Tue, 2 Aug 2022 16:38:09 -0700 Subject: [PATCH 03/10] restrict which tests are run --- .github/workflows/amd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/amd.yml b/.github/workflows/amd.yml index 3f4f2b55ec0c..79dfd5a798e4 100644 --- a/.github/workflows/amd.yml +++ b/.github/workflows/amd.yml @@ -66,5 +66,5 @@ jobs: run: | if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi cd tests - TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -n 4 unit/ - TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -m 'sequential' unit/ + TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -n 4 unit/{comm,inference,monitor,ops,profiling,runtime} + TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -m 'sequential' unit/{comm,inference,monitor,ops,profiling,runtime} From 5ff4ce6d1c5596529ca8f557f6fba9a428a9c8a1 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Tue, 2 Aug 2022 16:50:25 -0700 Subject: [PATCH 04/10] test fix for AMD --- tests/unit/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/common.py b/tests/unit/common.py index 7b418ebbf645..2eb82fab7bb6 100644 --- a/tests/unit/common.py +++ b/tests/unit/common.py @@ -91,7 +91,7 @@ def _get_test_kwargs(self, request): return test_kwargs def _launch_procs(self, num_procs): - mp.set_start_method('forkserver', force=True) + mp.set_start_method('spawn', force=True) skip_msg = mp.Queue() # Allows forked processes to share pytest.skip reason processes = [] for local_rank in range(num_procs): From d0494b5e68b704aa0dc016468748a99cb3e1d41d Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Tue, 2 Aug 2022 17:06:29 -0700 Subject: [PATCH 05/10] testing fix for cuda init errors --- .github/workflows/amd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/amd.yml b/.github/workflows/amd.yml index 79dfd5a798e4..9277a31b9bde 100644 --- a/.github/workflows/amd.yml +++ b/.github/workflows/amd.yml @@ -66,5 +66,5 @@ jobs: run: | if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi cd tests - TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -n 4 unit/{comm,inference,monitor,ops,profiling,runtime} + TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose unit/{comm,inference,monitor,ops,profiling,runtime} TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -m 'sequential' unit/{comm,inference,monitor,ops,profiling,runtime} From 9b15311214854244360eceb51fce1d4230442a2d Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Tue, 2 Aug 2022 17:14:31 -0700 Subject: [PATCH 06/10] test if DistTest will fix the error --- tests/unit/ops/adagrad/test_cpu_adagrad.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/unit/ops/adagrad/test_cpu_adagrad.py b/tests/unit/ops/adagrad/test_cpu_adagrad.py index 6f530d0309fa..9198df6b154b 100755 --- a/tests/unit/ops/adagrad/test_cpu_adagrad.py +++ b/tests/unit/ops/adagrad/test_cpu_adagrad.py @@ -125,12 +125,18 @@ def gen_sparse_grad(vocabulary_size, dim, num_indices, dtype, device): check_equal(param, param1, atol=1e-2, verbose=True) -def test_cpu_adam_gpu_error(): - model_size = 64 - device = 'cuda:0' - param = torch.nn.Parameter(torch.randn(model_size, device=device)) - optimizer = DeepSpeedCPUAdagrad([param]) +from tests.unit.common import DistributedTest - param.grad = torch.randn(model_size, device=device) - with pytest.raises(AssertionError): - optimizer.step() + +class TestCPUAdamGPUError(DistributedTest): + world_size = 1 + + def test(): + model_size = 64 + device = 'cuda:0' + param = torch.nn.Parameter(torch.randn(model_size, device=device)) + optimizer = DeepSpeedCPUAdagrad([param]) + + param.grad = torch.randn(model_size, device=device) + with pytest.raises(AssertionError): + optimizer.step() From 168a3e635c41e16695b243a13ebb2d900f8cf4cb Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Wed, 3 Aug 2022 10:10:10 -0700 Subject: [PATCH 07/10] remove --forked from pytest --- .github/workflows/amd.yml | 4 ++-- tests/unit/ops/adagrad/test_cpu_adagrad.py | 22 ++++++++-------------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/.github/workflows/amd.yml b/.github/workflows/amd.yml index 9277a31b9bde..305a659e574e 100644 --- a/.github/workflows/amd.yml +++ b/.github/workflows/amd.yml @@ -66,5 +66,5 @@ jobs: run: | if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi cd tests - TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose unit/{comm,inference,monitor,ops,profiling,runtime} - TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -m 'sequential' unit/{comm,inference,monitor,ops,profiling,runtime} + TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --verbose -n 4 unit/{comm,inference,monitor,ops,profiling,runtime} + TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --verbose -m 'sequential' unit/{comm,inference,monitor,ops,profiling,runtime} diff --git a/tests/unit/ops/adagrad/test_cpu_adagrad.py b/tests/unit/ops/adagrad/test_cpu_adagrad.py index 9198df6b154b..153dfaa292b1 100755 --- a/tests/unit/ops/adagrad/test_cpu_adagrad.py +++ b/tests/unit/ops/adagrad/test_cpu_adagrad.py @@ -125,18 +125,12 @@ def gen_sparse_grad(vocabulary_size, dim, num_indices, dtype, device): check_equal(param, param1, atol=1e-2, verbose=True) -from tests.unit.common import DistributedTest - - -class TestCPUAdamGPUError(DistributedTest): - world_size = 1 - - def test(): - model_size = 64 - device = 'cuda:0' - param = torch.nn.Parameter(torch.randn(model_size, device=device)) - optimizer = DeepSpeedCPUAdagrad([param]) +def test_cpu_adam_gpu_error(self): + model_size = 64 + device = 'cuda:0' + param = torch.nn.Parameter(torch.randn(model_size, device=device)) + optimizer = DeepSpeedCPUAdagrad([param]) - param.grad = torch.randn(model_size, device=device) - with pytest.raises(AssertionError): - optimizer.step() + param.grad = torch.randn(model_size, device=device) + with pytest.raises(AssertionError): + optimizer.step() From 8c8b37b140a3f8f5011a1f4c9befbc838d0b5f51 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Wed, 3 Aug 2022 10:16:25 -0700 Subject: [PATCH 08/10] Update test_cpu_adagrad.py --- tests/unit/ops/adagrad/test_cpu_adagrad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/ops/adagrad/test_cpu_adagrad.py b/tests/unit/ops/adagrad/test_cpu_adagrad.py index 153dfaa292b1..6f530d0309fa 100755 --- a/tests/unit/ops/adagrad/test_cpu_adagrad.py +++ b/tests/unit/ops/adagrad/test_cpu_adagrad.py @@ -125,7 +125,7 @@ def gen_sparse_grad(vocabulary_size, dim, num_indices, dtype, device): check_equal(param, param1, atol=1e-2, verbose=True) -def test_cpu_adam_gpu_error(self): +def test_cpu_adam_gpu_error(): model_size = 64 device = 'cuda:0' param = torch.nn.Parameter(torch.randn(model_size, device=device)) From a3a73adbee08c18750f2a2adfeb010ded60a8f67 Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Wed, 3 Aug 2022 10:24:49 -0700 Subject: [PATCH 09/10] Update amd.yml --- .github/workflows/amd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/amd.yml b/.github/workflows/amd.yml index 305a659e574e..bb2a72f8f09e 100644 --- a/.github/workflows/amd.yml +++ b/.github/workflows/amd.yml @@ -67,4 +67,4 @@ jobs: if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi cd tests TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --verbose -n 4 unit/{comm,inference,monitor,ops,profiling,runtime} - TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --verbose -m 'sequential' unit/{comm,inference,monitor,ops,profiling,runtime} + #TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --verbose -m 'sequential' unit/{comm,inference,monitor,ops,profiling,runtime} From 9065f1dbb22c0dce91c015f8906986b8aa8bc02c Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Wed, 3 Aug 2022 10:26:49 -0700 Subject: [PATCH 10/10] Update common.py --- tests/unit/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/common.py b/tests/unit/common.py index 2eb82fab7bb6..7b418ebbf645 100644 --- a/tests/unit/common.py +++ b/tests/unit/common.py @@ -91,7 +91,7 @@ def _get_test_kwargs(self, request): return test_kwargs def _launch_procs(self, num_procs): - mp.set_start_method('spawn', force=True) + mp.set_start_method('forkserver', force=True) skip_msg = mp.Queue() # Allows forked processes to share pytest.skip reason processes = [] for local_rank in range(num_procs):