From 24319adfaf23c9f4c25c3bbbe38d4eb36d0c6ceb Mon Sep 17 00:00:00 2001
From: Michael Wyatt <michaelwyatt@microsoft.com>
Date: Tue, 31 May 2022 12:02:38 -0700
Subject: [PATCH 01/14] added new workflows for latest and nightly torch

---
 .github/workflows/nv-torch-latest-v100.yml  | 56 +++++++++++++++++++++
 .github/workflows/nv-torch-nightly-v100.yml | 49 ++++++++++++++++++
 2 files changed, 105 insertions(+)
 create mode 100644 .github/workflows/nv-torch-latest-v100.yml
 create mode 100644 .github/workflows/nv-torch-nightly-v100.yml

diff --git a/.github/workflows/nv-torch-latest-v100.yml b/.github/workflows/nv-torch-latest-v100.yml
new file mode 100644
index 000000000000..25a0ed4a2d99
--- /dev/null
+++ b/.github/workflows/nv-torch-latest-v100.yml
@@ -0,0 +1,56 @@
+name: nv-torch-latest-v100
+
+on:
+  push:
+    branches:
+      - 'master'
+      - 'staging**'
+    paths-ignore:
+      - 'docs/**'
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit-tests:
+    runs-on: [self-hosted, nvidia, cu113, v100]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: environment
+        run: |
+          nvidia-smi
+          which python
+          python --version
+          which nvcc
+          nvcc --version
+          pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113
+          python -c "import torch; print('torch:', torch.__version__, torch)"
+          python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
+
+      - name: Install transformers
+        run: |
+          git clone https://github.com/huggingface/transformers
+          cd transformers
+          # if needed switch to the last known good SHA until transformers@master is fixed
+          # git checkout 1cc453d33
+          git rev-parse --short HEAD
+          pip install .
+
+      - name: Install deepspeed
+        run: |
+          pip install .[dev,1bit,autotuning,sparse_attn]
+          ds_report
+
+      - name: Unit tests
+        run: |
+          unset TORCH_CUDA_ARCH_LIST # only jit compile for current arch
+          if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi
+          cd tests
+          TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -n 4 -m 'not sequential' unit/
+          TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -m 'sequential' unit/
diff --git a/.github/workflows/nv-torch-nightly-v100.yml b/.github/workflows/nv-torch-nightly-v100.yml
new file mode 100644
index 000000000000..f969cadbdd4a
--- /dev/null
+++ b/.github/workflows/nv-torch-nightly-v100.yml
@@ -0,0 +1,49 @@
+name: nv-torch-nightly-v100
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit-tests:
+    runs-on: [self-hosted, nvidia, cu113, v100]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: environment
+        run: |
+          nvidia-smi
+          which python
+          python --version
+          which nvcc
+          nvcc --version
+          pip install --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cu113
+          python -c "import torch; print('torch:', torch.__version__, torch)"
+          python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
+
+      - name: Install transformers
+        run: |
+          git clone https://github.com/huggingface/transformers
+          cd transformers
+          # if needed switch to the last known good SHA until transformers@master is fixed
+          # git checkout 1cc453d33
+          git rev-parse --short HEAD
+          pip install .
+
+      - name: Install deepspeed
+        run: |
+          pip install .[dev,1bit,autotuning,sparse_attn]
+          ds_report
+
+      - name: Unit tests
+        run: |
+          unset TORCH_CUDA_ARCH_LIST # only jit compile for current arch
+          if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi
+          cd tests
+          TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -n 4 -m 'not sequential' unit/
+          TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --forked --verbose -m 'sequential' unit/

From 0f105d4e110ac1775ce2797cd2276f8d69a69a25 Mon Sep 17 00:00:00 2001
From: Michael Wyatt <michaelwyatt@microsoft.com>
Date: Tue, 31 May 2022 16:21:27 -0700
Subject: [PATCH 02/14] fixed issue with cuda initialize twice in distributed
 tests

---
 setup.py           |  6 ++++++
 tests/unit/util.py | 15 +++++----------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/setup.py b/setup.py
index b35bb48b70da..ae1678245976 100755
--- a/setup.py
+++ b/setup.py
@@ -221,17 +221,23 @@ def create_dir_symlink(src, dest):
     version_str += f'+{git_hash}'
 
 torch_version = ".".join([TORCH_MAJOR, TORCH_MINOR])
+bf16_support = False
 # Set cuda_version to 0.0 if cpu-only
 cuda_version = "0.0"
+nccl_version = "0.0"
 # Set hip_version to 0.0 if cpu-only
 hip_version = "0.0"
 if torch_available and torch.version.cuda is not None:
     cuda_version = ".".join(torch.version.cuda.split('.')[:2])
+    nccl_version = ".".join(map(str, torch.cuda.nccl.version()[:2]))
+    bf16_support = torch.cuda.is_bf16_supported()
 if torch_available and hasattr(torch.version, 'hip') and torch.version.hip is not None:
     hip_version = ".".join(torch.version.hip.split('.')[:2])
 torch_info = {
     "version": torch_version,
+    "bf16_support": bf16_support,
     "cuda_version": cuda_version,
+    "nccl_version": nccl_version,
     "hip_version": hip_version
 }
 
diff --git a/tests/unit/util.py b/tests/unit/util.py
index 966733b1d929..79a459da3c14 100644
--- a/tests/unit/util.py
+++ b/tests/unit/util.py
@@ -13,20 +13,15 @@ def required_torch_version():
 
 
 def bf16_required_version_check():
-    TORCH_MAJOR = int(torch.__version__.split('.')[0])
-    TORCH_MINOR = int(torch.__version__.split('.')[1])
-
-    if type(torch.cuda.nccl.version()) != tuple:
-        return False
-    else:
-        NCCL_MAJOR = torch.cuda.nccl.version()[0]
-        NCCL_MINOR = torch.cuda.nccl.version()[1]
+    split_version = lambda x: map(int, x.split('.')[:2])
+    TORCH_MAJOR, TORCH_MINOR = split_version(torch_info['version'])
+    NCCL_MAJOR, NCCL_MINOR = split_version(torch_info['nccl_version'])
+    CUDA_MAJOR, CUDA_MINOR = split_version(torch_info['cuda_version'])
 
-    CUDA_MAJOR = int(torch_info['cuda_version'].split('.')[0])
     if (TORCH_MAJOR > 1 or
         (TORCH_MAJOR == 1 and TORCH_MINOR >= 10)) and (CUDA_MAJOR >= 11) and (
             NCCL_MAJOR > 2 or
-            (NCCL_MAJOR == 2 and NCCL_MINOR >= 10)) and torch.cuda.is_bf16_supported():
+            (NCCL_MAJOR == 2 and NCCL_MINOR >= 10)) and torch_info['bf16_support']:
         return True
     else:
         return False

From 278ee17ba0947da19c00402260cd957313a93531 Mon Sep 17 00:00:00 2001
From: Michael Wyatt <michaelwyatt@microsoft.com>
Date: Tue, 31 May 2022 17:12:47 -0700
Subject: [PATCH 03/14] fix for older pytorch with nccl version

---
 setup.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index ae1678245976..dbb7f1532244 100755
--- a/setup.py
+++ b/setup.py
@@ -229,7 +229,11 @@ def create_dir_symlink(src, dest):
 hip_version = "0.0"
 if torch_available and torch.version.cuda is not None:
     cuda_version = ".".join(torch.version.cuda.split('.')[:2])
-    nccl_version = ".".join(map(str, torch.cuda.nccl.version()[:2]))
+    if isinstance(torch.cuda.nccl.version(), int):
+        # This will break if minor version > 9
+        nccl_version = ".".join(str(torch.cuda.nccl.version())[:2])
+    else:
+        nccl_version = ".".join(map(str, torch.cuda.nccl.version()[:2]))
     bf16_support = torch.cuda.is_bf16_supported()
 if torch_available and hasattr(torch.version, 'hip') and torch.version.hip is not None:
     hip_version = ".".join(torch.version.hip.split('.')[:2])

From 4f113bb78ea62875938cb9f1f020794ab69e9f6e Mon Sep 17 00:00:00 2001
From: Michael Wyatt <michaelwyatt@microsoft.com>
Date: Tue, 31 May 2022 17:14:56 -0700
Subject: [PATCH 04/14] another fix for older torch versions

---
 setup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index dbb7f1532244..40c48aa473b8 100755
--- a/setup.py
+++ b/setup.py
@@ -234,7 +234,8 @@ def create_dir_symlink(src, dest):
         nccl_version = ".".join(str(torch.cuda.nccl.version())[:2])
     else:
         nccl_version = ".".join(map(str, torch.cuda.nccl.version()[:2]))
-    bf16_support = torch.cuda.is_bf16_supported()
+    if torch.cuda.is_bf16_supported is not None:
+        bf16_support = torch.cuda.is_bf16_supported()
 if torch_available and hasattr(torch.version, 'hip') and torch.version.hip is not None:
     hip_version = ".".join(torch.version.hip.split('.')[:2])
 torch_info = {

From 70f257da68d584254ba6d7a1cd7c9a79bab33259 Mon Sep 17 00:00:00 2001
From: Michael Wyatt <michaelwyatt@microsoft.com>
Date: Tue, 31 May 2022 17:21:06 -0700
Subject: [PATCH 05/14] really fixing for old torch... I hope

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 40c48aa473b8..4a0105f17fe8 100755
--- a/setup.py
+++ b/setup.py
@@ -234,7 +234,7 @@ def create_dir_symlink(src, dest):
         nccl_version = ".".join(str(torch.cuda.nccl.version())[:2])
     else:
         nccl_version = ".".join(map(str, torch.cuda.nccl.version()[:2]))
-    if torch.cuda.is_bf16_supported is not None:
+    if hasattr(torch.cuda, 'is_bf16_supported'):
         bf16_support = torch.cuda.is_bf16_supported()
 if torch_available and hasattr(torch.version, 'hip') and torch.version.hip is not None:
     hip_version = ".".join(torch.version.hip.split('.')[:2])

From 279e7f13041079e1c998b3b701c8a3b249aa22ae Mon Sep 17 00:00:00 2001
From: Michael Wyatt <mrwyattii@gmail.com>
Date: Fri, 3 Jun 2022 14:02:47 -0700
Subject: [PATCH 06/14] Update nv-torch-latest-v100.yml

---
 .github/workflows/nv-torch-latest-v100.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/nv-torch-latest-v100.yml b/.github/workflows/nv-torch-latest-v100.yml
index 25a0ed4a2d99..6cc9ae21e72a 100644
--- a/.github/workflows/nv-torch-latest-v100.yml
+++ b/.github/workflows/nv-torch-latest-v100.yml
@@ -29,7 +29,9 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113
+          PIP=$(which pip)
+          sudo $PIP uninstall --yes torch torchvision
+          sudo $PIP install torch torchvision --extra-index-url https://download.pytorch.org/whl/cu113
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
@@ -40,11 +42,15 @@ jobs:
           # if needed switch to the last known good SHA until transformers@master is fixed
           # git checkout 1cc453d33
           git rev-parse --short HEAD
-          pip install .
+          PIP=$(which pip)
+          sudo $PIP uninstall --yes transformers
+          sudo $PIP install .
 
       - name: Install deepspeed
         run: |
-          pip install .[dev,1bit,autotuning,sparse_attn]
+          PIP=$(which pip)
+          sudo $PIP uninstall --yes deepspeed
+          sudo $PIP install .[dev,1bit,autotuning,sparse_attn]
           ds_report
 
       - name: Unit tests

From 4da17c7a7c4f755c87b0bfadd47656e7cb6a5b7e Mon Sep 17 00:00:00 2001
From: Michael Wyatt <michaelwyatt@microsoft.com>
Date: Fri, 3 Jun 2022 15:25:36 -0700
Subject: [PATCH 07/14] modified nvidia tests

---
 .github/workflows/nv-lightning-v100.yml     | 16 +++++++++++-----
 .github/workflows/nv-torch-latest-v100.yml  |  6 +++---
 .github/workflows/nv-torch-nightly-v100.yml | 12 +++++++++---
 .github/workflows/nv-torch12-p40.yml        | 13 ++++++++++---
 .github/workflows/nv-torch18-v100.yml       | 14 ++++++++++----
 .github/workflows/nv-transformers-v100.yml  | 13 +++++++++----
 6 files changed, 52 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/nv-lightning-v100.yml b/.github/workflows/nv-lightning-v100.yml
index 6caceb48f2c4..eb9306f87079 100644
--- a/.github/workflows/nv-lightning-v100.yml
+++ b/.github/workflows/nv-lightning-v100.yml
@@ -17,7 +17,10 @@ concurrency:
 
 jobs:
   unit-tests:
-    runs-on: [self-hosted, nvidia, torch18, v100]
+    runs-on: [self-hosted, nvidia, cu111, v100]
+
+    env:
+      PIP: $(which pip)
 
     steps:
       - uses: actions/checkout@v2
@@ -29,17 +32,20 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          pip install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
+          sudo $PIP uninstall --yes torch torchvision
+          sudo $PIP install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
       - name: Install deepspeed
         run: |
-          pip install .[dev,autotuning]
+          sudo $PIP uninstall --yes deepspeed
+          sudo $PIP install .[dev,autotuning]
           ds_report
       - name: PyTorch Lightning Tests
         run: |
           if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi
-          pip install pytorch-lightning
-          pip install "protobuf<4.21.0"
+          sudo $PIP uninstall --yes pytorch-lightning
+          sudo $PIP install pytorch-lightning
+          sudo $PIP install "protobuf<4.21.0"
           cd tests
           TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --verbose lightning/
diff --git a/.github/workflows/nv-torch-latest-v100.yml b/.github/workflows/nv-torch-latest-v100.yml
index 6cc9ae21e72a..1a52f4b9405c 100644
--- a/.github/workflows/nv-torch-latest-v100.yml
+++ b/.github/workflows/nv-torch-latest-v100.yml
@@ -19,6 +19,9 @@ jobs:
   unit-tests:
     runs-on: [self-hosted, nvidia, cu113, v100]
 
+    env:
+      PIP: $(which pip)
+
     steps:
       - uses: actions/checkout@v2
 
@@ -29,7 +32,6 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          PIP=$(which pip)
           sudo $PIP uninstall --yes torch torchvision
           sudo $PIP install torch torchvision --extra-index-url https://download.pytorch.org/whl/cu113
           python -c "import torch; print('torch:', torch.__version__, torch)"
@@ -42,13 +44,11 @@ jobs:
           # if needed switch to the last known good SHA until transformers@master is fixed
           # git checkout 1cc453d33
           git rev-parse --short HEAD
-          PIP=$(which pip)
           sudo $PIP uninstall --yes transformers
           sudo $PIP install .
 
       - name: Install deepspeed
         run: |
-          PIP=$(which pip)
           sudo $PIP uninstall --yes deepspeed
           sudo $PIP install .[dev,1bit,autotuning,sparse_attn]
           ds_report
diff --git a/.github/workflows/nv-torch-nightly-v100.yml b/.github/workflows/nv-torch-nightly-v100.yml
index f969cadbdd4a..8bc127274261 100644
--- a/.github/workflows/nv-torch-nightly-v100.yml
+++ b/.github/workflows/nv-torch-nightly-v100.yml
@@ -12,6 +12,9 @@ jobs:
   unit-tests:
     runs-on: [self-hosted, nvidia, cu113, v100]
 
+    env:
+      PIP: $(which pip)
+
     steps:
       - uses: actions/checkout@v2
 
@@ -22,7 +25,8 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          pip install --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cu113
+          sudo $PIP uninstall --yes torch torchvision
+          sudo $PIP install --pre torch torchvision --extra-index-url https://download.pytorch.org/whl/nightly/cu113
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
@@ -33,11 +37,13 @@ jobs:
           # if needed switch to the last known good SHA until transformers@master is fixed
           # git checkout 1cc453d33
           git rev-parse --short HEAD
-          pip install .
+          sudo $PIP uninstall --yes transformers
+          sudo $PIP install .
 
       - name: Install deepspeed
         run: |
-          pip install .[dev,1bit,autotuning,sparse_attn]
+          sudo $PIP uninstall --yes deepspeed
+          sudo $PIP install .[dev,1bit,autotuning,sparse_attn]
           ds_report
 
       - name: Unit tests
diff --git a/.github/workflows/nv-torch12-p40.yml b/.github/workflows/nv-torch12-p40.yml
index 080543df6980..be47cb056eac 100644
--- a/.github/workflows/nv-torch12-p40.yml
+++ b/.github/workflows/nv-torch12-p40.yml
@@ -17,7 +17,10 @@ concurrency:
 
 jobs:
   unit-tests:
-    runs-on: [self-hosted, nvidia, torch12, p40]
+    runs-on: [self-hosted, nvidia, cu101, p40]
+
+    env:
+      PIP: $(which pip)
 
     steps:
       - uses: actions/checkout@v2
@@ -29,6 +32,8 @@ jobs:
           python --version
           which nvcc
           nvcc --version
+          sudo $PIP uninstall --yes torch torchvision
+          sudo $PIP torch==1.2.0 torchvision==0.4.0
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
@@ -39,11 +44,13 @@ jobs:
           # if needed switch to the last known good SHA until transformers@master is fixed
           # git checkout 1cc453d33
           git rev-parse --short HEAD
-          pip install .
+          sudo $PIP uninstall --yes transformers
+          sudo $PIP install .
 
       - name: Install deepspeed
         run: |
-          pip install .[dev,autotuning]
+          sudo $PIP uninstall --yes deepspeed
+          sudo $PIP install .[dev,1bit,autotuning,sparse_attn]
           ds_report
 
       - name: Unit tests
diff --git a/.github/workflows/nv-torch18-v100.yml b/.github/workflows/nv-torch18-v100.yml
index 0afac798119a..6042e5e7ea1a 100644
--- a/.github/workflows/nv-torch18-v100.yml
+++ b/.github/workflows/nv-torch18-v100.yml
@@ -17,7 +17,10 @@ concurrency:
 
 jobs:
   unit-tests:
-    runs-on: [self-hosted, nvidia, torch18, v100]
+    runs-on: [self-hosted, nvidia, cu111, v100]
+
+    env:
+      PIP: $(which pip)
 
     steps:
       - uses: actions/checkout@v2
@@ -29,7 +32,8 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          pip install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
+          sudo $PIP uninstall --yes torch torchvision
+          sudo $PIP install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
@@ -40,11 +44,13 @@ jobs:
           # if needed switch to the last known good SHA until transformers@master is fixed
           # git checkout 1cc453d33
           git rev-parse --short HEAD
-          pip install .
+          sudo $PIP uninstall --yes transformers
+          sudo $PIP install .
 
       - name: Install deepspeed
         run: |
-          pip install .[dev,1bit,autotuning,sparse_attn]
+          sudo $PIP uninstall --yes deepspeed
+          sudo $PIP install .[dev,1bit,autotuning,sparse_attn]
           ds_report
 
       - name: Unit tests
diff --git a/.github/workflows/nv-transformers-v100.yml b/.github/workflows/nv-transformers-v100.yml
index c39e6978b15e..1db4ba7ea763 100644
--- a/.github/workflows/nv-transformers-v100.yml
+++ b/.github/workflows/nv-transformers-v100.yml
@@ -17,7 +17,10 @@ concurrency:
 
 jobs:
   unit-tests:
-    runs-on: [self-hosted, nvidia, torch18, v100]
+    runs-on: [self-hosted, nvidia, cu111, v100]
+
+    env:
+      PIP: $(which pip)
 
     steps:
       - uses: actions/checkout@v2
@@ -29,12 +32,14 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          pip install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
+          sudo $PIP uninstall --yes torch torchvision
+          sudo $PIP install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
       - name: Install deepspeed
         run: |
-          pip install .[dev,autotuning]
+          sudo $PIP uninstall --yes deepspeed
+          sudo $PIP install .[dev,autotuning]
           ds_report
       - name: HF transformers tests
         run: |
@@ -45,7 +50,7 @@ jobs:
           # git checkout 1cc453d33
           git rev-parse --short HEAD
           # scipy/sklearn required for tests, using the 'dev' extra forces torch re-install
-          pip install .[testing]
+          sudo $PIP install .[testing]
           # find reqs used in ds integration tests
           find examples/pytorch  -regextype posix-egrep -regex '.*(language-modeling|question-answering|summarization|image-classification|text-classification|translation).*/requirements.txt' -exec pip install -r {} \;
           TORCH_EXTENSIONS_DIR=./torch-extensions RUN_SLOW=1 pytest --color=yes --durations=0 --verbose tests/deepspeed

From befb0c3d3e6375444e9f96c5fd8cf4be45e71890 Mon Sep 17 00:00:00 2001
From: Michael Wyatt <michaelwyatt@microsoft.com>
Date: Fri, 3 Jun 2022 15:30:00 -0700
Subject: [PATCH 08/14] fix for env var pointing to pip binary

---
 .github/workflows/nv-torch-latest-v100.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/nv-torch-latest-v100.yml b/.github/workflows/nv-torch-latest-v100.yml
index 1a52f4b9405c..bf28843c1c0c 100644
--- a/.github/workflows/nv-torch-latest-v100.yml
+++ b/.github/workflows/nv-torch-latest-v100.yml
@@ -19,14 +19,12 @@ jobs:
   unit-tests:
     runs-on: [self-hosted, nvidia, cu113, v100]
 
-    env:
-      PIP: $(which pip)
-
     steps:
       - uses: actions/checkout@v2
 
       - name: environment
         run: |
+          echo "PIP=$(which pip)" >> $GITHUB_ENV
           nvidia-smi
           which python
           python --version

From de8f3a7c3cfc16d4b93c4f4cac7804a755912f5e Mon Sep 17 00:00:00 2001
From: Michael Wyatt <michaelwyatt@microsoft.com>
Date: Fri, 3 Jun 2022 15:31:40 -0700
Subject: [PATCH 09/14] fix for env var pointing to pip binary

---
 .github/workflows/nv-torch-latest-v100.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/nv-torch-latest-v100.yml b/.github/workflows/nv-torch-latest-v100.yml
index bf28843c1c0c..9d95ae8fab62 100644
--- a/.github/workflows/nv-torch-latest-v100.yml
+++ b/.github/workflows/nv-torch-latest-v100.yml
@@ -30,8 +30,8 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          sudo $PIP uninstall --yes torch torchvision
-          sudo $PIP install torch torchvision --extra-index-url https://download.pytorch.org/whl/cu113
+          sudo ${{ env.PIP }} uninstall --yes torch torchvision
+          sudo ${{ env.PIP }} install torch torchvision --extra-index-url https://download.pytorch.org/whl/cu113
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
@@ -42,13 +42,13 @@ jobs:
           # if needed switch to the last known good SHA until transformers@master is fixed
           # git checkout 1cc453d33
           git rev-parse --short HEAD
-          sudo $PIP uninstall --yes transformers
-          sudo $PIP install .
+          sudo ${{ env.PIP }} uninstall --yes transformers
+          sudo ${{ env.PIP }} install .
 
       - name: Install deepspeed
         run: |
-          sudo $PIP uninstall --yes deepspeed
-          sudo $PIP install .[dev,1bit,autotuning,sparse_attn]
+          sudo ${{ env.PIP }} uninstall --yes deepspeed
+          sudo ${{ env.PIP }} install .[dev,1bit,autotuning,sparse_attn]
           ds_report
 
       - name: Unit tests

From 3599f961c09c651e257797ba18a5424dca009702 Mon Sep 17 00:00:00 2001
From: Michael Wyatt <michaelwyatt@microsoft.com>
Date: Fri, 3 Jun 2022 15:37:15 -0700
Subject: [PATCH 10/14] fix for env var pointing to pip binary

---
 .github/workflows/nv-torch-latest-v100.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/nv-torch-latest-v100.yml b/.github/workflows/nv-torch-latest-v100.yml
index 9d95ae8fab62..71f18417c742 100644
--- a/.github/workflows/nv-torch-latest-v100.yml
+++ b/.github/workflows/nv-torch-latest-v100.yml
@@ -22,9 +22,12 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: environment
+      - name: set pip
         run: |
           echo "PIP=$(which pip)" >> $GITHUB_ENV
+
+      - name: environment
+        run: |
           nvidia-smi
           which python
           python --version

From 86f86d8aaa40d76c03e1cd9dbf3487748235e520 Mon Sep 17 00:00:00 2001
From: Michael Wyatt <michaelwyatt@microsoft.com>
Date: Fri, 3 Jun 2022 15:42:35 -0700
Subject: [PATCH 11/14] propagate fix to rest of workflows

---
 .github/workflows/nv-lightning-v100.yml     | 20 ++++++++++----------
 .github/workflows/nv-torch-nightly-v100.yml | 19 ++++++++++---------
 .github/workflows/nv-torch12-p40.yml        | 19 ++++++++++---------
 .github/workflows/nv-torch18-v100.yml       | 19 ++++++++++---------
 .github/workflows/nv-transformers-v100.yml  | 17 +++++++++--------
 5 files changed, 49 insertions(+), 45 deletions(-)

diff --git a/.github/workflows/nv-lightning-v100.yml b/.github/workflows/nv-lightning-v100.yml
index eb9306f87079..921cbce2326c 100644
--- a/.github/workflows/nv-lightning-v100.yml
+++ b/.github/workflows/nv-lightning-v100.yml
@@ -19,12 +19,12 @@ jobs:
   unit-tests:
     runs-on: [self-hosted, nvidia, cu111, v100]
 
-    env:
-      PIP: $(which pip)
-
     steps:
       - uses: actions/checkout@v2
 
+      - name: set pip
+          echo "PIP=$(which pip)" >> $GITHUB_ENV
+
       - name: environment
         run: |
           nvidia-smi
@@ -32,20 +32,20 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          sudo $PIP uninstall --yes torch torchvision
-          sudo $PIP install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
+          sudo ${{ env.PIP }} uninstall --yes torch torchvision
+          sudo ${{ env.PIP }} install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
       - name: Install deepspeed
         run: |
-          sudo $PIP uninstall --yes deepspeed
-          sudo $PIP install .[dev,autotuning]
+          sudo ${{ env.PIP }} uninstall --yes deepspeed
+          sudo ${{ env.PIP }} install .[dev,autotuning]
           ds_report
       - name: PyTorch Lightning Tests
         run: |
           if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi
-          sudo $PIP uninstall --yes pytorch-lightning
-          sudo $PIP install pytorch-lightning
-          sudo $PIP install "protobuf<4.21.0"
+          sudo ${{ env.PIP }} uninstall --yes pytorch-lightning
+          sudo ${{ env.PIP }} install pytorch-lightning
+          sudo ${{ env.PIP }} install "protobuf<4.21.0"
           cd tests
           TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --verbose lightning/
diff --git a/.github/workflows/nv-torch-nightly-v100.yml b/.github/workflows/nv-torch-nightly-v100.yml
index 8bc127274261..0160acc0c6fb 100644
--- a/.github/workflows/nv-torch-nightly-v100.yml
+++ b/.github/workflows/nv-torch-nightly-v100.yml
@@ -12,12 +12,13 @@ jobs:
   unit-tests:
     runs-on: [self-hosted, nvidia, cu113, v100]
 
-    env:
-      PIP: $(which pip)
-
     steps:
       - uses: actions/checkout@v2
 
+      - name: set pip
+        run: |
+          echo "PIP=$(which pip)" >> $GITHUB_ENV
+
       - name: environment
         run: |
           nvidia-smi
@@ -25,8 +26,8 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          sudo $PIP uninstall --yes torch torchvision
-          sudo $PIP install --pre torch torchvision --extra-index-url https://download.pytorch.org/whl/nightly/cu113
+          sudo ${{ env.PIP }} uninstall --yes torch torchvision
+          sudo ${{ env.PIP }} install --pre torch torchvision --extra-index-url https://download.pytorch.org/whl/nightly/cu113
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
@@ -37,13 +38,13 @@ jobs:
           # if needed switch to the last known good SHA until transformers@master is fixed
           # git checkout 1cc453d33
           git rev-parse --short HEAD
-          sudo $PIP uninstall --yes transformers
-          sudo $PIP install .
+          sudo ${{ env.PIP }} uninstall --yes transformers
+          sudo ${{ env.PIP }} install .
 
       - name: Install deepspeed
         run: |
-          sudo $PIP uninstall --yes deepspeed
-          sudo $PIP install .[dev,1bit,autotuning,sparse_attn]
+          sudo ${{ env.PIP }} uninstall --yes deepspeed
+          sudo ${{ env.PIP }} install .[dev,1bit,autotuning,sparse_attn]
           ds_report
 
       - name: Unit tests
diff --git a/.github/workflows/nv-torch12-p40.yml b/.github/workflows/nv-torch12-p40.yml
index be47cb056eac..ed2a31164ab1 100644
--- a/.github/workflows/nv-torch12-p40.yml
+++ b/.github/workflows/nv-torch12-p40.yml
@@ -19,12 +19,13 @@ jobs:
   unit-tests:
     runs-on: [self-hosted, nvidia, cu101, p40]
 
-    env:
-      PIP: $(which pip)
-
     steps:
       - uses: actions/checkout@v2
 
+      - name: set pip
+        run: |
+          echo "PIP=$(which pip)" >> $GITHUB_ENV
+
       - name: environment
         run: |
           nvidia-smi
@@ -32,8 +33,8 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          sudo $PIP uninstall --yes torch torchvision
-          sudo $PIP torch==1.2.0 torchvision==0.4.0
+          sudo ${{ env.PIP }} uninstall --yes torch torchvision
+          sudo ${{ env.PIP }} torch==1.2.0 torchvision==0.4.0
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
@@ -44,13 +45,13 @@ jobs:
           # if needed switch to the last known good SHA until transformers@master is fixed
           # git checkout 1cc453d33
           git rev-parse --short HEAD
-          sudo $PIP uninstall --yes transformers
-          sudo $PIP install .
+          sudo ${{ env.PIP }} uninstall --yes transformers
+          sudo ${{ env.PIP }} install .
 
       - name: Install deepspeed
         run: |
-          sudo $PIP uninstall --yes deepspeed
-          sudo $PIP install .[dev,1bit,autotuning,sparse_attn]
+          sudo ${{ env.PIP }} uninstall --yes deepspeed
+          sudo ${{ env.PIP }} install .[dev,1bit,autotuning,sparse_attn]
           ds_report
 
       - name: Unit tests
diff --git a/.github/workflows/nv-torch18-v100.yml b/.github/workflows/nv-torch18-v100.yml
index 6042e5e7ea1a..b8e320600c6b 100644
--- a/.github/workflows/nv-torch18-v100.yml
+++ b/.github/workflows/nv-torch18-v100.yml
@@ -19,12 +19,13 @@ jobs:
   unit-tests:
     runs-on: [self-hosted, nvidia, cu111, v100]
 
-    env:
-      PIP: $(which pip)
-
     steps:
       - uses: actions/checkout@v2
 
+      - name: set pip
+        run: |
+          echo "PIP=$(which pip)" >> $GITHUB_ENV
+
       - name: environment
         run: |
           nvidia-smi
@@ -32,8 +33,8 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          sudo $PIP uninstall --yes torch torchvision
-          sudo $PIP install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
+          sudo ${{ env.PIP }} uninstall --yes torch torchvision
+          sudo ${{ env.PIP }} install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
@@ -44,13 +45,13 @@ jobs:
           # if needed switch to the last known good SHA until transformers@master is fixed
           # git checkout 1cc453d33
           git rev-parse --short HEAD
-          sudo $PIP uninstall --yes transformers
-          sudo $PIP install .
+          sudo ${{ env.PIP }} uninstall --yes transformers
+          sudo ${{ env.PIP }} install .
 
       - name: Install deepspeed
         run: |
-          sudo $PIP uninstall --yes deepspeed
-          sudo $PIP install .[dev,1bit,autotuning,sparse_attn]
+          sudo ${{ env.PIP }} uninstall --yes deepspeed
+          sudo ${{ env.PIP }} install .[dev,1bit,autotuning,sparse_attn]
           ds_report
 
       - name: Unit tests
diff --git a/.github/workflows/nv-transformers-v100.yml b/.github/workflows/nv-transformers-v100.yml
index 1db4ba7ea763..7e08c8118873 100644
--- a/.github/workflows/nv-transformers-v100.yml
+++ b/.github/workflows/nv-transformers-v100.yml
@@ -19,12 +19,13 @@ jobs:
   unit-tests:
     runs-on: [self-hosted, nvidia, cu111, v100]
 
-    env:
-      PIP: $(which pip)
-
     steps:
       - uses: actions/checkout@v2
 
+      - name: set pip
+        run: |
+          echo "PIP=$(which pip)" >> $GITHUB_ENV
+
       - name: environment
         run: |
           nvidia-smi
@@ -32,14 +33,14 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          sudo $PIP uninstall --yes torch torchvision
-          sudo $PIP install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
+          sudo ${{ env.PIP }} uninstall --yes torch torchvision
+          sudo ${{ env.PIP }} install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
       - name: Install deepspeed
         run: |
-          sudo $PIP uninstall --yes deepspeed
-          sudo $PIP install .[dev,autotuning]
+          sudo ${{ env.PIP }} uninstall --yes deepspeed
+          sudo ${{ env.PIP }} install .[dev,autotuning]
           ds_report
       - name: HF transformers tests
         run: |
@@ -50,7 +51,7 @@ jobs:
           # git checkout 1cc453d33
           git rev-parse --short HEAD
           # scipy/sklearn required for tests, using the 'dev' extra forces torch re-install
-          sudo $PIP install .[testing]
+          sudo ${{ env.PIP }} install .[testing]
           # find reqs used in ds integration tests
           find examples/pytorch  -regextype posix-egrep -regex '.*(language-modeling|question-answering|summarization|image-classification|text-classification|translation).*/requirements.txt' -exec pip install -r {} \;
           TORCH_EXTENSIONS_DIR=./torch-extensions RUN_SLOW=1 pytest --color=yes --durations=0 --verbose tests/deepspeed

From 87418c8c958670d34b73fa7bf9c355f9688020c9 Mon Sep 17 00:00:00 2001
From: Michael Wyatt <michaelwyatt@microsoft.com>
Date: Fri, 3 Jun 2022 16:18:26 -0700
Subject: [PATCH 12/14] fixed typo

---
 .github/workflows/nv-torch12-p40.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/nv-torch12-p40.yml b/.github/workflows/nv-torch12-p40.yml
index ed2a31164ab1..4cf8c4ef62a6 100644
--- a/.github/workflows/nv-torch12-p40.yml
+++ b/.github/workflows/nv-torch12-p40.yml
@@ -34,7 +34,7 @@ jobs:
           which nvcc
           nvcc --version
           sudo ${{ env.PIP }} uninstall --yes torch torchvision
-          sudo ${{ env.PIP }} torch==1.2.0 torchvision==0.4.0
+          sudo ${{ env.PIP }} install torch==1.2.0 torchvision==0.4.0
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 

From 85750304779342491b86be9a64574bb04c6f4c73 Mon Sep 17 00:00:00 2001
From: Michael Wyatt <michaelwyatt@microsoft.com>
Date: Mon, 6 Jun 2022 11:44:24 -0700
Subject: [PATCH 13/14] removed sudo pip commands in favor of venv setup with
 runner script

---
 .github/workflows/nv-lightning-v100.yml     | 17 +++++++----------
 .github/workflows/nv-torch-latest-v100.yml  | 16 ++++++----------
 .github/workflows/nv-torch-nightly-v100.yml | 16 ++++++----------
 .github/workflows/nv-torch12-p40.yml        | 16 ++++++----------
 .github/workflows/nv-torch18-v100.yml       | 16 ++++++----------
 .github/workflows/nv-transformers-v100.yml  | 14 +++++---------
 6 files changed, 36 insertions(+), 59 deletions(-)

diff --git a/.github/workflows/nv-lightning-v100.yml b/.github/workflows/nv-lightning-v100.yml
index 921cbce2326c..f7a6a9be3879 100644
--- a/.github/workflows/nv-lightning-v100.yml
+++ b/.github/workflows/nv-lightning-v100.yml
@@ -22,9 +22,6 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: set pip
-          echo "PIP=$(which pip)" >> $GITHUB_ENV
-
       - name: environment
         run: |
           nvidia-smi
@@ -32,20 +29,20 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          sudo ${{ env.PIP }} uninstall --yes torch torchvision
-          sudo ${{ env.PIP }} install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
+          pip uninstall --yes torch torchvision
+          pip install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
       - name: Install deepspeed
         run: |
-          sudo ${{ env.PIP }} uninstall --yes deepspeed
-          sudo ${{ env.PIP }} install .[dev,autotuning]
+          pip uninstall --yes deepspeed
+          pip install .[dev,autotuning]
           ds_report
       - name: PyTorch Lightning Tests
         run: |
           if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi
-          sudo ${{ env.PIP }} uninstall --yes pytorch-lightning
-          sudo ${{ env.PIP }} install pytorch-lightning
-          sudo ${{ env.PIP }} install "protobuf<4.21.0"
+          pip uninstall --yes pytorch-lightning
+          pip install pytorch-lightning
+          pip install "protobuf<4.21.0"
           cd tests
           TORCH_EXTENSIONS_DIR=./torch-extensions pytest --color=yes --durations=0 --verbose lightning/
diff --git a/.github/workflows/nv-torch-latest-v100.yml b/.github/workflows/nv-torch-latest-v100.yml
index 71f18417c742..28cfe911e822 100644
--- a/.github/workflows/nv-torch-latest-v100.yml
+++ b/.github/workflows/nv-torch-latest-v100.yml
@@ -22,10 +22,6 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: set pip
-        run: |
-          echo "PIP=$(which pip)" >> $GITHUB_ENV
-
       - name: environment
         run: |
           nvidia-smi
@@ -33,8 +29,8 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          sudo ${{ env.PIP }} uninstall --yes torch torchvision
-          sudo ${{ env.PIP }} install torch torchvision --extra-index-url https://download.pytorch.org/whl/cu113
+          pip uninstall --yes torch torchvision
+          pip install torch torchvision --extra-index-url https://download.pytorch.org/whl/cu113
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
@@ -45,13 +41,13 @@ jobs:
           # if needed switch to the last known good SHA until transformers@master is fixed
           # git checkout 1cc453d33
           git rev-parse --short HEAD
-          sudo ${{ env.PIP }} uninstall --yes transformers
-          sudo ${{ env.PIP }} install .
+          pip uninstall --yes transformers
+          pip install .
 
       - name: Install deepspeed
         run: |
-          sudo ${{ env.PIP }} uninstall --yes deepspeed
-          sudo ${{ env.PIP }} install .[dev,1bit,autotuning,sparse_attn]
+          pip uninstall --yes deepspeed
+          pip install .[dev,1bit,autotuning,sparse_attn]
           ds_report
 
       - name: Unit tests
diff --git a/.github/workflows/nv-torch-nightly-v100.yml b/.github/workflows/nv-torch-nightly-v100.yml
index 0160acc0c6fb..de5f33804619 100644
--- a/.github/workflows/nv-torch-nightly-v100.yml
+++ b/.github/workflows/nv-torch-nightly-v100.yml
@@ -15,10 +15,6 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: set pip
-        run: |
-          echo "PIP=$(which pip)" >> $GITHUB_ENV
-
       - name: environment
         run: |
           nvidia-smi
@@ -26,8 +22,8 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          sudo ${{ env.PIP }} uninstall --yes torch torchvision
-          sudo ${{ env.PIP }} install --pre torch torchvision --extra-index-url https://download.pytorch.org/whl/nightly/cu113
+          pip uninstall --yes torch torchvision
+          pip install --pre torch torchvision --extra-index-url https://download.pytorch.org/whl/nightly/cu113
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
@@ -38,13 +34,13 @@ jobs:
           # if needed switch to the last known good SHA until transformers@master is fixed
           # git checkout 1cc453d33
           git rev-parse --short HEAD
-          sudo ${{ env.PIP }} uninstall --yes transformers
-          sudo ${{ env.PIP }} install .
+          pip uninstall --yes transformers
+          pip install .
 
       - name: Install deepspeed
         run: |
-          sudo ${{ env.PIP }} uninstall --yes deepspeed
-          sudo ${{ env.PIP }} install .[dev,1bit,autotuning,sparse_attn]
+          pip uninstall --yes deepspeed
+          pip install .[dev,1bit,autotuning,sparse_attn]
           ds_report
 
       - name: Unit tests
diff --git a/.github/workflows/nv-torch12-p40.yml b/.github/workflows/nv-torch12-p40.yml
index 4cf8c4ef62a6..5d4a6ea08ea9 100644
--- a/.github/workflows/nv-torch12-p40.yml
+++ b/.github/workflows/nv-torch12-p40.yml
@@ -22,10 +22,6 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: set pip
-        run: |
-          echo "PIP=$(which pip)" >> $GITHUB_ENV
-
       - name: environment
         run: |
           nvidia-smi
@@ -33,8 +29,8 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          sudo ${{ env.PIP }} uninstall --yes torch torchvision
-          sudo ${{ env.PIP }} install torch==1.2.0 torchvision==0.4.0
+          pip uninstall --yes torch torchvision
+          pip install torch==1.2.0 torchvision==0.4.0
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
@@ -45,13 +41,13 @@ jobs:
           # if needed switch to the last known good SHA until transformers@master is fixed
           # git checkout 1cc453d33
           git rev-parse --short HEAD
-          sudo ${{ env.PIP }} uninstall --yes transformers
-          sudo ${{ env.PIP }} install .
+          pip uninstall --yes transformers
+          pip install .
 
       - name: Install deepspeed
         run: |
-          sudo ${{ env.PIP }} uninstall --yes deepspeed
-          sudo ${{ env.PIP }} install .[dev,1bit,autotuning,sparse_attn]
+          pip uninstall --yes deepspeed
+          pip install .[dev,1bit,autotuning,sparse_attn]
           ds_report
 
       - name: Unit tests
diff --git a/.github/workflows/nv-torch18-v100.yml b/.github/workflows/nv-torch18-v100.yml
index b8e320600c6b..a26510695443 100644
--- a/.github/workflows/nv-torch18-v100.yml
+++ b/.github/workflows/nv-torch18-v100.yml
@@ -22,10 +22,6 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: set pip
-        run: |
-          echo "PIP=$(which pip)" >> $GITHUB_ENV
-
       - name: environment
         run: |
           nvidia-smi
@@ -33,8 +29,8 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          sudo ${{ env.PIP }} uninstall --yes torch torchvision
-          sudo ${{ env.PIP }} install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
+          pip uninstall --yes torch torchvision
+          pip install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
 
@@ -45,13 +41,13 @@ jobs:
           # if needed switch to the last known good SHA until transformers@master is fixed
           # git checkout 1cc453d33
           git rev-parse --short HEAD
-          sudo ${{ env.PIP }} uninstall --yes transformers
-          sudo ${{ env.PIP }} install .
+          pip uninstall --yes transformers
+          pip install .
 
       - name: Install deepspeed
         run: |
-          sudo ${{ env.PIP }} uninstall --yes deepspeed
-          sudo ${{ env.PIP }} install .[dev,1bit,autotuning,sparse_attn]
+          pip uninstall --yes deepspeed
+          pip install .[dev,1bit,autotuning,sparse_attn]
           ds_report
 
       - name: Unit tests
diff --git a/.github/workflows/nv-transformers-v100.yml b/.github/workflows/nv-transformers-v100.yml
index 7e08c8118873..c79eca32ef58 100644
--- a/.github/workflows/nv-transformers-v100.yml
+++ b/.github/workflows/nv-transformers-v100.yml
@@ -22,10 +22,6 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: set pip
-        run: |
-          echo "PIP=$(which pip)" >> $GITHUB_ENV
-
       - name: environment
         run: |
           nvidia-smi
@@ -33,14 +29,14 @@ jobs:
           python --version
           which nvcc
           nvcc --version
-          sudo ${{ env.PIP }} uninstall --yes torch torchvision
-          sudo ${{ env.PIP }} install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
+          pip uninstall --yes torch torchvision
+          pip install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
           python -c "import torch; print('torch:', torch.__version__, torch)"
           python -c "import torch; print('CUDA available:', torch.cuda.is_available())"
       - name: Install deepspeed
         run: |
-          sudo ${{ env.PIP }} uninstall --yes deepspeed
-          sudo ${{ env.PIP }} install .[dev,autotuning]
+          pip uninstall --yes deepspeed
+          pip install .[dev,autotuning]
           ds_report
       - name: HF transformers tests
         run: |
@@ -51,7 +47,7 @@ jobs:
           # git checkout 1cc453d33
           git rev-parse --short HEAD
           # scipy/sklearn required for tests, using the 'dev' extra forces torch re-install
-          sudo ${{ env.PIP }} install .[testing]
+          pip install .[testing]
           # find reqs used in ds integration tests
           find examples/pytorch  -regextype posix-egrep -regex '.*(language-modeling|question-answering|summarization|image-classification|text-classification|translation).*/requirements.txt' -exec pip install -r {} \;
           TORCH_EXTENSIONS_DIR=./torch-extensions RUN_SLOW=1 pytest --color=yes --durations=0 --verbose tests/deepspeed

From 6db80da25e1542bcacc51942a68988016b20860e Mon Sep 17 00:00:00 2001
From: Michael Wyatt <michaelwyatt@microsoft.com>
Date: Mon, 6 Jun 2022 11:51:32 -0700
Subject: [PATCH 14/14] upgrade pip as part of env setup

---
 .github/workflows/nv-lightning-v100.yml     | 1 +
 .github/workflows/nv-torch-latest-v100.yml  | 1 +
 .github/workflows/nv-torch-nightly-v100.yml | 1 +
 .github/workflows/nv-torch12-p40.yml        | 1 +
 .github/workflows/nv-torch18-v100.yml       | 1 +
 .github/workflows/nv-transformers-v100.yml  | 1 +
 6 files changed, 6 insertions(+)

diff --git a/.github/workflows/nv-lightning-v100.yml b/.github/workflows/nv-lightning-v100.yml
index f7a6a9be3879..8fb38525a8c7 100644
--- a/.github/workflows/nv-lightning-v100.yml
+++ b/.github/workflows/nv-lightning-v100.yml
@@ -29,6 +29,7 @@ jobs:
           python --version
           which nvcc
           nvcc --version
+          pip install --upgrade pip
           pip uninstall --yes torch torchvision
           pip install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
           python -c "import torch; print('torch:', torch.__version__, torch)"
diff --git a/.github/workflows/nv-torch-latest-v100.yml b/.github/workflows/nv-torch-latest-v100.yml
index 28cfe911e822..e38a7094ebae 100644
--- a/.github/workflows/nv-torch-latest-v100.yml
+++ b/.github/workflows/nv-torch-latest-v100.yml
@@ -29,6 +29,7 @@ jobs:
           python --version
           which nvcc
           nvcc --version
+          pip install --upgrade pip
           pip uninstall --yes torch torchvision
           pip install torch torchvision --extra-index-url https://download.pytorch.org/whl/cu113
           python -c "import torch; print('torch:', torch.__version__, torch)"
diff --git a/.github/workflows/nv-torch-nightly-v100.yml b/.github/workflows/nv-torch-nightly-v100.yml
index de5f33804619..26941bb29c92 100644
--- a/.github/workflows/nv-torch-nightly-v100.yml
+++ b/.github/workflows/nv-torch-nightly-v100.yml
@@ -22,6 +22,7 @@ jobs:
           python --version
           which nvcc
           nvcc --version
+          pip install --upgrade pip
           pip uninstall --yes torch torchvision
           pip install --pre torch torchvision --extra-index-url https://download.pytorch.org/whl/nightly/cu113
           python -c "import torch; print('torch:', torch.__version__, torch)"
diff --git a/.github/workflows/nv-torch12-p40.yml b/.github/workflows/nv-torch12-p40.yml
index 5d4a6ea08ea9..ff6d4b481746 100644
--- a/.github/workflows/nv-torch12-p40.yml
+++ b/.github/workflows/nv-torch12-p40.yml
@@ -29,6 +29,7 @@ jobs:
           python --version
           which nvcc
           nvcc --version
+          pip install --upgrade pip
           pip uninstall --yes torch torchvision
           pip install torch==1.2.0 torchvision==0.4.0
           python -c "import torch; print('torch:', torch.__version__, torch)"
diff --git a/.github/workflows/nv-torch18-v100.yml b/.github/workflows/nv-torch18-v100.yml
index a26510695443..abea2e06ea41 100644
--- a/.github/workflows/nv-torch18-v100.yml
+++ b/.github/workflows/nv-torch18-v100.yml
@@ -29,6 +29,7 @@ jobs:
           python --version
           which nvcc
           nvcc --version
+          pip install --upgrade pip
           pip uninstall --yes torch torchvision
           pip install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
           python -c "import torch; print('torch:', torch.__version__, torch)"
diff --git a/.github/workflows/nv-transformers-v100.yml b/.github/workflows/nv-transformers-v100.yml
index c79eca32ef58..9c0ce2757292 100644
--- a/.github/workflows/nv-transformers-v100.yml
+++ b/.github/workflows/nv-transformers-v100.yml
@@ -29,6 +29,7 @@ jobs:
           python --version
           which nvcc
           nvcc --version
+          pip install --upgrade pip
           pip uninstall --yes torch torchvision
           pip install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
           python -c "import torch; print('torch:', torch.__version__, torch)"