diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2dd9bf203d6c..db67534e1936 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,6 +42,14 @@ repos: exclude: ^(deepspeed/comm/|docs/|benchmarks/|scripts/check-torchdist.py|deepspeed/moe/sharded_moe.py|deepspeed/runtime/comm/coalesced_collectives.py|deepspeed/elasticity/elastic_agent.py|deepspeed/launcher/launch.py|tests/unit/comm/test_dist.py) # Specific deepspeed/ files are excluded for now until we wrap ProcessGroup in deepspeed.comm +- repo: local + hooks: + - id: check-license + name: check-license + entry: ./scripts/check-license.py + language: script + files: \.(py|cc|cu|h|cuh|hip)$ + - repo: https://github.com/codespell-project/codespell rev: v2.1.0 hooks: diff --git a/accelerator/__init__.py b/accelerator/__init__.py index aae3e8f9d32e..d29c3b15467b 100644 --- a/accelerator/__init__.py +++ b/accelerator/__init__.py @@ -1,2 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .abstract_accelerator import DeepSpeedAccelerator from .real_accelerator import get_accelerator, set_accelerator diff --git a/accelerator/abstract_accelerator.py b/accelerator/abstract_accelerator.py index 1ff4d0318c4a..03dfbe9df43d 100644 --- a/accelerator/abstract_accelerator.py +++ b/accelerator/abstract_accelerator.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import abc from abc import ABC diff --git a/accelerator/cuda_accelerator.py b/accelerator/cuda_accelerator.py index 0f9807cfbf7a..cdf4cd447764 100644 --- a/accelerator/cuda_accelerator.py +++ b/accelerator/cuda_accelerator.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import pkgutil import importlib diff --git a/accelerator/real_accelerator.py b/accelerator/real_accelerator.py index bc24d583871f..06cbb0b08e7a 100644 --- a/accelerator/real_accelerator.py +++ b/accelerator/real_accelerator.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + try: from accelerator.abstract_accelerator import DeepSpeedAccelerator as dsa1 except ImportError as e: diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/benchmarks/__init__.py +++ b/benchmarks/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/benchmarks/communication/__init__.py b/benchmarks/communication/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/benchmarks/communication/__init__.py +++ b/benchmarks/communication/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/benchmarks/communication/all_gather.py b/benchmarks/communication/all_gather.py index d99d2aa0e4c9..101f129cca45 100644 --- a/benchmarks/communication/all_gather.py +++ b/benchmarks/communication/all_gather.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from benchmarks.communication.utils import * from benchmarks.communication.constants import * diff --git a/benchmarks/communication/all_reduce.py b/benchmarks/communication/all_reduce.py index e31f51733609..9d46f70c93c2 100644 --- a/benchmarks/communication/all_reduce.py +++ b/benchmarks/communication/all_reduce.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from benchmarks.communication.utils import * from benchmarks.communication.constants import * diff --git a/benchmarks/communication/all_to_all.py b/benchmarks/communication/all_to_all.py index 6ee99a48ee62..f5ce3b37d514 100644 --- a/benchmarks/communication/all_to_all.py +++ b/benchmarks/communication/all_to_all.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from benchmarks.communication.utils import * from benchmarks.communication.constants import * diff --git a/benchmarks/communication/broadcast.py b/benchmarks/communication/broadcast.py index e9d89779ec66..d0480cb15b5a 100644 --- a/benchmarks/communication/broadcast.py +++ b/benchmarks/communication/broadcast.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from benchmarks.communication.utils import * from benchmarks.communication.constants import * diff --git a/benchmarks/communication/constants.py b/benchmarks/communication/constants.py index 4b3356894b5f..71416b16f084 100644 --- a/benchmarks/communication/constants.py +++ b/benchmarks/communication/constants.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + DEFAULT_WARMUPS = 5 DEFAULT_TRIALS = 50 DEFAULT_TYPE = 'float' diff --git a/benchmarks/communication/pt2pt.py b/benchmarks/communication/pt2pt.py index cb99b20b9097..89a2ec045b39 100644 --- a/benchmarks/communication/pt2pt.py +++ b/benchmarks/communication/pt2pt.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from benchmarks.communication.utils import * from benchmarks.communication.constants import * diff --git a/benchmarks/communication/run_all.py b/benchmarks/communication/run_all.py index 9d0f0f5f191f..7ec562cc9ae0 100644 --- a/benchmarks/communication/run_all.py +++ b/benchmarks/communication/run_all.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from benchmarks.communication.utils import * from benchmarks.communication.all_reduce import run_all_reduce from benchmarks.communication.all_gather import run_all_gather diff --git a/benchmarks/communication/utils.py b/benchmarks/communication/utils.py index 305f2f3dad37..b2f7367bb703 100644 --- a/benchmarks/communication/utils.py +++ b/benchmarks/communication/utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import os import math diff --git a/benchmarks/inference/bert-bench.py b/benchmarks/inference/bert-bench.py index 0436d011ce98..5b88ba235e3f 100644 --- a/benchmarks/inference/bert-bench.py +++ b/benchmarks/inference/bert-bench.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import time import deepspeed diff --git a/benchmarks/inference/collect_results.py b/benchmarks/inference/collect_results.py index 3420e663dc1a..0e51033114db 100644 --- a/benchmarks/inference/collect_results.py +++ b/benchmarks/inference/collect_results.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import re import argparse diff --git a/benchmarks/inference/gpt-bench.py b/benchmarks/inference/gpt-bench.py index 09c71101895b..b41ca921064e 100644 --- a/benchmarks/inference/gpt-bench.py +++ b/benchmarks/inference/gpt-bench.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import torch import time diff --git a/csrc/aio/py_test/perf_sweep_utils.py b/csrc/aio/py_test/perf_sweep_utils.py index 2fd1a4c55d0f..78dd93b0a175 100644 --- a/csrc/aio/py_test/perf_sweep_utils.py +++ b/csrc/aio/py_test/perf_sweep_utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + SCRIPT_PREFIX = '_aio_bench' WRITE_OP_DESC = 'write' READ_OP_DESC = 'read' diff --git a/csrc/common/custom_cuda_kernel.cu b/csrc/common/custom_cuda_kernel.cu index f7a2b5d480df..9994a096efa3 100644 --- a/csrc/common/custom_cuda_kernel.cu +++ b/csrc/common/custom_cuda_kernel.cu @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #include "custom_cuda_layers.h" __global__ void param_update_kernel(const float* input, __half* output, int size) diff --git a/csrc/includes/StopWatch.h b/csrc/includes/StopWatch.h index 9bf0401ebc78..a5576047c967 100644 --- a/csrc/includes/StopWatch.h +++ b/csrc/includes/StopWatch.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #pragma once #ifdef _WIN32 #include diff --git a/csrc/includes/Timer.h b/csrc/includes/Timer.h index efc7fff84abb..d325cae1c531 100644 --- a/csrc/includes/Timer.h +++ b/csrc/includes/Timer.h @@ -1,3 +1,6 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ #ifndef __TIMER_H__ #define __TIMER_H__ diff --git a/csrc/includes/context.h b/csrc/includes/context.h index 5f0424116546..a8968ba1f0b4 100644 --- a/csrc/includes/context.h +++ b/csrc/includes/context.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #pragma once #include diff --git a/csrc/includes/cpu_adagrad.h b/csrc/includes/cpu_adagrad.h index 38796b069763..0dda4f759ece 100644 --- a/csrc/includes/cpu_adagrad.h +++ b/csrc/includes/cpu_adagrad.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #pragma once #define NOMINMAX // Windows idiosyncrasy diff --git a/csrc/includes/cpu_adam.h b/csrc/includes/cpu_adam.h index f7a7c0c7df23..e9e139aa8492 100644 --- a/csrc/includes/cpu_adam.h +++ b/csrc/includes/cpu_adam.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #pragma once #define NOMINMAX // Windows idiosyncrasy diff --git a/csrc/includes/cublas_wrappers.h b/csrc/includes/cublas_wrappers.h index 9bb6cc30f6ae..68e7f48e5083 100644 --- a/csrc/includes/cublas_wrappers.h +++ b/csrc/includes/cublas_wrappers.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #pragma once #include diff --git a/csrc/includes/dropout.h b/csrc/includes/dropout.h index a72572d0876e..2fdf22495295 100644 --- a/csrc/includes/dropout.h +++ b/csrc/includes/dropout.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #pragma once #include diff --git a/csrc/includes/ds_transformer_cuda.h b/csrc/includes/ds_transformer_cuda.h index 09afeb9d4b19..d289a24c6b4c 100755 --- a/csrc/includes/ds_transformer_cuda.h +++ b/csrc/includes/ds_transformer_cuda.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #pragma once #include diff --git a/csrc/includes/feed_forward.h b/csrc/includes/feed_forward.h index de7a9cf1bf9e..0f2ece4aabd3 100644 --- a/csrc/includes/feed_forward.h +++ b/csrc/includes/feed_forward.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #ifndef __FEEDFORWARD_H__ #define __FEEDFORWARD_H__ diff --git a/csrc/includes/gelu.h b/csrc/includes/gelu.h index 560f4140ed61..07d431484482 100644 --- a/csrc/includes/gelu.h +++ b/csrc/includes/gelu.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #pragma once #include diff --git a/csrc/includes/gemm_test.h b/csrc/includes/gemm_test.h index 22c35123f2c7..7ddb8b122798 100644 --- a/csrc/includes/gemm_test.h +++ b/csrc/includes/gemm_test.h @@ -1,3 +1,6 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ #pragma once diff --git a/csrc/includes/general_kernels.h b/csrc/includes/general_kernels.h index e949309483ce..875df84195ea 100644 --- a/csrc/includes/general_kernels.h +++ b/csrc/includes/general_kernels.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #include #include #include diff --git a/csrc/includes/normalize_layer.h b/csrc/includes/normalize_layer.h index b4d135ec495e..092129362f36 100644 --- a/csrc/includes/normalize_layer.h +++ b/csrc/includes/normalize_layer.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #pragma once #include diff --git a/csrc/includes/quantization.h b/csrc/includes/quantization.h index 5475a6d2b2b2..89867748280c 100644 --- a/csrc/includes/quantization.h +++ b/csrc/includes/quantization.h @@ -1,3 +1,6 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ #pragma once diff --git a/csrc/includes/quantizer.h b/csrc/includes/quantizer.h index 79eeb14e2a06..fe363c809607 100644 --- a/csrc/includes/quantizer.h +++ b/csrc/includes/quantizer.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #pragma once #include diff --git a/csrc/includes/simd.h b/csrc/includes/simd.h index 76c7bb25116b..61f6ffdf3cbc 100644 --- a/csrc/includes/simd.h +++ b/csrc/includes/simd.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #pragma once #if (__x86_64__ || __i386__) diff --git a/csrc/includes/softmax.h b/csrc/includes/softmax.h index 8d541a6fe0e6..676a35d5292a 100755 --- a/csrc/includes/softmax.h +++ b/csrc/includes/softmax.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #pragma once #include diff --git a/csrc/includes/strided_batch_gemm.h b/csrc/includes/strided_batch_gemm.h index 037319ba0dd9..6a1d65cfc7ba 100644 --- a/csrc/includes/strided_batch_gemm.h +++ b/csrc/includes/strided_batch_gemm.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #pragma once #include diff --git a/csrc/includes/type_shim.h b/csrc/includes/type_shim.h index 4f4e7a539ac1..9d28019be877 100644 --- a/csrc/includes/type_shim.h +++ b/csrc/includes/type_shim.h @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + /* Taken from NVIDIA/apex commit 855808f3fc268e9715d613f3c2e56469d8c986d8 */ #include diff --git a/csrc/quantization/fake_quantizer.cu b/csrc/quantization/fake_quantizer.cu index 8a085c7d84ea..5d23c6ea4a6b 100644 --- a/csrc/quantization/fake_quantizer.cu +++ b/csrc/quantization/fake_quantizer.cu @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #include #include "custom_cuda_layers.h" #include "memory_access_utils.h" diff --git a/csrc/transformer/cublas_wrappers.cu b/csrc/transformer/cublas_wrappers.cu index 75ecd3fb4ef9..97ecdd6f13ce 100644 --- a/csrc/transformer/cublas_wrappers.cu +++ b/csrc/transformer/cublas_wrappers.cu @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #include "cublas_wrappers.h" #ifdef __HIP_PLATFORM_HCC__ diff --git a/csrc/transformer/dropout_kernels.cu b/csrc/transformer/dropout_kernels.cu index d1ba135f4900..b2f2abc3f098 100755 --- a/csrc/transformer/dropout_kernels.cu +++ b/csrc/transformer/dropout_kernels.cu @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #include "custom_cuda_layers.h" const int unroll_factor = 4; diff --git a/csrc/transformer/gelu_kernels.cu b/csrc/transformer/gelu_kernels.cu index d683cf0af83d..1f113a988d74 100644 --- a/csrc/transformer/gelu_kernels.cu +++ b/csrc/transformer/gelu_kernels.cu @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #include "custom_cuda_layers.h" inline __device__ float gelu(const float x) diff --git a/csrc/transformer/general_kernels.cu b/csrc/transformer/general_kernels.cu index 1eaa94e1e71a..ea5491003dc9 100644 --- a/csrc/transformer/general_kernels.cu +++ b/csrc/transformer/general_kernels.cu @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #include "general_kernels.h" namespace cg = cooperative_groups; diff --git a/csrc/transformer/normalize_kernels.cu b/csrc/transformer/normalize_kernels.cu index 86753b4ffac2..84f27b2c43af 100644 --- a/csrc/transformer/normalize_kernels.cu +++ b/csrc/transformer/normalize_kernels.cu @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #include "custom_cuda_layers.h" namespace cg = cooperative_groups; diff --git a/csrc/transformer/softmax_kernels.cu b/csrc/transformer/softmax_kernels.cu index 2d9dac1bb1db..bad814b8629d 100644 --- a/csrc/transformer/softmax_kernels.cu +++ b/csrc/transformer/softmax_kernels.cu @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #include #include "custom_cuda_layers.h" #include "general_kernels.h" diff --git a/csrc/transformer/transform_kernels.cu b/csrc/transformer/transform_kernels.cu index 15a2219333e4..37a6e79bb545 100755 --- a/csrc/transformer/transform_kernels.cu +++ b/csrc/transformer/transform_kernels.cu @@ -1,3 +1,7 @@ +/* +Copyright The Microsoft DeepSpeed Team +*/ + #include "custom_cuda_layers.h" #define rows_trans 16 diff --git a/deepspeed/autotuning/__init__.py b/deepspeed/autotuning/__init__.py index deea832eb6da..11ad4700017a 100644 --- a/deepspeed/autotuning/__init__.py +++ b/deepspeed/autotuning/__init__.py @@ -1 +1,3 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .autotuner import Autotuner diff --git a/deepspeed/autotuning/autotuner.py b/deepspeed/autotuning/autotuner.py index 5d656fe816a2..569b62666fc0 100755 --- a/deepspeed/autotuning/autotuner.py +++ b/deepspeed/autotuning/autotuner.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import shutil import subprocess import time diff --git a/deepspeed/autotuning/config.py b/deepspeed/autotuning/config.py index a1a2f2b8b642..6f6b6903efc5 100644 --- a/deepspeed/autotuning/config.py +++ b/deepspeed/autotuning/config.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/autotuning/constants.py b/deepspeed/autotuning/constants.py index 6d1c530a10c8..d0306bb09bb9 100644 --- a/deepspeed/autotuning/constants.py +++ b/deepspeed/autotuning/constants.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/autotuning/scheduler.py b/deepspeed/autotuning/scheduler.py index 3e8d40af23ca..2a4c0c70d955 100755 --- a/deepspeed/autotuning/scheduler.py +++ b/deepspeed/autotuning/scheduler.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import copy from numpy import BUFSIZE diff --git a/deepspeed/autotuning/tuner/__init__.py b/deepspeed/autotuning/tuner/__init__.py index 7ce9fe4f9712..9f2e5675d9fd 100755 --- a/deepspeed/autotuning/tuner/__init__.py +++ b/deepspeed/autotuning/tuner/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .index_based_tuner import RandomTuner, GridSearchTuner # from .ga_tuner import GATuner from .model_based_tuner import ModelBasedTuner diff --git a/deepspeed/autotuning/tuner/base_tuner.py b/deepspeed/autotuning/tuner/base_tuner.py index fe00e27457e7..89eb2d384c94 100755 --- a/deepspeed/autotuning/tuner/base_tuner.py +++ b/deepspeed/autotuning/tuner/base_tuner.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import sys from deepspeed.autotuning.constants import * diff --git a/deepspeed/autotuning/tuner/cost_model.py b/deepspeed/autotuning/tuner/cost_model.py index 0cdcef6483b4..858ab6d3d1ea 100755 --- a/deepspeed/autotuning/tuner/cost_model.py +++ b/deepspeed/autotuning/tuner/cost_model.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .utils import * try: diff --git a/deepspeed/autotuning/tuner/index_based_tuner.py b/deepspeed/autotuning/tuner/index_based_tuner.py index 1685d799d44b..f19694871c53 100755 --- a/deepspeed/autotuning/tuner/index_based_tuner.py +++ b/deepspeed/autotuning/tuner/index_based_tuner.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import random from .base_tuner import BaseTuner diff --git a/deepspeed/autotuning/tuner/model_based_tuner.py b/deepspeed/autotuning/tuner/model_based_tuner.py index c3e0b1ac3fd1..ec475005abe0 100755 --- a/deepspeed/autotuning/tuner/model_based_tuner.py +++ b/deepspeed/autotuning/tuner/model_based_tuner.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import hjson from ..constants import AUTOTUNING, AUTOTUNING_METRIC_PATH diff --git a/deepspeed/autotuning/tuner/utils.py b/deepspeed/autotuning/tuner/utils.py index f87c7563966a..f6719f0b37d2 100755 --- a/deepspeed/autotuning/tuner/utils.py +++ b/deepspeed/autotuning/tuner/utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import numpy as np import itertools from ..utils import * diff --git a/deepspeed/autotuning/utils.py b/deepspeed/autotuning/utils.py index 41184b4dacd6..40da7ec9613a 100644 --- a/deepspeed/autotuning/utils.py +++ b/deepspeed/autotuning/utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import re import collections.abc import os diff --git a/deepspeed/checkpoint/__init__.py b/deepspeed/checkpoint/__init__.py index 1a7a3f18b27d..2ef0817c724e 100644 --- a/deepspeed/checkpoint/__init__.py +++ b/deepspeed/checkpoint/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .reshape_meg_2d import reshape_meg_2d_parallel from .deepspeed_checkpoint import DeepSpeedCheckpoint diff --git a/deepspeed/checkpoint/constants.py b/deepspeed/checkpoint/constants.py index 7fe8afaec745..787398486ca2 100644 --- a/deepspeed/checkpoint/constants.py +++ b/deepspeed/checkpoint/constants.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' ''' Various symbolic constants used for model checkpointing ''' diff --git a/deepspeed/checkpoint/deepspeed_checkpoint.py b/deepspeed/checkpoint/deepspeed_checkpoint.py index fa8f80af4c11..c1a31b0fb4a5 100644 --- a/deepspeed/checkpoint/deepspeed_checkpoint.py +++ b/deepspeed/checkpoint/deepspeed_checkpoint.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os from typing import Dict import torch diff --git a/deepspeed/checkpoint/reshape_3d_utils.py b/deepspeed/checkpoint/reshape_3d_utils.py index 558f4b65972a..15faffb2a680 100644 --- a/deepspeed/checkpoint/reshape_3d_utils.py +++ b/deepspeed/checkpoint/reshape_3d_utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .reshape_utils import (get_files, get_files_with_prefix, partition_data, diff --git a/deepspeed/checkpoint/reshape_meg_2d.py b/deepspeed/checkpoint/reshape_meg_2d.py index 0d7cd233c78a..4800b55583b8 100644 --- a/deepspeed/checkpoint/reshape_meg_2d.py +++ b/deepspeed/checkpoint/reshape_meg_2d.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .reshape_utils import partition_data diff --git a/deepspeed/checkpoint/reshape_utils.py b/deepspeed/checkpoint/reshape_utils.py index 0d1a7286af6b..4d6b272f43e3 100644 --- a/deepspeed/checkpoint/reshape_utils.py +++ b/deepspeed/checkpoint/reshape_utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import torch from collections import OrderedDict diff --git a/deepspeed/checkpoint/utils.py b/deepspeed/checkpoint/utils.py index cb25e524a201..4e96d073067d 100644 --- a/deepspeed/checkpoint/utils.py +++ b/deepspeed/checkpoint/utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os from .constants import (MODEL_FILE_PREFIX, MODEL_FILE_SUFFIX, diff --git a/deepspeed/checkpoint/zero_checkpoint.py b/deepspeed/checkpoint/zero_checkpoint.py index 57de3fd5be26..cb33e8e37671 100644 --- a/deepspeed/checkpoint/zero_checkpoint.py +++ b/deepspeed/checkpoint/zero_checkpoint.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from .constants import (BASE_OPTIMIZER_STATE, diff --git a/deepspeed/comm/__init__.py b/deepspeed/comm/__init__.py index 62fee357ae05..84d0f54c76a1 100644 --- a/deepspeed/comm/__init__.py +++ b/deepspeed/comm/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from .utils import * from deepspeed import utils diff --git a/deepspeed/comm/backend.py b/deepspeed/comm/backend.py index 72b860ce870d..cd436865c5ad 100644 --- a/deepspeed/comm/backend.py +++ b/deepspeed/comm/backend.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' ''' DeepSpeed Communication Backend. In the future, directly use NCCL/MPI/Gloo/etc without requiring torch.distributed. Simply wrap torch.distributed for now. diff --git a/deepspeed/comm/config.py b/deepspeed/comm/config.py index d0238331de9e..25839514fcfc 100644 --- a/deepspeed/comm/config.py +++ b/deepspeed/comm/config.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/comm/constants.py b/deepspeed/comm/constants.py index b3a526a5afbc..45fd5ac7e3ed 100644 --- a/deepspeed/comm/constants.py +++ b/deepspeed/comm/constants.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + NCCL_BACKEND = 'nccl' MPI_BACKEND = 'mpi' GLOO_BACKEND = 'gloo' diff --git a/deepspeed/comm/utils.py b/deepspeed/comm/utils.py index 0b903b0b9b3f..120838e72eed 100644 --- a/deepspeed/comm/utils.py +++ b/deepspeed/comm/utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import torch import inspect diff --git a/deepspeed/compression/__init__.py b/deepspeed/compression/__init__.py index 50446e42b5ed..8e4974e69819 100644 --- a/deepspeed/compression/__init__.py +++ b/deepspeed/compression/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .compress import init_compression, redundancy_clean from .scheduler import compression_scheduler from .helper import convert_conv1d_to_linear diff --git a/deepspeed/compression/basic_layer.py b/deepspeed/compression/basic_layer.py index 7dffc8a94790..a15f80da188e 100644 --- a/deepspeed/compression/basic_layer.py +++ b/deepspeed/compression/basic_layer.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import math from torch import nn diff --git a/deepspeed/compression/compress.py b/deepspeed/compression/compress.py index 84ee53aab921..bf3b6c2760fa 100644 --- a/deepspeed/compression/compress.py +++ b/deepspeed/compression/compress.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import re from .helper import compression_preparation, fix_compression, recursive_getattr, is_module_compressible from .config import get_compression_config diff --git a/deepspeed/compression/config.py b/deepspeed/compression/config.py index d53246e2ed87..e6a710dfa3ea 100644 --- a/deepspeed/compression/config.py +++ b/deepspeed/compression/config.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .constants import * import copy from ..runtime.config_utils import get_scalar_param diff --git a/deepspeed/compression/constants.py b/deepspeed/compression/constants.py index 6507ae6fbccf..593b86e5f5c9 100644 --- a/deepspeed/compression/constants.py +++ b/deepspeed/compression/constants.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + ######################################### # Compression Methods # It has several sub-components diff --git a/deepspeed/compression/helper.py b/deepspeed/compression/helper.py index e7f877ee5723..e839a5d03582 100644 --- a/deepspeed/compression/helper.py +++ b/deepspeed/compression/helper.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from .basic_layer import Embedding_Compress, LinearLayer_Compress, Conv2dLayer_Compress, BNLayer_Compress, ColumnParallelLinear_Compress, RowParallelLinear_Compress from .constants import * diff --git a/deepspeed/compression/scheduler.py b/deepspeed/compression/scheduler.py index 02c8fb904af8..67955a825251 100644 --- a/deepspeed/compression/scheduler.py +++ b/deepspeed/compression/scheduler.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .compress import get_module_name from .constants import * from .helper import recursive_getattr diff --git a/deepspeed/compression/utils.py b/deepspeed/compression/utils.py index 3d6724939c12..2492c8ce9d86 100644 --- a/deepspeed/compression/utils.py +++ b/deepspeed/compression/utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from torch import autograd import math diff --git a/deepspeed/elasticity/__init__.py b/deepspeed/elasticity/__init__.py index 6ad9f20cf936..be95e3a3ee19 100644 --- a/deepspeed/elasticity/__init__.py +++ b/deepspeed/elasticity/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .elasticity import compute_elastic_config, elasticity_enabled, ensure_immutable_elastic_config from .utils import is_torch_elastic_compatible from .constants import ENABLED, ENABLED_DEFAULT, ELASTICITY diff --git a/deepspeed/elasticity/elastic_agent.py b/deepspeed/elasticity/elastic_agent.py index 8d0981612725..db0a61b4fa7a 100644 --- a/deepspeed/elasticity/elastic_agent.py +++ b/deepspeed/elasticity/elastic_agent.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from torch.distributed.elastic.agent.server.local_elastic_agent import LocalElasticAgent from typing import Any, Dict, Optional, Tuple from datetime import datetime diff --git a/deepspeed/elasticity/utils.py b/deepspeed/elasticity/utils.py index a1001c6b3806..3fba6949be03 100644 --- a/deepspeed/elasticity/utils.py +++ b/deepspeed/elasticity/utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch diff --git a/deepspeed/env_report.py b/deepspeed/env_report.py index ca115bf106fb..e8be40a0535d 100644 --- a/deepspeed/env_report.py +++ b/deepspeed/env_report.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed import subprocess diff --git a/deepspeed/git_version_info.py b/deepspeed/git_version_info.py index 071d4289b948..dfb8864fe687 100644 --- a/deepspeed/git_version_info.py +++ b/deepspeed/git_version_info.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + try: # This is populated by setup.py from .git_version_info_installed import * # noqa: F401 diff --git a/deepspeed/inference/__init__.py b/deepspeed/inference/__init__.py index 8ee60d6547b4..449e6651e5d4 100644 --- a/deepspeed/inference/__init__.py +++ b/deepspeed/inference/__init__.py @@ -1 +1,3 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .engine import InferenceEngine diff --git a/deepspeed/inference/config.py b/deepspeed/inference/config.py index 546680e99abf..61298db3fbd4 100644 --- a/deepspeed/inference/config.py +++ b/deepspeed/inference/config.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from deepspeed.runtime.config_utils import DeepSpeedConfigModel from deepspeed.runtime.zero.config import DeepSpeedZeroConfig diff --git a/deepspeed/launcher/__init__.py b/deepspeed/launcher/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/launcher/__init__.py +++ b/deepspeed/launcher/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/launcher/multinode_runner.py b/deepspeed/launcher/multinode_runner.py index fab98a5e7d24..a8fc58b41cf0 100644 --- a/deepspeed/launcher/multinode_runner.py +++ b/deepspeed/launcher/multinode_runner.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import sys import shutil diff --git a/deepspeed/model_implementations/__init__.py b/deepspeed/model_implementations/__init__.py index 79e9ef88834a..77d1c5b1e7cb 100644 --- a/deepspeed/model_implementations/__init__.py +++ b/deepspeed/model_implementations/__init__.py @@ -1,2 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .transformers.ds_transformer import DeepSpeedTransformerInference from .transformers.clip_encoder import DSClipEncoder diff --git a/deepspeed/model_implementations/diffusers/__init__.py b/deepspeed/model_implementations/diffusers/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/model_implementations/diffusers/__init__.py +++ b/deepspeed/model_implementations/diffusers/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/model_implementations/transformers/__init__.py b/deepspeed/model_implementations/transformers/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/model_implementations/transformers/__init__.py +++ b/deepspeed/model_implementations/transformers/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/model_implementations/transformers/ds_base.py b/deepspeed/model_implementations/transformers/ds_base.py index 7c30dfce12eb..9a848eaae1ad 100644 --- a/deepspeed/model_implementations/transformers/ds_base.py +++ b/deepspeed/model_implementations/transformers/ds_base.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch.nn as nn diff --git a/deepspeed/module_inject/__init__.py b/deepspeed/module_inject/__init__.py index e968e8ffc242..aab302864051 100755 --- a/deepspeed/module_inject/__init__.py +++ b/deepspeed/module_inject/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .replace_module import replace_transformer_layer, revert_transformer_layer, ReplaceWithTensorSlicing, GroupQuantizer, generic_injection from .module_quantize import quantize_transformer_layer from .replace_policy import HFBertLayerPolicy diff --git a/deepspeed/module_inject/auto_tp.py b/deepspeed/module_inject/auto_tp.py index 4385331b112e..56068ce638e9 100644 --- a/deepspeed/module_inject/auto_tp.py +++ b/deepspeed/module_inject/auto_tp.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + # Automatic Tensor Parallelism import re diff --git a/deepspeed/module_inject/containers/__init__.py b/deepspeed/module_inject/containers/__init__.py index 72e7577a8e73..6abef517a500 100644 --- a/deepspeed/module_inject/containers/__init__.py +++ b/deepspeed/module_inject/containers/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .bert import DS_BERTContainer, HFBertLayerPolicy from .bloom import DS_BloomContainer, BLOOMLayerPolicy, supported_models from .distil_bert import DS_DistilBERTContainer, HFDistilBertLayerPolicy diff --git a/deepspeed/module_inject/containers/base.py b/deepspeed/module_inject/containers/base.py index e5139db3b855..ca5f0e23ffe3 100644 --- a/deepspeed/module_inject/containers/base.py +++ b/deepspeed/module_inject/containers/base.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + # Create a container object to save model-specific tensors using the policy file above. from abc import ABC import torch diff --git a/deepspeed/module_inject/containers/base_moe.py b/deepspeed/module_inject/containers/base_moe.py index f70cb9b84538..f120a5ef6006 100644 --- a/deepspeed/module_inject/containers/base_moe.py +++ b/deepspeed/module_inject/containers/base_moe.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + # Create a container object to save model-specific tensors using the policy file above. from .base import * import torch diff --git a/deepspeed/module_inject/containers/bert.py b/deepspeed/module_inject/containers/bert.py index 4eb470f72078..95d8b485a9d6 100644 --- a/deepspeed/module_inject/containers/bert.py +++ b/deepspeed/module_inject/containers/bert.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .base import * from deepspeed.model_implementations.transformers.ds_bert import DeepSpeedBERTInference import torch diff --git a/deepspeed/module_inject/containers/bloom.py b/deepspeed/module_inject/containers/bloom.py index 37c2c26a58bc..f6e8edbe01ff 100644 --- a/deepspeed/module_inject/containers/bloom.py +++ b/deepspeed/module_inject/containers/bloom.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .base import * from .features.meta_tensor import MetaTensorContainer from deepspeed.model_implementations.transformers.ds_bloom import DeepSpeedBloomInference diff --git a/deepspeed/module_inject/containers/clip.py b/deepspeed/module_inject/containers/clip.py index 522e20cb8fc4..8e6975091c03 100644 --- a/deepspeed/module_inject/containers/clip.py +++ b/deepspeed/module_inject/containers/clip.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .base import * from deepspeed.model_implementations.transformers.ds_gpt import DeepSpeedGPTInference import torch diff --git a/deepspeed/module_inject/containers/distil_bert.py b/deepspeed/module_inject/containers/distil_bert.py index bcaa8b30744a..71f46dc8ff12 100644 --- a/deepspeed/module_inject/containers/distil_bert.py +++ b/deepspeed/module_inject/containers/distil_bert.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .base import * from deepspeed.model_implementations.transformers.ds_bert import DeepSpeedBERTInference import torch diff --git a/deepspeed/module_inject/containers/features/__init__.py b/deepspeed/module_inject/containers/features/__init__.py index 01779a60aff7..0bd29647d48a 100644 --- a/deepspeed/module_inject/containers/features/__init__.py +++ b/deepspeed/module_inject/containers/features/__init__.py @@ -1,2 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .megatron import MegatronContainer from .meta_tensor import MetaTensorContainer diff --git a/deepspeed/module_inject/containers/features/megatron.py b/deepspeed/module_inject/containers/features/megatron.py index edf371b0311e..45a013be913e 100644 --- a/deepspeed/module_inject/containers/features/megatron.py +++ b/deepspeed/module_inject/containers/features/megatron.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from abc import ABC diff --git a/deepspeed/module_inject/containers/features/meta_tensor.py b/deepspeed/module_inject/containers/features/meta_tensor.py index 5f03d0eabd2e..5b63c5cf5fbd 100644 --- a/deepspeed/module_inject/containers/features/meta_tensor.py +++ b/deepspeed/module_inject/containers/features/meta_tensor.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from abc import ABC, abstractmethod diff --git a/deepspeed/module_inject/containers/gpt2.py b/deepspeed/module_inject/containers/gpt2.py index c3f1bb3f64dd..dc194d71d459 100644 --- a/deepspeed/module_inject/containers/gpt2.py +++ b/deepspeed/module_inject/containers/gpt2.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .base import * from deepspeed.model_implementations.transformers.ds_gpt import DeepSpeedGPTInference from ..policy import TransformerPolicy diff --git a/deepspeed/module_inject/containers/gptj.py b/deepspeed/module_inject/containers/gptj.py index 4c4ac2af3c3c..c0b8c29d4d32 100644 --- a/deepspeed/module_inject/containers/gptj.py +++ b/deepspeed/module_inject/containers/gptj.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .base import * from .features.meta_tensor import MetaTensorContainer from deepspeed.model_implementations.transformers.ds_gpt import DeepSpeedGPTInference diff --git a/deepspeed/module_inject/containers/gptneo.py b/deepspeed/module_inject/containers/gptneo.py index a0e9c98f15c0..6b7e41837a5c 100644 --- a/deepspeed/module_inject/containers/gptneo.py +++ b/deepspeed/module_inject/containers/gptneo.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .base import * from .features.meta_tensor import MetaTensorContainer from deepspeed.model_implementations.transformers.ds_gpt import DeepSpeedGPTInference diff --git a/deepspeed/module_inject/containers/gptneox.py b/deepspeed/module_inject/containers/gptneox.py index 19def006bd42..c1863602b176 100644 --- a/deepspeed/module_inject/containers/gptneox.py +++ b/deepspeed/module_inject/containers/gptneox.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .base import * from .features.meta_tensor import MetaTensorContainer from .features.megatron import MegatronContainer diff --git a/deepspeed/module_inject/containers/megatron_gpt.py b/deepspeed/module_inject/containers/megatron_gpt.py index ec17bb80b046..7a8db9108f38 100644 --- a/deepspeed/module_inject/containers/megatron_gpt.py +++ b/deepspeed/module_inject/containers/megatron_gpt.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .base import * from .features.megatron import MegatronContainer from deepspeed.model_implementations.transformers.ds_megatron_gpt import DeepSpeedMegatronGPTInference diff --git a/deepspeed/module_inject/containers/megatron_gpt_moe.py b/deepspeed/module_inject/containers/megatron_gpt_moe.py index 612a2207b5d3..2968161305c4 100644 --- a/deepspeed/module_inject/containers/megatron_gpt_moe.py +++ b/deepspeed/module_inject/containers/megatron_gpt_moe.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .base import * from .base_moe import * from .features.megatron import MegatronContainer diff --git a/deepspeed/module_inject/containers/opt.py b/deepspeed/module_inject/containers/opt.py index f26f97d8e8be..88bd23c48cb2 100644 --- a/deepspeed/module_inject/containers/opt.py +++ b/deepspeed/module_inject/containers/opt.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .base import * from .features.meta_tensor import MetaTensorContainer from deepspeed.model_implementations.transformers.ds_opt import DeepSpeedOPTInference diff --git a/deepspeed/module_inject/inject.py b/deepspeed/module_inject/inject.py index a601ef10e1d2..384bb7279fdf 100755 --- a/deepspeed/module_inject/inject.py +++ b/deepspeed/module_inject/inject.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import copy import torch from deepspeed.ops.transformer import DeepSpeedTransformerLayer, DeepSpeedTransformerConfig diff --git a/deepspeed/module_inject/layers.py b/deepspeed/module_inject/layers.py index 3657657d1bb4..0636e89289d5 100644 --- a/deepspeed/module_inject/layers.py +++ b/deepspeed/module_inject/layers.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from deepspeed import comm as dist from torch import nn diff --git a/deepspeed/module_inject/load_checkpoint.py b/deepspeed/module_inject/load_checkpoint.py index 709b9c8300f2..b251cc9987b6 100644 --- a/deepspeed/module_inject/load_checkpoint.py +++ b/deepspeed/module_inject/load_checkpoint.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from torch import nn from deepspeed.model_implementations.transformers.ds_bloom import DeepSpeedBloomInference from deepspeed.model_implementations.transformers.ds_gpt import DeepSpeedGPTInference diff --git a/deepspeed/module_inject/module_quantize.py b/deepspeed/module_inject/module_quantize.py index 8485c0451214..4123a1214f7b 100755 --- a/deepspeed/module_inject/module_quantize.py +++ b/deepspeed/module_inject/module_quantize.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch diff --git a/deepspeed/module_inject/replace_module.py b/deepspeed/module_inject/replace_module.py index 3fe874e56b57..909b7cb417be 100644 --- a/deepspeed/module_inject/replace_module.py +++ b/deepspeed/module_inject/replace_module.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import torch import tqdm diff --git a/deepspeed/module_inject/utils.py b/deepspeed/module_inject/utils.py index c82c9cca50a7..7ebd797ec31c 100644 --- a/deepspeed/module_inject/utils.py +++ b/deepspeed/module_inject/utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from deepspeed.utils import log_dist diff --git a/deepspeed/moe/__init__.py b/deepspeed/moe/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/moe/__init__.py +++ b/deepspeed/moe/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/moe/utils.py b/deepspeed/moe/utils.py index 16f59c4fe70b..1bf52795195b 100644 --- a/deepspeed/moe/utils.py +++ b/deepspeed/moe/utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from typing import List, Tuple, Dict import torch from .layer import MoE diff --git a/deepspeed/monitor/__init__.py b/deepspeed/monitor/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/monitor/__init__.py +++ b/deepspeed/monitor/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/monitor/config.py b/deepspeed/monitor/config.py index 2bd1cf1aa73a..09ba7ef1af47 100644 --- a/deepspeed/monitor/config.py +++ b/deepspeed/monitor/config.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/monitor/csv_monitor.py b/deepspeed/monitor/csv_monitor.py index 298950de33e3..7de4fbcede58 100644 --- a/deepspeed/monitor/csv_monitor.py +++ b/deepspeed/monitor/csv_monitor.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .monitor import Monitor import os diff --git a/deepspeed/monitor/monitor.py b/deepspeed/monitor/monitor.py index ca0c8b9b8932..918699482552 100644 --- a/deepspeed/monitor/monitor.py +++ b/deepspeed/monitor/monitor.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Support different forms of monitoring such as wandb and tensorboard """ diff --git a/deepspeed/monitor/tensorboard.py b/deepspeed/monitor/tensorboard.py index 8a2a777b4c37..db3a50d75e23 100644 --- a/deepspeed/monitor/tensorboard.py +++ b/deepspeed/monitor/tensorboard.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .utils import check_tb_availability from .monitor import Monitor import os diff --git a/deepspeed/monitor/utils.py b/deepspeed/monitor/utils.py index 0733268f3873..aa37e8d27cff 100644 --- a/deepspeed/monitor/utils.py +++ b/deepspeed/monitor/utils.py @@ -1,3 +1,6 @@ +'''Copyright The Microsoft DeepSpeed Team''' + + def check_tb_availability(): try: # torch.utils.tensorboard will fail if `tensorboard` is not available, diff --git a/deepspeed/monitor/wandb.py b/deepspeed/monitor/wandb.py index 85c9c23c821b..49fc0332070f 100644 --- a/deepspeed/monitor/wandb.py +++ b/deepspeed/monitor/wandb.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .utils import check_wandb_availability from .monitor import Monitor diff --git a/deepspeed/nebula/__init__.py b/deepspeed/nebula/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/nebula/__init__.py +++ b/deepspeed/nebula/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/nebula/config.py b/deepspeed/nebula/config.py index f9928d66147c..93b879fd77b7 100644 --- a/deepspeed/nebula/config.py +++ b/deepspeed/nebula/config.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/nebula/constants.py b/deepspeed/nebula/constants.py index 0e66fa8d1536..6ad876a8d1a3 100644 --- a/deepspeed/nebula/constants.py +++ b/deepspeed/nebula/constants.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/ops/__init__.py b/deepspeed/ops/__init__.py index 698f9599bf9f..efec4e62c3c9 100755 --- a/deepspeed/ops/__init__.py +++ b/deepspeed/ops/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from . import adam from . import adagrad from . import lamb diff --git a/deepspeed/ops/adagrad/__init__.py b/deepspeed/ops/adagrad/__init__.py index 3067c5c90389..a5ab6de0086c 100644 --- a/deepspeed/ops/adagrad/__init__.py +++ b/deepspeed/ops/adagrad/__init__.py @@ -1 +1,3 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .cpu_adagrad import DeepSpeedCPUAdagrad diff --git a/deepspeed/ops/adam/__init__.py b/deepspeed/ops/adam/__init__.py index 6ab6cbd37f35..111d3175f89e 100755 --- a/deepspeed/ops/adam/__init__.py +++ b/deepspeed/ops/adam/__init__.py @@ -1,2 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .cpu_adam import DeepSpeedCPUAdam from .fused_adam import FusedAdam diff --git a/deepspeed/ops/lamb/__init__.py b/deepspeed/ops/lamb/__init__.py index 9d3448dd68ea..942d76687a60 100644 --- a/deepspeed/ops/lamb/__init__.py +++ b/deepspeed/ops/lamb/__init__.py @@ -1 +1,3 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .fused_lamb import FusedLamb diff --git a/deepspeed/ops/quantizer/__init__.py b/deepspeed/ops/quantizer/__init__.py index 0bf4045a4afd..922256532d43 100644 --- a/deepspeed/ops/quantizer/__init__.py +++ b/deepspeed/ops/quantizer/__init__.py @@ -1 +1,3 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .quantizer import ds_quantizer diff --git a/deepspeed/ops/random_ltd/__init__.py b/deepspeed/ops/random_ltd/__init__.py index de48d7cb8916..34b0dd30e294 100644 --- a/deepspeed/ops/random_ltd/__init__.py +++ b/deepspeed/ops/random_ltd/__init__.py @@ -1 +1,3 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .dropping_utils import gpt_sample_tokens, bert_sample_tokens, GatherTokens, ScatterTokens diff --git a/deepspeed/ops/sparse_attention/__init__.py b/deepspeed/ops/sparse_attention/__init__.py index c36b48b85ea8..870d73768d1d 100644 --- a/deepspeed/ops/sparse_attention/__init__.py +++ b/deepspeed/ops/sparse_attention/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .sparsity_config import SparsityConfig, DenseSparsityConfig, FixedSparsityConfig, VariableSparsityConfig, BigBirdSparsityConfig, BSLongformerSparsityConfig, LocalSlidingWindowSparsityConfig from .sparse_self_attention import SparseSelfAttention from .bert_sparse_self_attention import BertSparseSelfAttention diff --git a/deepspeed/ops/sparse_attention/matmul.py b/deepspeed/ops/sparse_attention/matmul.py index 554b0b18ce6f..17b0898fdd0a 100755 --- a/deepspeed/ops/sparse_attention/matmul.py +++ b/deepspeed/ops/sparse_attention/matmul.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + # DeepSpeed note, code taken & adapted from commit 9aa94789f13ada713af36cfd8cca2fc9a7f6b79a # https://github.com/ptillet/torch-blocksparse/blob/master/torch_blocksparse/matmul.py import importlib diff --git a/deepspeed/ops/sparse_attention/softmax.py b/deepspeed/ops/sparse_attention/softmax.py index ce155105988f..09560e103d16 100755 --- a/deepspeed/ops/sparse_attention/softmax.py +++ b/deepspeed/ops/sparse_attention/softmax.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + # DeepSpeed note, code taken & adapted from commit 9aa94789f13ada713af36cfd8cca2fc9a7f6b79a # https://github.com/ptillet/torch-blocksparse/blob/master/torch_blocksparse/matmul.py diff --git a/deepspeed/ops/sparse_attention/trsrc/__init__.py b/deepspeed/ops/sparse_attention/trsrc/__init__.py index 765d34f574a2..b21068e98f8e 100644 --- a/deepspeed/ops/sparse_attention/trsrc/__init__.py +++ b/deepspeed/ops/sparse_attention/trsrc/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import sys import os diff --git a/deepspeed/ops/transformer/__init__.py b/deepspeed/ops/transformer/__init__.py index 8799afcc42b0..77d666c869a9 100755 --- a/deepspeed/ops/transformer/__init__.py +++ b/deepspeed/ops/transformer/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .transformer import DeepSpeedTransformerLayer, DeepSpeedTransformerConfig from .inference.config import DeepSpeedInferenceConfig from ...model_implementations.transformers.ds_transformer import DeepSpeedTransformerInference diff --git a/deepspeed/ops/transformer/inference/__init__.py b/deepspeed/ops/transformer/inference/__init__.py index 074def3af300..6e9ca0051e78 100644 --- a/deepspeed/ops/transformer/inference/__init__.py +++ b/deepspeed/ops/transformer/inference/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .config import DeepSpeedInferenceConfig from ....model_implementations.transformers.ds_transformer import DeepSpeedTransformerInference from .moe_inference import DeepSpeedMoEInferenceConfig, DeepSpeedMoEInference diff --git a/deepspeed/ops/transformer/inference/op_binding/__init__.py b/deepspeed/ops/transformer/inference/op_binding/__init__.py index b53925c10395..890290692e66 100644 --- a/deepspeed/ops/transformer/inference/op_binding/__init__.py +++ b/deepspeed/ops/transformer/inference/op_binding/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .linear import LinearOp from .vector_matmul import VectorMatMulOp from .softmax_context import SoftmaxContextOp diff --git a/deepspeed/ops/transformer/inference/op_binding/base.py b/deepspeed/ops/transformer/inference/op_binding/base.py index 947dc685caf4..ad29e0f773cb 100644 --- a/deepspeed/ops/transformer/inference/op_binding/base.py +++ b/deepspeed/ops/transformer/inference/op_binding/base.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from ..config import DeepSpeedInferenceConfig diff --git a/deepspeed/ops/transformer/inference/op_binding/gelu_gemm.py b/deepspeed/ops/transformer/inference/op_binding/gelu_gemm.py index 80a87696b72f..9ab4ef926870 100644 --- a/deepspeed/ops/transformer/inference/op_binding/gelu_gemm.py +++ b/deepspeed/ops/transformer/inference/op_binding/gelu_gemm.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from ..config import DeepSpeedInferenceConfig from .base import BaseOp diff --git a/deepspeed/ops/transformer/inference/op_binding/linear.py b/deepspeed/ops/transformer/inference/op_binding/linear.py index 99eb0d570386..6d83ffce22bd 100644 --- a/deepspeed/ops/transformer/inference/op_binding/linear.py +++ b/deepspeed/ops/transformer/inference/op_binding/linear.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from ..config import DeepSpeedInferenceConfig from .base import BaseOp diff --git a/deepspeed/ops/transformer/inference/op_binding/mlp_gemm.py b/deepspeed/ops/transformer/inference/op_binding/mlp_gemm.py index 163256887054..4df8ef52c3fb 100644 --- a/deepspeed/ops/transformer/inference/op_binding/mlp_gemm.py +++ b/deepspeed/ops/transformer/inference/op_binding/mlp_gemm.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from ..config import DeepSpeedInferenceConfig from .base import BaseOp diff --git a/deepspeed/ops/transformer/inference/op_binding/qkv_gemm.py b/deepspeed/ops/transformer/inference/op_binding/qkv_gemm.py index af14e99050cd..d50dbfd3e7ed 100644 --- a/deepspeed/ops/transformer/inference/op_binding/qkv_gemm.py +++ b/deepspeed/ops/transformer/inference/op_binding/qkv_gemm.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from ..config import DeepSpeedInferenceConfig from .base import BaseOp diff --git a/deepspeed/ops/transformer/inference/op_binding/residual_add.py b/deepspeed/ops/transformer/inference/op_binding/residual_add.py index c2aca82300db..0fb1741a0095 100644 --- a/deepspeed/ops/transformer/inference/op_binding/residual_add.py +++ b/deepspeed/ops/transformer/inference/op_binding/residual_add.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from ..config import DeepSpeedInferenceConfig from .base import BaseOp diff --git a/deepspeed/ops/transformer/inference/op_binding/softmax.py b/deepspeed/ops/transformer/inference/op_binding/softmax.py index 634547c34145..4d58ba4a4825 100644 --- a/deepspeed/ops/transformer/inference/op_binding/softmax.py +++ b/deepspeed/ops/transformer/inference/op_binding/softmax.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from ..config import DeepSpeedInferenceConfig from .base import BaseOp diff --git a/deepspeed/ops/transformer/inference/op_binding/softmax_context.py b/deepspeed/ops/transformer/inference/op_binding/softmax_context.py index 8828ba720504..818af5f34a23 100644 --- a/deepspeed/ops/transformer/inference/op_binding/softmax_context.py +++ b/deepspeed/ops/transformer/inference/op_binding/softmax_context.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from deepspeed import comm as dist from ..config import DeepSpeedInferenceConfig diff --git a/deepspeed/ops/transformer/inference/op_binding/vector_matmul.py b/deepspeed/ops/transformer/inference/op_binding/vector_matmul.py index fc12f9a50f6f..c22b61cbd7e5 100644 --- a/deepspeed/ops/transformer/inference/op_binding/vector_matmul.py +++ b/deepspeed/ops/transformer/inference/op_binding/vector_matmul.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from ..config import DeepSpeedInferenceConfig from .base import BaseOp diff --git a/deepspeed/ops/transformer/inference/triton_ops.py b/deepspeed/ops/transformer/inference/triton_ops.py index 423a2ff1134d..45c6db66d107 100644 --- a/deepspeed/ops/transformer/inference/triton_ops.py +++ b/deepspeed/ops/transformer/inference/triton_ops.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Inspired by original Triton implementation: https://github.com/openai/triton/blob/b244db06da24a87453a40ad35b085ee37dac3705/python/tutorials/06-fused-attention.py diff --git a/deepspeed/pipe/__init__.py b/deepspeed/pipe/__init__.py index db1308172f08..bbabf4feb7e0 100644 --- a/deepspeed/pipe/__init__.py +++ b/deepspeed/pipe/__init__.py @@ -1 +1,3 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from ..runtime.pipe import PipelineModule, LayerSpec, TiedLayerSpec diff --git a/deepspeed/profiling/__init__.py b/deepspeed/profiling/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/profiling/__init__.py +++ b/deepspeed/profiling/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/profiling/config.py b/deepspeed/profiling/config.py index 0671211132c6..c22cd453fcba 100644 --- a/deepspeed/profiling/config.py +++ b/deepspeed/profiling/config.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/profiling/constants.py b/deepspeed/profiling/constants.py index d999dc61bd9f..162f1d3e7f2f 100644 --- a/deepspeed/profiling/constants.py +++ b/deepspeed/profiling/constants.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/profiling/flops_profiler/__init__.py b/deepspeed/profiling/flops_profiler/__init__.py index 2f033c862baa..7454821bc1cc 100644 --- a/deepspeed/profiling/flops_profiler/__init__.py +++ b/deepspeed/profiling/flops_profiler/__init__.py @@ -1 +1,3 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .profiler import * diff --git a/deepspeed/profiling/flops_profiler/profiler.py b/deepspeed/profiling/flops_profiler/profiler.py index 9a4889005782..b6684f6978c3 100644 --- a/deepspeed/profiling/flops_profiler/profiler.py +++ b/deepspeed/profiling/flops_profiler/profiler.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import time import torch import torch.nn as nn diff --git a/deepspeed/runtime/__init__.py b/deepspeed/runtime/__init__.py index 5b1224d252bb..aa1f39f9e194 100644 --- a/deepspeed/runtime/__init__.py +++ b/deepspeed/runtime/__init__.py @@ -1,3 +1,6 @@ +'''Copyright The Microsoft DeepSpeed Team''' + + class DeepSpeedOptimizer(object): pass diff --git a/deepspeed/runtime/activation_checkpointing/__init__.py b/deepspeed/runtime/activation_checkpointing/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/runtime/activation_checkpointing/__init__.py +++ b/deepspeed/runtime/activation_checkpointing/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/runtime/activation_checkpointing/checkpointing.py b/deepspeed/runtime/activation_checkpointing/checkpointing.py index 60d6e982b60a..74372540106f 100644 --- a/deepspeed/runtime/activation_checkpointing/checkpointing.py +++ b/deepspeed/runtime/activation_checkpointing/checkpointing.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' ''' Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/runtime/activation_checkpointing/config.py b/deepspeed/runtime/activation_checkpointing/config.py index 0ab59ac64eea..0e79579435b8 100755 --- a/deepspeed/runtime/activation_checkpointing/config.py +++ b/deepspeed/runtime/activation_checkpointing/config.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/runtime/checkpoint_engine/__init__.py b/deepspeed/runtime/checkpoint_engine/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/runtime/checkpoint_engine/__init__.py +++ b/deepspeed/runtime/checkpoint_engine/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/runtime/checkpoint_engine/checkpoint_engine.py b/deepspeed/runtime/checkpoint_engine/checkpoint_engine.py index 67c888815242..cb6decb8826e 100644 --- a/deepspeed/runtime/checkpoint_engine/checkpoint_engine.py +++ b/deepspeed/runtime/checkpoint_engine/checkpoint_engine.py @@ -1,3 +1,6 @@ +'''Copyright The Microsoft DeepSpeed Team''' + + class CheckpointEngine(object): # init checkpoint engine for save/load diff --git a/deepspeed/runtime/checkpoint_engine/nebula_checkpoint_engine.py b/deepspeed/runtime/checkpoint_engine/nebula_checkpoint_engine.py index 4cdd0856e7b7..42bb067d087d 100644 --- a/deepspeed/runtime/checkpoint_engine/nebula_checkpoint_engine.py +++ b/deepspeed/runtime/checkpoint_engine/nebula_checkpoint_engine.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import torch import torch_nebula diff --git a/deepspeed/runtime/checkpoint_engine/torch_checkpoint_engine.py b/deepspeed/runtime/checkpoint_engine/torch_checkpoint_engine.py index 9b4942f0a01f..5ee131800def 100644 --- a/deepspeed/runtime/checkpoint_engine/torch_checkpoint_engine.py +++ b/deepspeed/runtime/checkpoint_engine/torch_checkpoint_engine.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from deepspeed.utils import logger, log_dist from deepspeed.runtime.checkpoint_engine.checkpoint_engine import \ diff --git a/deepspeed/runtime/comm/__init__.py b/deepspeed/runtime/comm/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/runtime/comm/__init__.py +++ b/deepspeed/runtime/comm/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/runtime/comm/coalesced_collectives.py b/deepspeed/runtime/comm/coalesced_collectives.py index a18c3a55ad2d..07302b1b7ae2 100644 --- a/deepspeed/runtime/comm/coalesced_collectives.py +++ b/deepspeed/runtime/comm/coalesced_collectives.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """batched collective operations for overhead amortization and better bandwidth utilization""" diff --git a/deepspeed/runtime/compression/__init__.py b/deepspeed/runtime/compression/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/runtime/compression/__init__.py +++ b/deepspeed/runtime/compression/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/runtime/config.py b/deepspeed/runtime/config.py index 3e5d2cfff81c..9da1058f2c8e 100755 --- a/deepspeed/runtime/config.py +++ b/deepspeed/runtime/config.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/runtime/config_utils.py b/deepspeed/runtime/config_utils.py index 279bb0f7b9e8..cd262f14a6dd 100755 --- a/deepspeed/runtime/config_utils.py +++ b/deepspeed/runtime/config_utils.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/runtime/constants.py b/deepspeed/runtime/constants.py index 4b88a63f8cc0..6925745a8e5a 100755 --- a/deepspeed/runtime/constants.py +++ b/deepspeed/runtime/constants.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/runtime/data_pipeline/__init__.py b/deepspeed/runtime/data_pipeline/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/runtime/data_pipeline/__init__.py +++ b/deepspeed/runtime/data_pipeline/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/runtime/data_pipeline/data_routing/__init__.py b/deepspeed/runtime/data_pipeline/data_routing/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/runtime/data_pipeline/data_routing/__init__.py +++ b/deepspeed/runtime/data_pipeline/data_routing/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/runtime/data_pipeline/data_sampling/__init__.py b/deepspeed/runtime/data_pipeline/data_sampling/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/runtime/data_pipeline/data_sampling/__init__.py +++ b/deepspeed/runtime/data_pipeline/data_sampling/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/runtime/eigenvalue.py b/deepspeed/runtime/eigenvalue.py index 490899bda576..618ac00caff4 100755 --- a/deepspeed/runtime/eigenvalue.py +++ b/deepspeed/runtime/eigenvalue.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from deepspeed.utils import log_dist import numpy as np diff --git a/deepspeed/runtime/fp16/__init__.py b/deepspeed/runtime/fp16/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/deepspeed/runtime/fp16/__init__.py +++ b/deepspeed/runtime/fp16/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/deepspeed/runtime/fp16/onebit/__init__.py b/deepspeed/runtime/fp16/onebit/__init__.py index 59c55333d3eb..289769b423ce 100644 --- a/deepspeed/runtime/fp16/onebit/__init__.py +++ b/deepspeed/runtime/fp16/onebit/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .adam import OnebitAdam from .lamb import OnebitLamb from .zoadam import ZeroOneAdam diff --git a/deepspeed/runtime/pipe/__init__.py b/deepspeed/runtime/pipe/__init__.py index 55e4c2ca3277..dcffaa6101b4 100644 --- a/deepspeed/runtime/pipe/__init__.py +++ b/deepspeed/runtime/pipe/__init__.py @@ -1,2 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .module import PipelineModule, LayerSpec, TiedLayerSpec from .topology import ProcessTopology diff --git a/deepspeed/runtime/pipe/module.py b/deepspeed/runtime/pipe/module.py index b9cd88d92508..acf066bb710c 100644 --- a/deepspeed/runtime/pipe/module.py +++ b/deepspeed/runtime/pipe/module.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import glob diff --git a/deepspeed/runtime/pipe/schedule.py b/deepspeed/runtime/pipe/schedule.py index 181d43e8062a..19e73da17fb9 100644 --- a/deepspeed/runtime/pipe/schedule.py +++ b/deepspeed/runtime/pipe/schedule.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from ..utils import call_to_str from abc import ABC, abstractmethod diff --git a/deepspeed/runtime/progressive_layer_drop.py b/deepspeed/runtime/progressive_layer_drop.py index 41c08cfd9e7c..65e1a563096d 100755 --- a/deepspeed/runtime/progressive_layer_drop.py +++ b/deepspeed/runtime/progressive_layer_drop.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import numpy as np from deepspeed.utils import log_dist diff --git a/deepspeed/runtime/quantize.py b/deepspeed/runtime/quantize.py index 7d27fd210f34..81a7bd56991b 100755 --- a/deepspeed/runtime/quantize.py +++ b/deepspeed/runtime/quantize.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import math from deepspeed.utils import logger diff --git a/deepspeed/runtime/weight_quantizer.py b/deepspeed/runtime/weight_quantizer.py index 491493df3045..f88a9310ba32 100644 --- a/deepspeed/runtime/weight_quantizer.py +++ b/deepspeed/runtime/weight_quantizer.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from ..module_inject.replace_policy import HFBertLayerPolicy, replace_policies from deepspeed.accelerator import get_accelerator diff --git a/deepspeed/runtime/zero/config.py b/deepspeed/runtime/zero/config.py index 7a2b641e5f42..d7ab552188b8 100644 --- a/deepspeed/runtime/zero/config.py +++ b/deepspeed/runtime/zero/config.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/runtime/zero/contiguous_memory_allocator.py b/deepspeed/runtime/zero/contiguous_memory_allocator.py index 883e58301e04..46e55787e664 100644 --- a/deepspeed/runtime/zero/contiguous_memory_allocator.py +++ b/deepspeed/runtime/zero/contiguous_memory_allocator.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from deepspeed import comm as dist diff --git a/deepspeed/runtime/zero/linear.py b/deepspeed/runtime/zero/linear.py index 02b9ddbf21b4..6fbcabb1675a 100644 --- a/deepspeed/runtime/zero/linear.py +++ b/deepspeed/runtime/zero/linear.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + #Linear Module to use with ZeRO Stage 3 to allow for parameter memory release #after the module execution during forward #Instead of saving variables using save_for_backward, we save variable ids diff --git a/deepspeed/runtime/zero/offload_config.py b/deepspeed/runtime/zero/offload_config.py index efb0c9b600ae..7ea76c68178d 100644 --- a/deepspeed/runtime/zero/offload_config.py +++ b/deepspeed/runtime/zero/offload_config.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ Copyright (c) Microsoft Corporation Licensed under the MIT license. diff --git a/deepspeed/runtime/zero/test.py b/deepspeed/runtime/zero/test.py index 29213d604ce5..d13c247b3de4 100644 --- a/deepspeed/runtime/zero/test.py +++ b/deepspeed/runtime/zero/test.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from deepspeed.runtime.zero.contiguous_memory_allocator import ContiguousMemoryAllocator diff --git a/deepspeed/runtime/zero/tiling.py b/deepspeed/runtime/zero/tiling.py index 3a78253df496..d66caea85ee7 100644 --- a/deepspeed/runtime/zero/tiling.py +++ b/deepspeed/runtime/zero/tiling.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed from deepspeed.runtime.utils import partition_uniform as partition diff --git a/deepspeed/runtime/zero/utils.py b/deepspeed/runtime/zero/utils.py index c9a8807b396d..eafddad3f35f 100755 --- a/deepspeed/runtime/zero/utils.py +++ b/deepspeed/runtime/zero/utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os from typing import List diff --git a/deepspeed/utils/__init__.py b/deepspeed/utils/__init__.py index 6f71897e37a6..98222574ca6e 100644 --- a/deepspeed/utils/__init__.py +++ b/deepspeed/utils/__init__.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .logging import logger, log_dist from .comms_logging import get_caller_func #from .distributed import init_distributed diff --git a/deepspeed/utils/comms_logging.py b/deepspeed/utils/comms_logging.py index 34f8e2e2e1dc..d6b0410642a9 100644 --- a/deepspeed/utils/comms_logging.py +++ b/deepspeed/utils/comms_logging.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import math from deepspeed.utils import log_dist diff --git a/deepspeed/utils/debug.py b/deepspeed/utils/debug.py index 288f20198f77..f3f3ea56d5cf 100644 --- a/deepspeed/utils/debug.py +++ b/deepspeed/utils/debug.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """ debug utils """ # For lazy import with printflock() diff --git a/deepspeed/utils/logging.py b/deepspeed/utils/logging.py index 624077c1c218..631ee2177c65 100644 --- a/deepspeed/utils/logging.py +++ b/deepspeed/utils/logging.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import logging import sys import os diff --git a/deepspeed/utils/nvtx.py b/deepspeed/utils/nvtx.py index 97bf280f5edf..881000311c36 100644 --- a/deepspeed/utils/nvtx.py +++ b/deepspeed/utils/nvtx.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from deepspeed.accelerator import get_accelerator diff --git a/deepspeed/utils/types.py b/deepspeed/utils/types.py index 1e833c12b007..a6d5ffd24b8f 100644 --- a/deepspeed/utils/types.py +++ b/deepspeed/utils/types.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from enum import IntEnum diff --git a/deepspeed/utils/zero_to_fp32.py b/deepspeed/utils/zero_to_fp32.py index e5249853c891..cfa9ecc3521e 100755 --- a/deepspeed/utils/zero_to_fp32.py +++ b/deepspeed/utils/zero_to_fp32.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + #!/usr/bin/env python # This script extracts fp32 consolidated weights from a zero 2 and 3 DeepSpeed checkpoints. It gets diff --git a/docs/code-docs/source/conf.py b/docs/code-docs/source/conf.py index bcb9f361b6c6..059aa7c0b67c 100644 --- a/docs/code-docs/source/conf.py +++ b/docs/code-docs/source/conf.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full diff --git a/op_builder/quantizer.py b/op_builder/quantizer.py index e31aff170b18..e2c2c9564a29 100644 --- a/op_builder/quantizer.py +++ b/op_builder/quantizer.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .builder import CUDAOpBuilder diff --git a/op_builder/transformer_inference.py b/op_builder/transformer_inference.py index 20797b7938a4..9bb9bbb956b3 100755 --- a/op_builder/transformer_inference.py +++ b/op_builder/transformer_inference.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from .builder import CUDAOpBuilder, installed_cuda_version diff --git a/release/bump_patch_version.py b/release/bump_patch_version.py index 8f1150deab50..40d9badf09ec 100644 --- a/release/bump_patch_version.py +++ b/release/bump_patch_version.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from packaging import version as pkg_version with open('../version.txt') as fd: diff --git a/scripts/check-license.py b/scripts/check-license.py new file mode 100755 index 000000000000..519827d7df67 --- /dev/null +++ b/scripts/check-license.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +from __future__ import annotations +'''Copyright The Microsoft DeepSpeed Team''' +""" +Modified from https://github.com/jlebar/pre-commit-hooks/blob/master/check_do_not_submit.py +""" + +import subprocess +import sys + + +def err(s: str) -> None: + print(s, file=sys.stderr) + + +success = True +failures = [] +for f in sys.argv[1:]: + res = subprocess.run( + ["git", + "grep", + "--quiet", + "-e", + r"Copyright .* DeepSpeed Team", + f], + capture_output=True) + if res.returncode == 1: + success = False + failures.append(f) + elif res.returncode == 2: + err(f"Error invoking grep on {', '.join(sys.argv[1:])}:") + err(res.stderr.decode("utf-8")) + sys.exit(2) + +if not success: + err(f'{failures}: Missing license at top of file') + err(res.stdout.decode("utf-8")) + sys.exit(1) diff --git a/scripts/check-torchdist.py b/scripts/check-torchdist.py index dbbdc2cfa8ad..d655b7b9008e 100755 --- a/scripts/check-torchdist.py +++ b/scripts/check-torchdist.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 +from __future__ import annotations +'''Copyright The Microsoft DeepSpeed Team''' """ Checks each file in sys.argv for the string "torch.distributed". Modified from https://github.com/jlebar/pre-commit-hooks/blob/master/check_do_not_submit.py """ -from __future__ import annotations import subprocess import sys diff --git a/tests/benchmarks/flatten_bench.py b/tests/benchmarks/flatten_bench.py index d87971dc1a78..fdf53df0b131 100755 --- a/tests/benchmarks/flatten_bench.py +++ b/tests/benchmarks/flatten_bench.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + #!/usr/bin/env python # run the benchmark under timeit (-t), cProfile (-c), line_profiler (-l) # diff --git a/tests/benchmarks/unflatten_bench.py b/tests/benchmarks/unflatten_bench.py index 23fb3f87566d..0c60528891ea 100755 --- a/tests/benchmarks/unflatten_bench.py +++ b/tests/benchmarks/unflatten_bench.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + #!/usr/bin/env python # run the benchmark under timeit (-t), cProfile (-c), line_profiler (-l) diff --git a/tests/conftest.py b/tests/conftest.py index 3865b3498bec..86662993a4fb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + # tests directory-specific settings - this file is run automatically by pytest before any tests are run import sys diff --git a/tests/lightning/test_simple.py b/tests/lightning/test_simple.py index e27e32d06a88..c78768bc707a 100644 --- a/tests/lightning/test_simple.py +++ b/tests/lightning/test_simple.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from pytorch_lightning import LightningModule, Trainer from pytorch_lightning.strategies import DeepSpeedStrategy diff --git a/tests/model/BingBertSquad/test_e2e_squad.py b/tests/model/BingBertSquad/test_e2e_squad.py index 0854a8339e1b..7dfd718bc6bd 100644 --- a/tests/model/BingBertSquad/test_e2e_squad.py +++ b/tests/model/BingBertSquad/test_e2e_squad.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import subprocess as sp import os from math import isclose diff --git a/tests/onebit/test_mpi_backend.py b/tests/onebit/test_mpi_backend.py index 65cfb3ed96c5..3b9f67cce9ca 100644 --- a/tests/onebit/test_mpi_backend.py +++ b/tests/onebit/test_mpi_backend.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from mpi4py import MPI import torch import deepspeed.comm as dist diff --git a/tests/onebit/test_mpi_perf.py b/tests/onebit/test_mpi_perf.py index 1652e946985c..3345c20e5008 100644 --- a/tests/onebit/test_mpi_perf.py +++ b/tests/onebit/test_mpi_perf.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from mpi4py import MPI import torch import deepspeed diff --git a/tests/onebit/test_nccl_backend.py b/tests/onebit/test_nccl_backend.py index 395b1053f917..d569c7272f7b 100644 --- a/tests/onebit/test_nccl_backend.py +++ b/tests/onebit/test_nccl_backend.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed.comm as dist import numpy as np diff --git a/tests/onebit/test_nccl_perf.py b/tests/onebit/test_nccl_perf.py index 86923ef90b82..dcdb13acd4c2 100644 --- a/tests/onebit/test_nccl_perf.py +++ b/tests/onebit/test_nccl_perf.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed.comm as dist import numpy as np diff --git a/tests/perf/adagrad_test.py b/tests/perf/adagrad_test.py index 6c872b452b50..2359c28bbe52 100755 --- a/tests/perf/adagrad_test.py +++ b/tests/perf/adagrad_test.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from deepspeed.ops.adagrad import DeepSpeedCPUAdagrad import time diff --git a/tests/perf/adam_test.py b/tests/perf/adam_test.py index 947dc654b427..24240347451d 100755 --- a/tests/perf/adam_test.py +++ b/tests/perf/adam_test.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from deepspeed.ops.adam import DeepSpeedCPUAdam import time diff --git a/tests/perf/adam_test1.py b/tests/perf/adam_test1.py index 88f1a1c5961d..7ee6c71b509d 100755 --- a/tests/perf/adam_test1.py +++ b/tests/perf/adam_test1.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from deepspeed.ops.adam import DeepSpeedCPUAdam import time diff --git a/tests/small_model_debugging/stage3_test.py b/tests/small_model_debugging/stage3_test.py index 5eb1e7d6c14c..ca85c00be486 100644 --- a/tests/small_model_debugging/stage3_test.py +++ b/tests/small_model_debugging/stage3_test.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed diff --git a/tests/small_model_debugging/test.py b/tests/small_model_debugging/test.py index 331a8ef35ca6..799fa9872d74 100644 --- a/tests/small_model_debugging/test.py +++ b/tests/small_model_debugging/test.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from deepspeed.pt.deepspeed_linear import LinearModuleForZeroStage3 from deepspeed.pt.log_utils import logger diff --git a/tests/small_model_debugging/test_model.py b/tests/small_model_debugging/test_model.py index ec3325e24add..792d683ce47b 100755 --- a/tests/small_model_debugging/test_model.py +++ b/tests/small_model_debugging/test_model.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import json import argparse diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index e69de29bb2d1..fcb45ab2b685 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -0,0 +1 @@ +'''Copyright The Microsoft DeepSpeed Team''' diff --git a/tests/unit/alexnet_model.py b/tests/unit/alexnet_model.py index b94bd7052a9f..f7038f6a2072 100644 --- a/tests/unit/alexnet_model.py +++ b/tests/unit/alexnet_model.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import pytest import torch import torch.nn as nn diff --git a/tests/unit/autotuning/test_autotuning.py b/tests/unit/autotuning/test_autotuning.py index 28a886c658ba..90b9c5b3a2c8 100644 --- a/tests/unit/autotuning/test_autotuning.py +++ b/tests/unit/autotuning/test_autotuning.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import pytest from unit.simple_model import create_config_from_dict diff --git a/tests/unit/checkpoint/common.py b/tests/unit/checkpoint/common.py index e0d077656d91..5b89d6811b01 100644 --- a/tests/unit/checkpoint/common.py +++ b/tests/unit/checkpoint/common.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import torch import numbers diff --git a/tests/unit/checkpoint/test_latest_checkpoint.py b/tests/unit/checkpoint/test_latest_checkpoint.py index 1b62d232bb23..955edfdec3ac 100644 --- a/tests/unit/checkpoint/test_latest_checkpoint.py +++ b/tests/unit/checkpoint/test_latest_checkpoint.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import deepspeed from unit.common import DistributedTest diff --git a/tests/unit/checkpoint/test_lr_scheduler.py b/tests/unit/checkpoint/test_lr_scheduler.py index bd950b4183b6..f6a8f5ebdd4a 100644 --- a/tests/unit/checkpoint/test_lr_scheduler.py +++ b/tests/unit/checkpoint/test_lr_scheduler.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import deepspeed from deepspeed.ops.op_builder import CPUAdamBuilder diff --git a/tests/unit/checkpoint/test_moe_checkpoint.py b/tests/unit/checkpoint/test_moe_checkpoint.py index 41c97be747d6..edce2959aa20 100644 --- a/tests/unit/checkpoint/test_moe_checkpoint.py +++ b/tests/unit/checkpoint/test_moe_checkpoint.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from deepspeed.moe.utils import split_params_into_different_moe_groups_for_optimizer from unit.common import DistributedTest diff --git a/tests/unit/checkpoint/test_other_optimizer.py b/tests/unit/checkpoint/test_other_optimizer.py index 74a333399587..d09157a2c80d 100644 --- a/tests/unit/checkpoint/test_other_optimizer.py +++ b/tests/unit/checkpoint/test_other_optimizer.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import deepspeed from deepspeed.ops.op_builder import FusedLambBuilder diff --git a/tests/unit/checkpoint/test_pipeline.py b/tests/unit/checkpoint/test_pipeline.py index a50fd8a425c4..c698798fa965 100644 --- a/tests/unit/checkpoint/test_pipeline.py +++ b/tests/unit/checkpoint/test_pipeline.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from deepspeed.runtime.checkpoint_engine.torch_checkpoint_engine import TorchCheckpointEngine from unit.common import DistributedTest from unit.simple_model import * diff --git a/tests/unit/checkpoint/test_reshape_checkpoint.py b/tests/unit/checkpoint/test_reshape_checkpoint.py index ed83baf43178..c9ae854521ba 100644 --- a/tests/unit/checkpoint/test_reshape_checkpoint.py +++ b/tests/unit/checkpoint/test_reshape_checkpoint.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from deepspeed.checkpoint import model_3d_desc diff --git a/tests/unit/checkpoint/test_sparse.py b/tests/unit/checkpoint/test_sparse.py index c8db190f60c8..4f07acebc058 100644 --- a/tests/unit/checkpoint/test_sparse.py +++ b/tests/unit/checkpoint/test_sparse.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import deepspeed from unit.common import DistributedTest diff --git a/tests/unit/checkpoint/test_tag_validation.py b/tests/unit/checkpoint/test_tag_validation.py index 89d954859164..d9489622305d 100644 --- a/tests/unit/checkpoint/test_tag_validation.py +++ b/tests/unit/checkpoint/test_tag_validation.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import deepspeed from unit.common import DistributedTest diff --git a/tests/unit/checkpoint/test_zero_optimizer.py b/tests/unit/checkpoint/test_zero_optimizer.py index 73bde2fda940..7de8e9bff908 100644 --- a/tests/unit/checkpoint/test_zero_optimizer.py +++ b/tests/unit/checkpoint/test_zero_optimizer.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import deepspeed from deepspeed.ops.op_builder import CPUAdamBuilder diff --git a/tests/unit/comm/test_dist.py b/tests/unit/comm/test_dist.py index 2a2abeba680e..7ade69a2ef5a 100644 --- a/tests/unit/comm/test_dist.py +++ b/tests/unit/comm/test_dist.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import torch import deepspeed.comm as dist diff --git a/tests/unit/common.py b/tests/unit/common.py index f2b2dc387d38..15c98911f519 100644 --- a/tests/unit/common.py +++ b/tests/unit/common.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import time import inspect diff --git a/tests/unit/compression/test_compression.py b/tests/unit/compression/test_compression.py index 1b9ef4c53ed3..8b3617424225 100644 --- a/tests/unit/compression/test_compression.py +++ b/tests/unit/compression/test_compression.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import pytest import random diff --git a/tests/unit/elasticity/test_elastic.py b/tests/unit/elasticity/test_elastic.py index aa4b3df76cc3..e29b2a22e825 100644 --- a/tests/unit/elasticity/test_elastic.py +++ b/tests/unit/elasticity/test_elastic.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import pytest import deepspeed from unit.common import DistributedTest diff --git a/tests/unit/inference/test_checkpoint_sharding.py b/tests/unit/inference/test_checkpoint_sharding.py index c72661387090..a8ff1312279c 100644 --- a/tests/unit/inference/test_checkpoint_sharding.py +++ b/tests/unit/inference/test_checkpoint_sharding.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import pytest import torch diff --git a/tests/unit/inference/test_inference.py b/tests/unit/inference/test_inference.py index 740d135b14ec..ec791028deb4 100644 --- a/tests/unit/inference/test_inference.py +++ b/tests/unit/inference/test_inference.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import time import torch diff --git a/tests/unit/inference/test_inference_config.py b/tests/unit/inference/test_inference_config.py index ab55c86a7418..e19f73ea35d6 100644 --- a/tests/unit/inference/test_inference_config.py +++ b/tests/unit/inference/test_inference_config.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import pytest import torch import deepspeed diff --git a/tests/unit/inference/test_model_profiling.py b/tests/unit/inference/test_model_profiling.py index 0657ca60b15d..0259ce09a1d4 100644 --- a/tests/unit/inference/test_model_profiling.py +++ b/tests/unit/inference/test_model_profiling.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import time import pytest diff --git a/tests/unit/launcher/test_ds_arguments.py b/tests/unit/launcher/test_ds_arguments.py index a88a45d1bf46..9d7af74f2c57 100644 --- a/tests/unit/launcher/test_ds_arguments.py +++ b/tests/unit/launcher/test_ds_arguments.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import argparse import pytest import deepspeed diff --git a/tests/unit/launcher/test_multinode_runner.py b/tests/unit/launcher/test_multinode_runner.py index 649117fe1a48..af8c753c8ae7 100644 --- a/tests/unit/launcher/test_multinode_runner.py +++ b/tests/unit/launcher/test_multinode_runner.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from copy import deepcopy from deepspeed.launcher import multinode_runner as mnrunner from deepspeed.launcher.runner import encode_world_info, parse_args diff --git a/tests/unit/launcher/test_run.py b/tests/unit/launcher/test_run.py index 4677e1e3025a..1d7f4efc6815 100644 --- a/tests/unit/launcher/test_run.py +++ b/tests/unit/launcher/test_run.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import pytest from deepspeed.launcher import runner as dsrun diff --git a/tests/unit/megatron_model.py b/tests/unit/megatron_model.py index 6fc55393295f..00212a853570 100644 --- a/tests/unit/megatron_model.py +++ b/tests/unit/megatron_model.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import os import sys diff --git a/tests/unit/model_parallelism/test_configurable_parallel_mp.py b/tests/unit/model_parallelism/test_configurable_parallel_mp.py index dda4c22bcdae..93b66cee225a 100644 --- a/tests/unit/model_parallelism/test_configurable_parallel_mp.py +++ b/tests/unit/model_parallelism/test_configurable_parallel_mp.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import torch import deepspeed diff --git a/tests/unit/model_parallelism/test_configurable_parallel_pp.py b/tests/unit/model_parallelism/test_configurable_parallel_pp.py index becdafea21a7..33fd33ff2975 100644 --- a/tests/unit/model_parallelism/test_configurable_parallel_pp.py +++ b/tests/unit/model_parallelism/test_configurable_parallel_pp.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import torch import deepspeed diff --git a/tests/unit/modeling.py b/tests/unit/modeling.py index e8a38afc9538..2624e0e2eedb 100644 --- a/tests/unit/modeling.py +++ b/tests/unit/modeling.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, division, print_function, unicode_literals +# Copyright The Microsoft DeepSpeed Team # DeepSpeed note, code taken from commit 3d59216cec89a363649b4fe3d15295ba936ced0f # https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/LanguageModeling/BERT/modeling.py @@ -18,8 +20,6 @@ # limitations under the License. """PyTorch BERT model.""" -from __future__ import absolute_import, division, print_function, unicode_literals - import copy import json import logging diff --git a/tests/unit/modelingpreln.py b/tests/unit/modelingpreln.py index 673a73ac91f4..e9947b2079b3 100644 --- a/tests/unit/modelingpreln.py +++ b/tests/unit/modelingpreln.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, division, print_function, unicode_literals +# Copyright The Microsoft DeepSpeed Team # DeepSpeed note, code taken from commit 3d59216cec89a363649b4fe3d15295ba936ced0f # https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/LanguageModeling/BERT/modeling.py @@ -18,8 +20,6 @@ # limitations under the License. """PyTorch BERT model.""" -from __future__ import absolute_import, division, print_function, unicode_literals - import copy import json import logging diff --git a/tests/unit/moe/test_moe.py b/tests/unit/moe/test_moe.py index 96b33657605d..fe5359249dc8 100644 --- a/tests/unit/moe/test_moe.py +++ b/tests/unit/moe/test_moe.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed import pytest diff --git a/tests/unit/moe/test_moe_tp.py b/tests/unit/moe/test_moe_tp.py index 6956da228970..ba63a102a0ed 100644 --- a/tests/unit/moe/test_moe_tp.py +++ b/tests/unit/moe/test_moe_tp.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed import pytest diff --git a/tests/unit/monitor/test_monitor.py b/tests/unit/monitor/test_monitor.py index b489d0b9f42b..7cf10619661c 100644 --- a/tests/unit/monitor/test_monitor.py +++ b/tests/unit/monitor/test_monitor.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from deepspeed.monitor.tensorboard import TensorBoardMonitor from deepspeed.monitor.wandb import WandbMonitor from deepspeed.monitor.csv_monitor import csvMonitor diff --git a/tests/unit/multi_output_model.py b/tests/unit/multi_output_model.py index 240c1a4b7aa6..8993813aa545 100644 --- a/tests/unit/multi_output_model.py +++ b/tests/unit/multi_output_model.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch diff --git a/tests/unit/ops/adagrad/test_cpu_adagrad.py b/tests/unit/ops/adagrad/test_cpu_adagrad.py index 16f4f5490e4a..b5dc7dea1be3 100644 --- a/tests/unit/ops/adagrad/test_cpu_adagrad.py +++ b/tests/unit/ops/adagrad/test_cpu_adagrad.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import numpy as np import pytest diff --git a/tests/unit/ops/adam/test_adamw.py b/tests/unit/ops/adam/test_adamw.py index 6ca985efe0e6..03a7c3ca3266 100644 --- a/tests/unit/ops/adam/test_adamw.py +++ b/tests/unit/ops/adam/test_adamw.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import deepspeed import torch import pytest diff --git a/tests/unit/ops/adam/test_cpu_adam.py b/tests/unit/ops/adam/test_cpu_adam.py index d06250cbd2ed..a9a235d57612 100644 --- a/tests/unit/ops/adam/test_cpu_adam.py +++ b/tests/unit/ops/adam/test_cpu_adam.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import numpy as np import pytest diff --git a/tests/unit/ops/aio/test_aio.py b/tests/unit/ops/aio/test_aio.py index 129044ce5fbc..886354c38935 100644 --- a/tests/unit/ops/aio/test_aio.py +++ b/tests/unit/ops/aio/test_aio.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import pytest import os import filecmp diff --git a/tests/unit/ops/cuda/test_cuda_backward.py b/tests/unit/ops/cuda/test_cuda_backward.py index cf5362e52408..c7a460161792 100644 --- a/tests/unit/ops/cuda/test_cuda_backward.py +++ b/tests/unit/ops/cuda/test_cuda_backward.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import math import numpy as np import torch diff --git a/tests/unit/ops/cuda/test_cuda_forward.py b/tests/unit/ops/cuda/test_cuda_forward.py index 5425d49520ff..3958a220b35b 100644 --- a/tests/unit/ops/cuda/test_cuda_forward.py +++ b/tests/unit/ops/cuda/test_cuda_forward.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import math import numpy as np import torch diff --git a/tests/unit/ops/quantizer/test_fake_quantization.py b/tests/unit/ops/quantizer/test_fake_quantization.py index 10148f0272d2..62b154d34ff8 100644 --- a/tests/unit/ops/quantizer/test_fake_quantization.py +++ b/tests/unit/ops/quantizer/test_fake_quantization.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import pytest from deepspeed.ops import op_builder diff --git a/tests/unit/ops/sparse_attention/test_sparse_attention.py b/tests/unit/ops/sparse_attention/test_sparse_attention.py index 740dfacdd0de..d663698beb3e 100644 --- a/tests/unit/ops/sparse_attention/test_sparse_attention.py +++ b/tests/unit/ops/sparse_attention/test_sparse_attention.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + # DeepSpeed note, some parts of code taken & adapted from commit c368a9fd1b2c9dee4cc94de9a6bb0be3d447be41 # https://github.com/ptillet/torch-blocksparse/blob/master/tests/test_softmax.py # https://github.com/ptillet/torch-blocksparse/blob/master/tests/test_matmul.py diff --git a/tests/unit/ops/transformer/inference/test_bias_add.py b/tests/unit/ops/transformer/inference/test_bias_add.py index dbdb698b5707..f8d759c3f567 100644 --- a/tests/unit/ops/transformer/inference/test_bias_add.py +++ b/tests/unit/ops/transformer/inference/test_bias_add.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import pytest import torch import deepspeed diff --git a/tests/unit/pipe/test_pipe_module.py b/tests/unit/pipe/test_pipe_module.py index 2b8cce57d7a8..5454c7f8969e 100644 --- a/tests/unit/pipe/test_pipe_module.py +++ b/tests/unit/pipe/test_pipe_module.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import copy import torch diff --git a/tests/unit/profiling/flops_profiler/test_flops_profiler.py b/tests/unit/profiling/flops_profiler/test_flops_profiler.py index 63322618150b..1f93533587c0 100644 --- a/tests/unit/profiling/flops_profiler/test_flops_profiler.py +++ b/tests/unit/profiling/flops_profiler/test_flops_profiler.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import pytest import deepspeed diff --git a/tests/unit/runtime/activation_checkpointing/test_activation_checkpointing.py b/tests/unit/runtime/activation_checkpointing/test_activation_checkpointing.py index 375ef30de0f6..f1d0c79d05d5 100644 --- a/tests/unit/runtime/activation_checkpointing/test_activation_checkpointing.py +++ b/tests/unit/runtime/activation_checkpointing/test_activation_checkpointing.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + # TODO: add tests with model parallelism for activation partitioning and other features. import pytest diff --git a/tests/unit/runtime/comm/test_coalesced_collectives.py b/tests/unit/runtime/comm/test_coalesced_collectives.py index 92a081fb309b..a072eb0803c4 100644 --- a/tests/unit/runtime/comm/test_coalesced_collectives.py +++ b/tests/unit/runtime/comm/test_coalesced_collectives.py @@ -1,3 +1,4 @@ +'''Copyright The Microsoft DeepSpeed Team''' """unit tests for coalesced collectives""" import torch diff --git a/tests/unit/runtime/half_precision/onebit/test_onebit.py b/tests/unit/runtime/half_precision/onebit/test_onebit.py index 451d6abb6731..6b4a2f5d7989 100644 --- a/tests/unit/runtime/half_precision/onebit/test_onebit.py +++ b/tests/unit/runtime/half_precision/onebit/test_onebit.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import torch.nn as nn import deepspeed.comm as dist diff --git a/tests/unit/runtime/half_precision/test_bf16.py b/tests/unit/runtime/half_precision/test_bf16.py index de15a0868df4..3bc5cb138c9b 100644 --- a/tests/unit/runtime/half_precision/test_bf16.py +++ b/tests/unit/runtime/half_precision/test_bf16.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed import pytest diff --git a/tests/unit/runtime/half_precision/test_dynamic_loss_scale.py b/tests/unit/runtime/half_precision/test_dynamic_loss_scale.py index 7deb3661f428..3052c4ee117a 100644 --- a/tests/unit/runtime/half_precision/test_dynamic_loss_scale.py +++ b/tests/unit/runtime/half_precision/test_dynamic_loss_scale.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed import numpy as np diff --git a/tests/unit/runtime/half_precision/test_fp16.py b/tests/unit/runtime/half_precision/test_fp16.py index 6962a93a3f9f..57f6de1be09a 100644 --- a/tests/unit/runtime/half_precision/test_fp16.py +++ b/tests/unit/runtime/half_precision/test_fp16.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed.comm as dist import deepspeed diff --git a/tests/unit/runtime/pipe/test_pipe.py b/tests/unit/runtime/pipe/test_pipe.py index e5c4ab41172a..2c4d3aef1f63 100644 --- a/tests/unit/runtime/pipe/test_pipe.py +++ b/tests/unit/runtime/pipe/test_pipe.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import copy import torch.nn as nn import pytest diff --git a/tests/unit/runtime/pipe/test_pipe_schedule.py b/tests/unit/runtime/pipe/test_pipe_schedule.py index 68d13c2d9ba6..5ca3dfe1d2a0 100644 --- a/tests/unit/runtime/pipe/test_pipe_schedule.py +++ b/tests/unit/runtime/pipe/test_pipe_schedule.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import pytest import deepspeed.runtime.pipe.schedule as schedule diff --git a/tests/unit/runtime/pipe/test_topology.py b/tests/unit/runtime/pipe/test_topology.py index 35860c5f5167..9c71ce7d72d1 100644 --- a/tests/unit/runtime/pipe/test_topology.py +++ b/tests/unit/runtime/pipe/test_topology.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import pytest import torch diff --git a/tests/unit/runtime/sparse_tensor/test_averaging_sparse_gradients.py b/tests/unit/runtime/sparse_tensor/test_averaging_sparse_gradients.py index b137e2fbb529..638a17bad2ff 100644 --- a/tests/unit/runtime/sparse_tensor/test_averaging_sparse_gradients.py +++ b/tests/unit/runtime/sparse_tensor/test_averaging_sparse_gradients.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed from unit.common import DistributedTest diff --git a/tests/unit/runtime/sparse_tensor/test_csr.py b/tests/unit/runtime/sparse_tensor/test_csr.py index bd5f9933d2a2..1e4f81b986e8 100644 --- a/tests/unit/runtime/sparse_tensor/test_csr.py +++ b/tests/unit/runtime/sparse_tensor/test_csr.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import random from deepspeed.runtime.sparse_tensor import SparseTensor diff --git a/tests/unit/runtime/sparse_tensor/test_sparse_grads.py b/tests/unit/runtime/sparse_tensor/test_sparse_grads.py index d63a47347988..ba9a6b028223 100644 --- a/tests/unit/runtime/sparse_tensor/test_sparse_grads.py +++ b/tests/unit/runtime/sparse_tensor/test_sparse_grads.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed from unit.common import DistributedTest diff --git a/tests/unit/runtime/test_autocast.py b/tests/unit/runtime/test_autocast.py index 99b4df9fbba0..307feb106572 100644 --- a/tests/unit/runtime/test_autocast.py +++ b/tests/unit/runtime/test_autocast.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import pytest import torch from deepspeed.runtime.zero.linear import LinearModuleForZeroStage3 diff --git a/tests/unit/runtime/test_data.py b/tests/unit/runtime/test_data.py index e87f6c5e96f0..3bee3dc2d471 100644 --- a/tests/unit/runtime/test_data.py +++ b/tests/unit/runtime/test_data.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from deepspeed.utils import RepeatingLoader import torch import pytest diff --git a/tests/unit/runtime/test_data_efficiency.py b/tests/unit/runtime/test_data_efficiency.py index 02f766949582..74e1222997d2 100644 --- a/tests/unit/runtime/test_data_efficiency.py +++ b/tests/unit/runtime/test_data_efficiency.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import os import deepspeed diff --git a/tests/unit/runtime/test_ds_config_dict.py b/tests/unit/runtime/test_ds_config_dict.py index 8371dc7c7a2f..311517b3e052 100644 --- a/tests/unit/runtime/test_ds_config_dict.py +++ b/tests/unit/runtime/test_ds_config_dict.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + # A test on its own import os import torch diff --git a/tests/unit/runtime/test_ds_config_model.py b/tests/unit/runtime/test_ds_config_model.py index f0144a0df6b5..24343a999f69 100644 --- a/tests/unit/runtime/test_ds_config_model.py +++ b/tests/unit/runtime/test_ds_config_model.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import pytest import os import json diff --git a/tests/unit/runtime/test_ds_initialize.py b/tests/unit/runtime/test_ds_initialize.py index 5ca3469ad6c6..c7eeef863bda 100644 --- a/tests/unit/runtime/test_ds_initialize.py +++ b/tests/unit/runtime/test_ds_initialize.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import pytest from typing import Callable import torch diff --git a/tests/unit/runtime/test_lr_schedulers.py b/tests/unit/runtime/test_lr_schedulers.py index e17751e9e547..7afcad5426c4 100644 --- a/tests/unit/runtime/test_lr_schedulers.py +++ b/tests/unit/runtime/test_lr_schedulers.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed import pytest diff --git a/tests/unit/runtime/test_multi_output_model.py b/tests/unit/runtime/test_multi_output_model.py index a93ba7fc212f..0a802373a67a 100644 --- a/tests/unit/runtime/test_multi_output_model.py +++ b/tests/unit/runtime/test_multi_output_model.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed from pytest import approx diff --git a/tests/unit/runtime/test_pld.py b/tests/unit/runtime/test_pld.py index c7766a8be7e1..8b8ed2365d77 100644 --- a/tests/unit/runtime/test_pld.py +++ b/tests/unit/runtime/test_pld.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import numpy as np import deepspeed import pytest diff --git a/tests/unit/runtime/test_runtime_utils.py b/tests/unit/runtime/test_runtime_utils.py index 33f40ad30a06..751fae11971b 100644 --- a/tests/unit/runtime/test_runtime_utils.py +++ b/tests/unit/runtime/test_runtime_utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from torch._utils import _flatten_dense_tensors import deepspeed.comm as dist diff --git a/tests/unit/runtime/utils/test_partition.py b/tests/unit/runtime/utils/test_partition.py index e5e6ed14c586..04fa5c94374d 100644 --- a/tests/unit/runtime/utils/test_partition.py +++ b/tests/unit/runtime/utils/test_partition.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import pytest import torch diff --git a/tests/unit/runtime/zero/test_ignore_unused_parameters.py b/tests/unit/runtime/zero/test_ignore_unused_parameters.py index 329a221bb826..efd4949c9460 100644 --- a/tests/unit/runtime/zero/test_ignore_unused_parameters.py +++ b/tests/unit/runtime/zero/test_ignore_unused_parameters.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import pytest from unit.common import DistributedTest from unit.simple_model import UnusedParametersModel, random_dataloader diff --git a/tests/unit/runtime/zero/test_zero.py b/tests/unit/runtime/zero/test_zero.py index 3b3858b7d5a1..958998441a9e 100644 --- a/tests/unit/runtime/zero/test_zero.py +++ b/tests/unit/runtime/zero/test_zero.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import math from typing import Dict, List, Set import pytest diff --git a/tests/unit/runtime/zero/test_zero_config.py b/tests/unit/runtime/zero/test_zero_config.py index 252098fd5a27..84852ec2e6f8 100644 --- a/tests/unit/runtime/zero/test_zero_config.py +++ b/tests/unit/runtime/zero/test_zero_config.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from deepspeed.runtime.zero.config import DeepSpeedZeroConfig, DeepSpeedZeroOffloadParamConfig, DeepSpeedZeroOffloadOptimizerConfig diff --git a/tests/unit/runtime/zero/test_zero_context.py b/tests/unit/runtime/zero/test_zero_context.py index c349e03028a4..a88db44888ef 100644 --- a/tests/unit/runtime/zero/test_zero_context.py +++ b/tests/unit/runtime/zero/test_zero_context.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from types import SimpleNamespace import torch diff --git a/tests/unit/runtime/zero/test_zero_context_ancestry.py b/tests/unit/runtime/zero/test_zero_context_ancestry.py index 43f3c3e89ac8..6035efcff111 100644 --- a/tests/unit/runtime/zero/test_zero_context_ancestry.py +++ b/tests/unit/runtime/zero/test_zero_context_ancestry.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import deepspeed from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus diff --git a/tests/unit/runtime/zero/test_zero_context_return.py b/tests/unit/runtime/zero/test_zero_context_return.py index 4f5a24a11e62..68329cb886c2 100644 --- a/tests/unit/runtime/zero/test_zero_context_return.py +++ b/tests/unit/runtime/zero/test_zero_context_return.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from types import SimpleNamespace import torch import pytest diff --git a/tests/unit/runtime/zero/test_zero_tiled.py b/tests/unit/runtime/zero/test_zero_tiled.py index e76734308ac9..5858b5936872 100644 --- a/tests/unit/runtime/zero/test_zero_tiled.py +++ b/tests/unit/runtime/zero/test_zero_tiled.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import copy import torch diff --git a/tests/unit/runtime/zero/utils.py b/tests/unit/runtime/zero/utils.py index a4fb815d458e..5f0687892d43 100644 --- a/tests/unit/runtime/zero/utils.py +++ b/tests/unit/runtime/zero/utils.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os from unit.common import get_master_port diff --git a/tests/unit/simple_model.py b/tests/unit/simple_model.py index 96f1927cda92..dcc4958b26e9 100644 --- a/tests/unit/simple_model.py +++ b/tests/unit/simple_model.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import json import argparse diff --git a/tests/unit/util.py b/tests/unit/util.py index 8a3a6934945e..8ed7ceb26ec4 100644 --- a/tests/unit/util.py +++ b/tests/unit/util.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch from deepspeed.git_version_info import torch_info diff --git a/tests/unit/utils/test_get_optim_files.py b/tests/unit/utils/test_get_optim_files.py index b0b5b242c200..ccbd9626d6e8 100644 --- a/tests/unit/utils/test_get_optim_files.py +++ b/tests/unit/utils/test_get_optim_files.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import os import pytest from deepspeed.utils.zero_to_fp32 import get_optim_files diff --git a/tests/unit/utils/test_groups.py b/tests/unit/utils/test_groups.py index b2f33cf436d3..06b391e2e301 100644 --- a/tests/unit/utils/test_groups.py +++ b/tests/unit/utils/test_groups.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + from deepspeed.utils.groups import _get_expert_parallel_ranks diff --git a/tests/unit/utils/test_init_on_device.py b/tests/unit/utils/test_init_on_device.py index a7ce6444ba36..9b4f9970b365 100644 --- a/tests/unit/utils/test_init_on_device.py +++ b/tests/unit/utils/test_init_on_device.py @@ -1,3 +1,5 @@ +'''Copyright The Microsoft DeepSpeed Team''' + import torch import pytest from unit.simple_model import SimpleModel