From 942e5b87f551efeeb0e3d53270b35ff7a3baf6ea Mon Sep 17 00:00:00 2001 From: rpan Date: Mon, 1 Apr 2024 11:19:25 +0800 Subject: [PATCH 1/2] Fix OOM problem during init - Majorly caused by deepspeed --- scripts/run_finetune_with_lisa.sh | 12 +++--------- src/lmflow/models/hf_decoder_model.py | 1 - src/lmflow/pipeline/finetuner.py | 2 +- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/scripts/run_finetune_with_lisa.sh b/scripts/run_finetune_with_lisa.sh index 353adb9ff..54ee723ac 100755 --- a/scripts/run_finetune_with_lisa.sh +++ b/scripts/run_finetune_with_lisa.sh @@ -9,7 +9,6 @@ dataset_path=data/alpaca/train output_dir=output_models/finetune_lisa lisa_activated_layers=1 lisa_interval_steps=20 -deepspeed_args="--master_port=11000" # Other optional arguments that can improve memory saving gradient_checkpointing=True @@ -41,10 +40,6 @@ while [[ $# -ge 1 ]]; do output_dir="$2" shift ;; - --deepspeed_args) - deepspeed_args="$2" - shift - ;; --lisa_activated_layers) lisa_activated_layers="$2" shift @@ -90,8 +85,7 @@ project_dir=$(cd "$(dirname $0)"/..; pwd) log_dir=${project_dir}/log/${exp_id} mkdir -p ${output_dir} ${log_dir} -deepspeed ${deepspeed_args} \ - examples/finetune.py \ +python examples/finetune.py \ --model_name_or_path ${model_name_or_path} \ --dataset_path ${dataset_path} \ --output_dir ${output_dir} --overwrite_output_dir \ @@ -100,8 +94,8 @@ deepspeed ${deepspeed_args} \ --disable_group_texts 1 \ --block_size ${block_size} \ --per_device_train_batch_size ${per_device_train_batch_size} \ - --deepspeed ${ds_config_file} \ - --fp16 \ + --bf16 \ + --torch_dtype bfloat16 \ --run_name finetune \ --validation_split_percentage 0 \ --logging_steps 20 \ diff --git a/src/lmflow/models/hf_decoder_model.py b/src/lmflow/models/hf_decoder_model.py index 769728f5e..99d626bfd 100644 --- a/src/lmflow/models/hf_decoder_model.py +++ b/src/lmflow/models/hf_decoder_model.py @@ -242,7 +242,6 @@ def __init__( model = AutoModelForCausalLM.from_pretrained( model_args.model_name_or_path, from_tf=bool(".ckpt" in model_args.model_name_or_path), - config=config, quantization_config=quant_config if model_args.use_qlora else None, cache_dir=model_args.cache_dir, revision=model_args.model_revision, diff --git a/src/lmflow/pipeline/finetuner.py b/src/lmflow/pipeline/finetuner.py index 840422e6e..88184befc 100644 --- a/src/lmflow/pipeline/finetuner.py +++ b/src/lmflow/pipeline/finetuner.py @@ -317,7 +317,7 @@ def __init__(self, n_layers, interval_steps, model): self.layers_attribute = 'model.transformer.h' # General access path self.total_layers = len(eval('self.' + self.layers_attribute)) # Dynamically execute to get the number of layers - self.switch_active_layers() + # self.switch_active_layers() self.active_layers_indices = [] def freeze_all_layers(self): From d588b5a8a8fee83ba86c6b7f95a3c147bb8b4590 Mon Sep 17 00:00:00 2001 From: rpan Date: Mon, 1 Apr 2024 11:34:32 +0800 Subject: [PATCH 2/2] Add paged_adamw to avoid occasional OOM --- scripts/run_finetune_with_lisa.sh | 1 + src/lmflow/pipeline/finetuner.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run_finetune_with_lisa.sh b/scripts/run_finetune_with_lisa.sh index 54ee723ac..7d80bc859 100755 --- a/scripts/run_finetune_with_lisa.sh +++ b/scripts/run_finetune_with_lisa.sh @@ -97,6 +97,7 @@ python examples/finetune.py \ --bf16 \ --torch_dtype bfloat16 \ --run_name finetune \ + --optim paged_adamw_32bit \ --validation_split_percentage 0 \ --logging_steps 20 \ --do_train \ diff --git a/src/lmflow/pipeline/finetuner.py b/src/lmflow/pipeline/finetuner.py index 88184befc..9973c9714 100644 --- a/src/lmflow/pipeline/finetuner.py +++ b/src/lmflow/pipeline/finetuner.py @@ -317,7 +317,6 @@ def __init__(self, n_layers, interval_steps, model): self.layers_attribute = 'model.transformer.h' # General access path self.total_layers = len(eval('self.' + self.layers_attribute)) # Dynamically execute to get the number of layers - # self.switch_active_layers() self.active_layers_indices = [] def freeze_all_layers(self): @@ -338,7 +337,7 @@ def switch_active_layers(self): # Randomly select n_layers to activate layers = eval('self.' + self.layers_attribute) # Re-fetch layer references self.active_layers_indices = np.random.choice(range(self.total_layers), self.n_layers, replace=False) - print(f"Activating layers at indices: {self.active_layers_indices} for the next steps.") + print(f"Activating layers at indices: {self.active_layers_indices} for the next steps.", flush=True) # Enable gradients only for the selected layers for idx in self.active_layers_indices: