diff --git a/scripts/run_finetune_with_lisa.sh b/scripts/run_finetune_with_lisa.sh
index 353adb9ff..7d80bc859 100755
--- a/scripts/run_finetune_with_lisa.sh
+++ b/scripts/run_finetune_with_lisa.sh
@@ -9,7 +9,6 @@ dataset_path=data/alpaca/train
 output_dir=output_models/finetune_lisa
 lisa_activated_layers=1
 lisa_interval_steps=20
-deepspeed_args="--master_port=11000"
 
 # Other optional arguments that can improve memory saving
 gradient_checkpointing=True
@@ -41,10 +40,6 @@ while [[ $# -ge 1 ]]; do
       output_dir="$2"
       shift
       ;;
-    --deepspeed_args)
-      deepspeed_args="$2"
-      shift
-      ;;
     --lisa_activated_layers)
       lisa_activated_layers="$2"
       shift
@@ -90,8 +85,7 @@ project_dir=$(cd "$(dirname $0)"/..; pwd)
 log_dir=${project_dir}/log/${exp_id}
 mkdir -p ${output_dir} ${log_dir}
 
-deepspeed ${deepspeed_args} \
-  examples/finetune.py \
+python examples/finetune.py \
     --model_name_or_path ${model_name_or_path} \
     --dataset_path ${dataset_path} \
     --output_dir ${output_dir} --overwrite_output_dir \
@@ -100,9 +94,10 @@ deepspeed ${deepspeed_args} \
     --disable_group_texts 1 \
     --block_size ${block_size} \
     --per_device_train_batch_size ${per_device_train_batch_size} \
-    --deepspeed ${ds_config_file} \
-    --fp16 \
+    --bf16 \
+    --torch_dtype bfloat16 \
     --run_name finetune \
+    --optim paged_adamw_32bit \
     --validation_split_percentage 0 \
     --logging_steps 20 \
     --do_train \
diff --git a/src/lmflow/models/hf_decoder_model.py b/src/lmflow/models/hf_decoder_model.py
index 769728f5e..99d626bfd 100644
--- a/src/lmflow/models/hf_decoder_model.py
+++ b/src/lmflow/models/hf_decoder_model.py
@@ -242,7 +242,6 @@ def __init__(
                     model = AutoModelForCausalLM.from_pretrained(
                         model_args.model_name_or_path,
                         from_tf=bool(".ckpt" in model_args.model_name_or_path),
-                        config=config,
                         quantization_config=quant_config if model_args.use_qlora else None,
                         cache_dir=model_args.cache_dir,
                         revision=model_args.model_revision,
diff --git a/src/lmflow/pipeline/finetuner.py b/src/lmflow/pipeline/finetuner.py
index 840422e6e..9973c9714 100644
--- a/src/lmflow/pipeline/finetuner.py
+++ b/src/lmflow/pipeline/finetuner.py
@@ -317,7 +317,6 @@ def __init__(self, n_layers, interval_steps, model):
                         self.layers_attribute = 'model.transformer.h'  # General access path
                     self.total_layers = len(eval('self.' + self.layers_attribute))  # Dynamically execute to get the number of layers
 
-                    self.switch_active_layers()
                     self.active_layers_indices = []
 
                 def freeze_all_layers(self):
@@ -338,7 +337,7 @@ def switch_active_layers(self):
                     # Randomly select n_layers to activate
                     layers = eval('self.' + self.layers_attribute)  # Re-fetch layer references
                     self.active_layers_indices = np.random.choice(range(self.total_layers), self.n_layers, replace=False)
-                    print(f"Activating layers at indices: {self.active_layers_indices} for the next steps.")
+                    print(f"Activating layers at indices: {self.active_layers_indices} for the next steps.", flush=True)
 
                     # Enable gradients only for the selected layers
                     for idx in self.active_layers_indices: