diff --git a/scripts/run_finetune_with_lora_save_aggregated_weights.sh b/scripts/run_finetune_with_lora_save_aggregated_weights.sh
deleted file mode 100755
index c0feb5c67..000000000
--- a/scripts/run_finetune_with_lora_save_aggregated_weights.sh
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/bin/bash
-# Please run this script under ${project_id} in project directory of
-
-# Parses arguments
-model_name_or_path=gpt2
-dataset_path=data/alpaca/train_conversation
-conversation_template=llama2
-output_dir=output_models/finetune
-deepspeed_args="--master_port=11000"
-
-# Safety related arguments
-trust_remote_code=0
-
-while [[ $# -ge 1 ]]; do
-  key="$1"
-  case ${key} in
-    -m|--model_name_or_path)
-      model_name_or_path="$2"
-      shift
-      ;;
-    -d|--dataset_path)
-      dataset_path="$2"
-      shift
-      ;;
-    --conversation_template)
-      conversation_template="$2"
-      shift
-      ;;
-    -o|--output_model_path)
-      output_dir="$2"
-      shift
-      ;;
-    --deepspeed_args)
-      deepspeed_args="$2"
-      shift
-      ;;
-    --trust_remote_code)
-      trust_remote_code="$2"
-      shift
-      ;;
-    *)
-      echo "error: unknown option \"${key}\"" 1>&2
-      exit 1
-  esac
-  shift
-done
-
-# Finetune
-exp_id=finetune_with_lora
-project_dir=$(cd "$(dirname $0)"/..; pwd)
-log_dir=${project_dir}/log/${exp_id}
-mkdir -p ${output_dir} ${log_dir}
-
-deepspeed ${deepspeed_args} \
-  examples/finetune.py \
-    --model_name_or_path ${model_name_or_path} \
-    --trust_remote_code ${trust_remote_code} \
-    --dataset_path ${dataset_path} \
-    --conversation_template ${conversation_template} \
-    --output_dir ${output_dir} --overwrite_output_dir \
-    --num_train_epochs 0.01 \
-    --learning_rate 1e-4 \
-    --block_size 512 \
-    --per_device_train_batch_size 1 \
-    --use_lora 1 \
-    --lora_r 8 \
-    --save_aggregated_lora 1\
-    --deepspeed configs/ds_config_zero2.json \
-    --fp16 \
-    --run_name ${exp_id} \
-    --validation_split_percentage 0 \
-    --logging_steps 20 \
-    --do_train \
-    --ddp_timeout 72000 \
-    --save_steps 5000 \
-    --dataloader_num_workers 1 \
-    | tee ${log_dir}/train.log \
-    2> ${log_dir}/train.err
diff --git a/scripts/run_finetune_with_qlora_save_aggregated_weights.sh b/scripts/run_finetune_with_qlora.sh
similarity index 98%
rename from scripts/run_finetune_with_qlora_save_aggregated_weights.sh
rename to scripts/run_finetune_with_qlora.sh
index 8bbe6e89f..d1cfdf5aa 100644
--- a/scripts/run_finetune_with_qlora_save_aggregated_weights.sh
+++ b/scripts/run_finetune_with_qlora.sh
@@ -63,7 +63,7 @@ deepspeed ${deepspeed_args} \
     --block_size 512 \
     --per_device_train_batch_size 1 \
     --use_qlora 1 \
-    --save_aggregated_lora 1\
+    --save_aggregated_lora 0 \
     --deepspeed configs/ds_config_zero2.json \
     --fp16 \
     --run_name ${exp_id} \