I got an issue while making the quantization in a fine tuning for Llama2, i am using ludwig, i saw it has to do with the version of bitsandbytes, but the one i am using is the one compatible with ludwig. Any idea to solve the problem?
File "/home/misael/thesis/thesis3.py", line 85, in
results = model.train(dataset=DataFol)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/api.py", line 654, in train
self._tune_batch_size(trainer, training_set, random_seed=random_seed)
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/api.py", line 882, in _tune_batch_size
tuned_batch_size = trainer.tune_batch_size(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/trainers/trainer_llm.py", line 490, in tune_batch_size
return super().tune_batch_size(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/trainers/trainer.py", line 620, in tune_batch_size
self.resume_weights_and_optimizer(str(tmpdir), checkpoint)
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/trainers/trainer.py", line 1613, in resume_weights_and_optimizer
CheckpointManager.load_latest_checkpoint(checkpoint, model_weights_progress_path, self.device)
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/utils/checkpoint_utils.py", line 338, in load_latest_checkpoint
checkpoint.load(last_ckpt, device)
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/utils/checkpoint_utils.py", line 157, in load
raise e
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/utils/checkpoint_utils.py", line 142, in load
_, unexpected_keys = self.model.load_state_dict(state["model_weights"], strict=False)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/misael/ludwig1/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2139, in load_state_dict
load(self, state_dict)
File "/home/misael/ludwig1/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2127, in load
load(child, child_state_dict, child_prefix)
File "/home/misael/ludwig1/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2127, in load
load(child, child_state_dict, child_prefix)
File "/home/misael/ludwig1/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2127, in load
load(child, child_state_dict, child_prefix)
[Previous line repeated 6 more times]
File "/home/misael/ludwig1/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2121, in load
module._load_from_state_dict(
File "/home/misael/ludwig1/lib/python3.11/site-packages/bitsandbytes/nn/modules.py", line 388, in _load_from_state_dict
raise RuntimeError("Loading a quantized checkpoint into non-quantized Linear8bitLt is "
RuntimeError: Loading a quantized checkpoint into non-quantized Linear8bitLt is not supported. Please call module.cuda() before module.load_state_dict()
I got an issue while making the quantization in a fine tuning for Llama2, i am using ludwig, i saw it has to do with the version of bitsandbytes, but the one i am using is the one compatible with ludwig. Any idea to solve the problem?
File "/home/misael/thesis/thesis3.py", line 85, in
results = model.train(dataset=DataFol)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/api.py", line 654, in train
self._tune_batch_size(trainer, training_set, random_seed=random_seed)
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/api.py", line 882, in _tune_batch_size
tuned_batch_size = trainer.tune_batch_size(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/trainers/trainer_llm.py", line 490, in tune_batch_size
return super().tune_batch_size(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/trainers/trainer.py", line 620, in tune_batch_size
self.resume_weights_and_optimizer(str(tmpdir), checkpoint)
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/trainers/trainer.py", line 1613, in resume_weights_and_optimizer
CheckpointManager.load_latest_checkpoint(checkpoint, model_weights_progress_path, self.device)
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/utils/checkpoint_utils.py", line 338, in load_latest_checkpoint
checkpoint.load(last_ckpt, device)
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/utils/checkpoint_utils.py", line 157, in load
raise e
File "/home/misael/ludwig1/lib/python3.11/site-packages/ludwig/utils/checkpoint_utils.py", line 142, in load
_, unexpected_keys = self.model.load_state_dict(state["model_weights"], strict=False)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/misael/ludwig1/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2139, in load_state_dict
load(self, state_dict)
File "/home/misael/ludwig1/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2127, in load
load(child, child_state_dict, child_prefix)
File "/home/misael/ludwig1/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2127, in load
load(child, child_state_dict, child_prefix)
File "/home/misael/ludwig1/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2127, in load
load(child, child_state_dict, child_prefix)
[Previous line repeated 6 more times]
File "/home/misael/ludwig1/lib/python3.11/site-packages/torch/nn/modules/module.py", line 2121, in load
module._load_from_state_dict(
File "/home/misael/ludwig1/lib/python3.11/site-packages/bitsandbytes/nn/modules.py", line 388, in _load_from_state_dict
raise RuntimeError("Loading a quantized checkpoint into non-quantized Linear8bitLt is "
RuntimeError: Loading a quantized checkpoint into non-quantized Linear8bitLt is not supported. Please call module.cuda() before module.load_state_dict()