From 68063945120ce4ff970d2385ce0aa455050bd2ff Mon Sep 17 00:00:00 2001 From: seruva19 Date: Wed, 11 Dec 2024 20:03:06 +0300 Subject: [PATCH] fix error with loading T5 in low-VRAM mode --- src/models/model_30/kandinsky3/condition_encoders.py | 3 ++- src/models/model_31/kandinsky31/condition_encoders.py | 3 ++- src/utils/web.py | 0 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 src/utils/web.py diff --git a/src/models/model_30/kandinsky3/condition_encoders.py b/src/models/model_30/kandinsky3/condition_encoders.py index 959c3e4..6feffec 100644 --- a/src/models/model_30/kandinsky3/condition_encoders.py +++ b/src/models/model_30/kandinsky3/condition_encoders.py @@ -81,7 +81,8 @@ def __init__( if environment.kd30_low_vram: quantization_config = BitsAndBytesConfig( load_in_4bit=True, - bnb_4bit_compute_type=torch.bfloat16, + bnb_4bit_compute_dtype=torch.bfloat16, + llm_int8_enable_fp32_cpu_offload=True, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=False, ) diff --git a/src/models/model_31/kandinsky31/condition_encoders.py b/src/models/model_31/kandinsky31/condition_encoders.py index fa87df2..1d73002 100644 --- a/src/models/model_31/kandinsky31/condition_encoders.py +++ b/src/models/model_31/kandinsky31/condition_encoders.py @@ -30,7 +30,8 @@ def __init__( if environment.kd31_low_vram: quantization_config = BitsAndBytesConfig( load_in_4bit=True, - bnb_4bit_compute_type=torch.bfloat16, + bnb_4bit_compute_dtype=torch.bfloat16, + llm_int8_enable_fp32_cpu_offload=True, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=False, ) diff --git a/src/utils/web.py b/src/utils/web.py new file mode 100644 index 0000000..e69de29