Skip to content

Commit

Permalink
add warning and better compatibility
Browse files Browse the repository at this point in the history
  • Loading branch information
SunMarc committed Aug 31, 2023
1 parent 102ad89 commit 2344645
Showing 1 changed file with 7 additions and 0 deletions.
7 changes: 7 additions & 0 deletions optimum/gptq/quantizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ def __init__(
self.pad_token_id = pad_token_id
self.disable_exllama = disable_exllama
self.max_input_length = max_input_length
self.quant_method = QuantizationMethod.GPTQ

if self.bits not in [2, 3, 4, 8]:
raise ValueError("only support quantize to [2,3,4,8] bits.")
Expand Down Expand Up @@ -445,6 +446,12 @@ def tmp(_, input, output):
"Found modules on cpu/disk. Using Exllama backend requires all the modules to be on GPU. Setting `disable_exllama=True`"
)
self.disable_exllama = True
elif self.desc_act:
logger.warning(
"Using Exllama backend with act_order will reorder the weights offline, thus you will not be able to save the model with the right weights."
"Setting `disable_exllama=True`. You should only use Exllama backend with act_order for inference. "
)
self.disable_exllama = True
# Step 4: Pack the model at the end (Replacing the layers)
self.pack_model(model=model, quantizers=quantizers)

Expand Down

0 comments on commit 2344645

Please sign in to comment.