From ee7d94a1f3e94cd74460b8f874719fcb137180b7 Mon Sep 17 00:00:00 2001 From: Daniel Nouri Date: Mon, 10 Apr 2023 12:41:16 +0200 Subject: [PATCH] Bugfix in custom_monkey_patch for v1 models Previously generation would fail with: File "/alpaca_lora_4bit/text-generation-webui/matmul_utils_4bit.py", line 79, in _matmul4bit_v1_recons quant_cuda.vecquant4recons_v1(qweight, buffer, scales, zeros) RuntimeError: expected scalar type Half but found Float See #71 --- text-generation-webui/custom_monkey_patch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/text-generation-webui/custom_monkey_patch.py b/text-generation-webui/custom_monkey_patch.py index 2e11db6..c59132e 100644 --- a/text-generation-webui/custom_monkey_patch.py +++ b/text-generation-webui/custom_monkey_patch.py @@ -24,7 +24,7 @@ def load_model_llama(*args, **kwargs): print('Apply auto switch and half') for n, m in model.named_modules(): if isinstance(m, Autograd4bitQuantLinear) or isinstance(m, Linear4bitLt): - if m.groupsize == -1: + if m.is_v1_model: m.zeros = m.zeros.half() m.scales = m.scales.half() m.bias = m.bias.half()