ilotoki_thu/alpaca_lora_4bit
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add v2 model support

Browse Source
This commit is contained in:
John Smith
2023-03-28 20:33:55 +08:00
parent 667e43cb5b
commit bff039de95
7 changed files with 213 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
text-generation-webui/custom_monkey_patch.py
View File

@@ -14,7 +14,7 @@ def load_model_llama(*args, **kwargs):
print("Loading {} ...".format(model_path))
t0 = time.time()
model, tokenizer = load_llama_model_4bit_low_ram(config_path, model_path)
model, tokenizer = load_llama_model_4bit_low_ram(config_path, model_path, groupsize=-1)
model = PeftModel.from_pretrained(model, lora_path, device_map={'': 0}, torch_dtype=torch.float32)
print('{} Lora Applied.'.format(lora_path))
@@ -22,7 +22,8 @@ def load_model_llama(*args, **kwargs):
print('Apply auto switch and half')
for n, m in model.named_modules():
if isinstance(m, Autograd4bitQuantLinear) or isinstance(m, Linear4bitLt):
m.zeros = m.zeros.half()
if m.groupsize == -1:
m.zeros = m.zeros.half()
m.scales = m.scales.half()
m.bias = m.bias.half()
autograd_4bit.use_new = True