ilotoki_thu/alpaca_lora_4bit
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add more scripts and adjust code for transformer branch

Browse Source
This commit is contained in:
John Smith
2023-03-22 04:09:04 +00:00
parent a955a1c2a5
commit dc036373b2
7 changed files with 246 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
GPTQ-for-LLaMa/autograd_4bit.py
View File

@@ -198,18 +198,18 @@ def model_to_float(model):
def load_llama_model_4bit_low_ram(config_path, model_path, half=False):
import transformers
import accelerate
from transformers import LLaMAConfig, LLaMAForCausalLM, LLaMATokenizer
from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer
from modelutils import find_layers
print("Loading Model ...")
t0 = time.time()
with accelerate.init_empty_weights():
config = LLaMAConfig.from_pretrained(config_path)
config = LlamaConfig.from_pretrained(config_path)
torch.set_default_dtype(torch.half)
transformers.modeling_utils._init_weights = False
torch.set_default_dtype(torch.half)
model = LLaMAForCausalLM(config)
model = LlamaForCausalLM(config)
torch.set_default_dtype(torch.float)
model = model.eval()
layers = find_layers(model)
@@ -224,7 +224,7 @@ def load_llama_model_4bit_low_ram(config_path, model_path, half=False):
if half:
model_to_half(model)
tokenizer = LLaMATokenizer.from_pretrained(config_path)
tokenizer = LlamaTokenizer.from_pretrained(config_path)
tokenizer.truncation_side = 'left'
print(f"Loaded the model in {(time.time()-t0):.2f} seconds.")