BuddAi / tokenizergen.py
CaptMetal's picture
Update tokenizergen.py
dbeeca4 verified
raw
history blame contribute delete
383 Bytes
from transformers import AutoModelForCausalLM, AutoTokenizer
model_id = "teknium/OpenHermes-2.5-Mistral-7B"
tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(model_id)
# Save to BuddAi/
model.save_pretrained("BuddAi", safe_serialization=True) # Creates model.safetensors
tokenizer.save_pretrained("BuddAi") # Creates tokenizer files