Rename tokenizer.model to tokenizergen.py

tokenizer.model → tokenizergen.py

@@ -1,4 +1,6 @@
 from transformers import AutoTokenizer
 
+# Download and save the tokenizer files
 tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B-Instruct")
-tokenizer.save_pretrained("BuddAi")  # Creates tokenizer.json, special_tokens_map.json, etc.
+tokenizer.save_pretrained("BuddAi")  # Creates a folder with all tokenizer files
+print("Tokenizer files saved to 'BuddAi/' directory!")