cpt core 4

scripts/cpt_core_model_4.py

@@ -20,7 +20,7 @@ model, tokenizer = FastLanguageModel.from_pretrained(
     dtype=dtype,
     load_in_4bit=load_in_4bit,
 )
-print(f'{model=}')
+# print(f'{model=}')
 
 # print('Ignore loaded tokenizer by FastLanguageModel.from_pretrained and using AutoTokenizer.from_pretrained')
 # tokenizer = AutoTokenizer.from_pretrained('..', trust_remote_code=True, use_fast=True)
@@ -44,8 +44,7 @@ model = FastLanguageModel.get_peft_model(
     use_rslora = True,  # We support rank stabilized LoRA
     loftq_config = None, # And LoftQ
 )
-
-print(f'{model=}')
+# print(f'{model=}')
 
 
 from datasets import Dataset
@@ -85,8 +84,10 @@ trainer = UnslothTrainer(
     packing=False, # Can make training 5x faster for short sequences.
 
     args = UnslothTrainingArguments(
-        per_device_train_batch_size=16,
-        gradient_accumulation_steps=64,
+        # per_device_train_batch_size=16,
+        # gradient_accumulation_steps=64,
+        per_device_train_batch_size=2,
+        gradient_accumulation_steps=8,
 
         warmup_ratio=0,
         num_train_epochs=1,

scripts/requirements-unsloth.in

@@ -0,0 +1,10 @@
+# litgpt compatible litdata
+litdata==0.2.17
+
+# unsloth
+unsloth
+xformers
+trl
+peft
+accelerate
+bitsandbytes