cpt core 4

scripts/cpt_core_model_4.py

@@ -1,6 +1,6 @@
 from unsloth import FastLanguageModel
 import torch
-# from transformers import AutoTokenizer
+from transformers import AutoTokenizer
 
 max_seq_length = 16384
 dtype = torch.bfloat16
@@ -20,12 +20,10 @@ model, tokenizer = FastLanguageModel.from_pretrained(
     dtype=dtype,
     load_in_4bit=load_in_4bit,
 )
-
 print(f'{model=}')
 
 # print('Ignore loaded tokenizer by FastLanguageModel.from_pretrained and using AutoTokenizer.from_pretrained')
 # tokenizer = AutoTokenizer.from_pretrained('..', trust_remote_code=True, use_fast=True)
-
 # print(f'{tokenizer=}')
 
 model = FastLanguageModel.get_peft_model(
@@ -69,33 +67,28 @@ final_dataset = concatenate_datasets(core_datasets)
 print(f'{final_dataset=}')
 '''
 
+from datasets import Dataset
 from litdata import TokensLoader, StreamingDataset
 
-dataset = StreamingDataset(
+
+litgpt_streaming_dataset = StreamingDataset(
     input_dir=dataset_input_dir,
     item_loader=TokensLoader(block_size=dataset_block_size),
 )
 
 
-def unlsoth_generator(dataset):
-    for batch in dataset:
-        print(batch)
-
-        yield {
-            'input_ids': batch['input_ids'].tolist()  # Convert tensor to list
-        }
+def unlsoth_generator():
+    global litgpt_streaming_dataset
 
+    for batch in litgpt_streaming_dataset:
+        # print(batch)
+        yield {'input_ids': batch}
         break
-        # # Assuming TokensLoader returns tensors with 'input_ids'
-        # yield {
-        #     'input_ids': batch['input_ids'].tolist()  # Convert tensor to list
-        # }
 
-for n in unlsoth_generator(dataset):
-    print(n)
-    break
 
-'''
+train_dataset = Dataset.from_generator(unlsoth_generator, streaming=True)
+
+
 from trl import SFTTrainer
 from transformers import TrainingArguments
 from unsloth import is_bfloat16_supported
@@ -105,7 +98,7 @@ from unsloth import UnslothTrainer, UnslothTrainingArguments
 trainer = UnslothTrainer(
     model=model,
     tokenizer=tokenizer,
-    train_dataset=final_dataset,
+    train_dataset=train_dataset,
     dataset_text_field='text',
     max_seq_length=max_seq_length,
     dataset_num_proc=32,
@@ -133,4 +126,3 @@ trainer = UnslothTrainer(
 )
 
 trainer_stats = trainer.train()
-'''