cpt core 4

README.md

@@ -407,7 +407,7 @@ python -B convert_pth_to_safetensors.py
 ```
 
 ```bash
-python -B cpt_core_model_4.py
+CUDA_VISIBLE_DEVICES=0 python -B cpt_core_model_4.py
 ```
 
 ```

scripts/cpt_core_model_4.py

@@ -80,12 +80,11 @@ def unlsoth_generator():
     global litgpt_streaming_dataset
 
     for batch in litgpt_streaming_dataset:
-        # print(batch)
         yield {'input_ids': batch}
-        break
 
 
-train_dataset = Dataset.from_generator(unlsoth_generator, streaming=True)
+# train_dataset = Dataset.from_generator(unlsoth_generator, streaming=True)
+train_dataset = Dataset.from_generator(unlsoth_generator, batched=True)
 
 
 from trl import SFTTrainer