cpt core 4

README.md

@@ -402,10 +402,12 @@ litgpt convert_pretrained_checkpoint ../out/pretrain-core-3/final ../out/pretrai
 ```bash
 litgpt convert_from_litgpt ../out/pretrain-core-3/final ../out/pretrain-core-3/hf
 cp ../config-3.json ../out/pretrain-core-3/hf/config.json
+cp -rv ../tokenizer/* ../out/pretrain-core-3/hf
+python -B convert_pth_to_safetensors.py
 ```
 
 ```bash
-CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0
+python -B cpt_core_model_4.py
 ```
 
 ```

scripts/cpt_core_model_4.py

@@ -8,6 +8,12 @@ load_in_4bit = True
 model_name = '../out/pretrain-core-3/hf'
 output_dir = '../out/cpt-core-4'
 
+dataset_input_dir = '../core-data-4-8193-16385-16385-1000/'
+dataset_block_size = 16385
+
+#
+# model
+#
 model, tokenizer = FastLanguageModel.from_pretrained(
     model_name=model_name,
     max_seq_length=max_seq_length,
@@ -63,6 +69,32 @@ final_dataset = concatenate_datasets(core_datasets)
 print(f'{final_dataset=}')
 '''
 
+from litdata import TokensLoader, StreamingDataset
+
+dataset = StreamingDataset(
+    input_dir=dataset_input_dir,
+    item_loader=TokensLoader(block_size=dataset_block_size),
+)
+
+
+def unlsoth_generator(dataset):
+    for batch in dataset:
+        print(batch)
+
+        yield {
+            'input_ids': batch['input_ids'].tolist()  # Convert tensor to list
+        }
+
+        break
+        # # Assuming TokensLoader returns tensors with 'input_ids'
+        # yield {
+        #     'input_ids': batch['input_ids'].tolist()  # Convert tensor to list
+        # }
+
+for n in unlsoth_generator(dataset):
+    print(n)
+    break
+
 '''
 from trl import SFTTrainer
 from transformers import TrainingArguments