pretrain 1

scripts/pretrain_core_model_1.yaml

@@ -85,7 +85,7 @@ train:
   max_norm: 1.0
 
   #   (type: float, default: 4e-05)
-  min_lr: 3e-6
+  min_lr: 5e-5
 
 # Evaluation-related arguments. See ``litgpt.args.EvalArgs`` for details
 eval:
@@ -125,7 +125,7 @@ eval:
 optimizer:
   class_path: sophia_opt.SophiaG
   init_args:
-    lr: 3e-5
+    lr: 1e-4
     betas:
       - 0.9
       - 0.95