pretrain core 0

scripts/pretrain-core-model-0.yaml

@@ -110,29 +110,29 @@ eval:
 
 # Optimizer-related arguments
 
-# optimizer:
-#   # class_path: torch.optim.AdamW
-#   class_path: torchao.prototype.low_bit_optim.AdamW8bit
-#   # class_path: torchao.prototype.low_bit_optim.AdamW4bit
-#   # class_path: bitsandbytes.optim.AdamW8bit
-#   # class_path: bitsandbytes.optim.PagedAdamW8bit
-#   init_args:
-#     # (type: float, default: 0.001)
-#     lr: 1e-4
-#     # (type: float, default: 0.01)
-#     weight_decay: 0.01
-#     # (type: tuple, default: (0.9,0.999))
-#     betas:
-#       - 0.9
-#       - 0.99
-
 optimizer:
-  class_path: dolphinflow.DolphinFlow
+  class_path: torch.optim.AdamW
+  # class_path: torchao.prototype.low_bit_optim.AdamW8bit
+  # class_path: torchao.prototype.low_bit_optim.AdamW4bit
+  # class_path: bitsandbytes.optim.AdamW8bit
+  # class_path: bitsandbytes.optim.PagedAdamW8bit
   init_args:
+    # (type: float, default: 0.001)
     lr: 1e-4
+    # (type: float, default: 0.01)
     weight_decay: 0.01
-    momentum: 0.9
-    nesterov: true
+    # (type: tuple, default: (0.9,0.999))
+    betas:
+      - 0.9
+      - 0.999
+
+# optimizer:
+#   class_path: dolphinflow.DolphinFlow
+#   init_args:
+#     lr: 1e-4
+#     weight_decay: 0.01
+#     momentum: 0.9
+#     nesterov: true
 
 # How many devices/GPUs to use. Uses all GPUs by default. (type: Union[int, str], default: auto)
 devices: auto