nvidia
/

Nemotron-4-340B-Reward

Model card Files Files and versions

zhilinw commited on Jun 13, 2024

Commit

154bbb8

·

verified ·

1 Parent(s): 08c78f2

Update model_config.yaml

Files changed (1) hide show

model_config.yaml +2 -2

model_config.yaml CHANGED Viewed

@@ -81,7 +81,7 @@ reduce_amax: true
 use_emha: false
 optim:
   name: distributed_fused_adam
-  lr: 3.0e-07
   weight_decay: 0.1
   betas:
   - 0.9
@@ -90,7 +90,7 @@ optim:
     name: CosineAnnealing
     warmup_steps: 10
     constant_steps: 0
-    min_lr: 3.0e-07
   bucket_cap_mb: 200
   overlap_grad_sync: false
   contiguous_grad_buffer: true

 use_emha: false
 optim:
   name: distributed_fused_adam
+  lr: 7.0e-07
   weight_decay: 0.1
   betas:
   - 0.9
     name: CosineAnnealing
     warmup_steps: 10
     constant_steps: 0
+    min_lr: 7.0e-07
   bucket_cap_mb: 200
   overlap_grad_sync: false
   contiguous_grad_buffer: true