Update default.yaml

Author: lhallee

example_yamls/default.yaml

@@ -20,11 +20,11 @@ vocab_size: 33
 expansion_ratio: 2.6667  # 8/3
 soft_logit_cap: 32.0
 attention_soft_cap: 64.0
-add_att_soft_cap: true
+add_att_soft_cap: false
 p_attention: false
 tie_embeddings: false
 unet: true
-token_dropout: true
+token_dropout: false
 
 # Data Configuration
 input_bin: "data/omgprot50/omgprot50_train_*.bin"
@@ -38,7 +38,7 @@ mask_rate_steps: 2500
 
 # Training Hyperparameters
 batch_size: 524288  # 8*64*1024 tokens
-grad_accum: 1
+grad_accum: 8
 num_steps: 50000
 cooldown_steps: 5000
 max_length: 1024
@@ -52,7 +52,7 @@ lr_embed: 0.06
 lr_head: 0.008
 lr_scalar: 0.04
 grad_clip: 0.0
-auto_grad_clip: true
+auto_grad_clip: false
 auto_grad_clip_percentile: 10
 
 # Muon optimizer (for hidden layers)