Disable normalizer

Author: luciaquirke

bergson/__main__.py

@@ -132,12 +132,6 @@ class Trackstar:
     trackstar_cfg: TrackstarConfig
 
     def execute(self):
-        if self.index_cfg.normalizer != "none":
-            print(
-                "Warning: using a normalizer. We have been unable to validate"
-                "normalizers in our ablations. Use at your own risk. "
-            )
-
         trackstar(
             self.index_cfg, self.score_cfg, self.preprocess_cfg, self.trackstar_cfg
         )

bergson/config.py

@@ -160,8 +160,10 @@ class IndexConfig:
     processor_path: str = ""
     """Path to a precomputed processor."""
 
-    normalizer: Literal["adafactor", "adam", "none"] = "none"
-    """Type of normalizer to use for the gradients."""
+    normalizer: Literal["none"] = "none"  # "adafactor", "adam",
+    """Type of normalizer to use for the gradients. We are disabling
+    optimizers due to lack of empirical validation - contact Eleuther
+    if you'd like to use them."""
 
     skip_preconditioners: bool = False
     """Whether to skip estimating preconditioner statistics"""

examples/exp_log.md

@@ -50,4 +50,4 @@ with lr schedule from double backwards, 1e-2
 
 without weight decay
 
-512 tok:  LDS Spearman=0.9338
+512 tok:  LDS Spearman=0.9338