diverges on language model training with default beta2 in adopt, brin…

…g it down to 0.99

adam_atan2_pytorch/adopt.py

@@ -23,7 +23,7 @@ def __init__(
         self,
         params,
         lr = 1e-4,
-        betas: tuple[float, float] = (0.9, 0.9999),
+        betas: tuple[float, float] = (0.9, 0.99),
         eps = 1e-6,
         weight_decay = 0.,
         decoupled_wd = True

adam_atan2_pytorch/adopt_atan2.py

@@ -15,6 +15,7 @@ def exists(val):
 class AdoptAtan2(Optimizer):
     """
     the proposed Adam substitute from University of Tokyo
+    combined with the proposed atan2 method for ridding of the eps from Google
 
     Algorithm 2 in https://arxiv.org/abs/2411.02853
     """
@@ -23,7 +24,7 @@ def __init__(
         self,
         params,
         lr = 1e-4,
-        betas: tuple[float, float] = (0.9, 0.9999),
+        betas: tuple[float, float] = (0.9, 0.99),
         weight_decay = 0.,
         decoupled_wd = True,
         a = 1.27,

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "adam-atan2-pytorch"
-version = "0.1.5"
+version = "0.1.6"
 description = "Adam-atan2 for Pytorch"
 authors = [
     { name = "Phil Wang", email = "[email protected]" }