From b4f6304e367a94d17c0c5e92a50a33a60c961132 Mon Sep 17 00:00:00 2001 From: Shane A Date: Mon, 18 Nov 2024 12:04:34 -0800 Subject: [PATCH] Add intermediate size to hf_olmo --- hf_olmo/configuration_olmo.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hf_olmo/configuration_olmo.py b/hf_olmo/configuration_olmo.py index 77e4cd28c..76a1f3136 100644 --- a/hf_olmo/configuration_olmo.py +++ b/hf_olmo/configuration_olmo.py @@ -37,6 +37,13 @@ def num_hidden_layers(self): def hidden_size(self): return self.d_model + @property + def intermediate_size(self): + if hasattr(self, "mlp_hidden_size") and self.mlp_hidden_size is not None: + return self.mlp_hidden_size // 2 + else: + return (self.mlp_ratio * self.d_model) // 2 + @property def effective_n_kv_heads(self) -> int: if self.n_kv_heads is None: