[bugfix] remove useless codes

mikecovlee · Jul 30, 2024 · 6cd8dc2 · 6cd8dc2
1 parent 4bd6362
commit 6cd8dc2
Showing 1 changed file with 3 additions and 4 deletions.
diff --git a/mlora/models/modeling_phi3.py b/mlora/models/modeling_phi3.py
@@ -305,12 +305,11 @@ def _mixlora_forward(
 
             lora_name = f"moe.{moe_name}.experts.{expert_idx}"
             if lora_name in self.gate_up_proj_.loras_:
-                lora_data = _mixtral_slice_tensor(hidden_states, top_x, input_dtype)
                 gate_up_states = self.gate_up_proj_.loras_[lora_name].forward(
-                    _mixtral_slice_tensor(common_gate_up, top_x, input_dtype), lora_data
+                    _mixtral_slice_tensor(common_gate_up, top_x, input_dtype),
+                    _mixtral_slice_tensor(hidden_states, top_x, input_dtype),
                 )
             else:
-                lora_data = None
                 gate_up_states = _mixtral_slice_tensor(
                     common_gate_up, top_x, input_dtype
                 )
@@ -320,7 +319,7 @@ def _mixlora_forward(
 
             if lora_name in self.down_proj_.loras_:
                 final_expert_states.append(
-                    self.down_proj_.loras_[lora_name].forward(  # LoRA a,b
+                    self.down_proj_.loras_[lora_name].forward(
                         self.down_proj_.base_layer_.forward(act_result),
                         act_result,
                     )