From e6a689550c9e27622841b36fe3c9a19a45325b43 Mon Sep 17 00:00:00 2001
From: Souvik Bhattacharyya <78871654+souvikshanku@users.noreply.github.com>
Date: Sun, 27 Jul 2025 19:19:22 +0000
Subject: [PATCH] fix activation function

---
 modified_llama.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modified_llama.py b/modified_llama.py
index e1b395b..d647c6c 100644
--- a/modified_llama.py
+++ b/modified_llama.py
@@ -34,7 +34,7 @@ def forward(self, x):
         gate_proj = self.gate_proj.weight[:self.current_subset_hd]
         up_proj = self.up_proj.weight[:self.current_subset_hd]
         down_proj = self.down_proj.weight[:, :self.current_subset_hd]
-        down_proj = F.linear(self.act_fn(F.linear(x, gate_proj) * F.linear(x, up_proj)), down_proj)
+        down_proj = F.linear(self.act_fn(F.linear(x, gate_proj)) * F.linear(x, up_proj), down_proj)
         self.current_subset_hd = None
 
         return down_proj