自我出题 2.FFN SwiGLU ——附题目判题ipynb

class FFN(nn.Module): def __init__(self, model_dim, intermediate_dim): super().__init__() # TODO 1: 定义 self.w_up self.w_upnn.Linear(model_dim,intermediate_dim) # TODO 2: 定义 self.w_down self.w_downnn.Linear(intermediate_dim,model_dim) def forward(self, x): # TODO 3: 上投影并用 F.relu 激活 # TODO 4: 下投影并 return return self.w_down(F.relu(self.w_up(x)))class SwiGLUFFN(nn.Module): def __init__(self, model_dim, intermediate_dim): super().__init__() # TODO 1: self.w_gate ... (D - H, biasFalse) self.w_gatenn.Linear(model_dim, intermediate_dim, biasFalse) # TODO 2: self.w_up ... (D - H, biasFalse) self.w_upnn.Linear(model_dim, intermediate_dim, biasFalse) # TODO 3: self.w_down ... (H - D, biasFalse) self.w_downnn.Linear(intermediate_dim, model_dim, biasFalse) def forward(self, x): # TODO 4: gate F.silu(...) gateF.silu(self.w_gate(x)) # TODO 5: up ... upself.w_up(x) # TODO 6: return self.w_down(gate * up) return self.w_down(gate*up)