books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rename drop_resid to drop_shortcut (#136)

Browse Source
This commit is contained in:
Sebastian Raschka
2024-04-28 14:31:27 -05:00
committed by GitHub
parent 70cd174091
commit 97ed38116a
10 changed files with 37 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
ch05/02_alternative_weight_loading/previous_chapters.py
View File

@@ -167,21 +167,21 @@ class TransformerBlock(nn.Module):
self.ff = FeedForward(cfg)
self.norm1 = LayerNorm(cfg["emb_dim"])
self.norm2 = LayerNorm(cfg["emb_dim"])
self.drop_resid = nn.Dropout(cfg["drop_rate"])
self.drop_shortcut = nn.Dropout(cfg["drop_rate"])
def forward(self, x):
# Shortcut connection for attention block
shortcut = x
x = self.norm1(x)
x = self.att(x) # Shape [batch_size, num_tokens, emb_size]
x = self.drop_resid(x)
x = self.drop_shortcut(x)
x = x + shortcut # Add the original input back
# Shortcut connection for feed-forward block
shortcut = x
x = self.norm2(x)
x = self.ff(x)
x = self.drop_resid(x)
x = self.drop_shortcut(x)
x = x + shortcut # Add the original input back
return x