books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rename drop_resid to drop_shortcut (#136)

Browse Source
This commit is contained in:
Sebastian Raschka
2024-04-28 14:31:27 -05:00
committed by GitHub
parent 70cd174091
commit 97ed38116a
10 changed files with 37 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
ch04/01_main-chapter-code/ch04.ipynb
View File

@@ -950,21 +950,21 @@
" self.ff = FeedForward(cfg)\n",
" self.norm1 = LayerNorm(cfg[\"emb_dim\"])\n",
" self.norm2 = LayerNorm(cfg[\"emb_dim\"])\n",
" self.drop_resid = nn.Dropout(cfg[\"drop_rate\"])\n",
" self.drop_shortcut = nn.Dropout(cfg[\"drop_rate\"])\n",
"\n",
" def forward(self, x):\n",
" # Shortcut connection for attention block\n",
" shortcut = x\n",
" x = self.norm1(x)\n",
" x = self.att(x) # Shape [batch_size, num_tokens, emb_size]\n",
" x = self.drop_resid(x)\n",
" x = self.drop_shortcut(x)\n",
" x = x + shortcut # Add the original input back\n",
"\n",
" # Shortcut connection for feed forward block\n",
" shortcut = x\n",
" x = self.norm2(x)\n",
" x = self.ff(x)\n",
" x = self.drop_resid(x)\n",
" x = self.drop_shortcut(x)\n",
" x = x + shortcut # Add the original input back\n",
"\n",
" return x"
@@ -1489,7 +1489,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.4"
"version": "3.10.6"
}
},
"nbformat": 4,