books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

small readability updates

Browse Source
This commit is contained in:
rasbt
2024-01-14 11:58:42 -06:00
parent c79499572f
commit a7b4880179
3 changed files with 26 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ch03/01_main-chapter-code/exercise-solutions.ipynb
View File

@@ -61,7 +61,7 @@
" values = x @ self.W_value\n",
" \n",
" attn_scores = queries @ keys.T # omega\n",
" attn_weights = torch.softmax(attn_scores / self.d_out**0.5, dim=-1)\n",
" attn_weights = torch.softmax(attn_scores / keys.shape[-1]**0.5, dim=-1)\n",
"\n",
" context_vec = attn_weights @ values\n",
" return context_vec\n",
@@ -92,7 +92,7 @@
" values = self.W_value(x)\n",
" \n",
" attn_scores = queries @ keys.T\n",
" attn_weights = torch.softmax(attn_scores / self.d_out**0.5, dim=1)\n",
" attn_weights = torch.softmax(attn_scores / keys.shape[-1]**0.5, dim=1)\n",
"\n",
" context_vec = attn_weights @ values\n",
" return context_vec\n",