books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed gqa qkv code comments (#660)

Browse Source
This commit is contained in:
Daniel Kleine
2025-06-13 15:21:28 +02:00
committed by GitHub
parent 7632eb018b
commit c2cfb47b1a
3 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
ch05/07_gpt_to_llama/converting-llama2-to-llama3.ipynb
View File

@@ -501,9 +501,9 @@
" ################################################\n",
"\n",
" # Transpose keys, values, and queries\n",
" keys = keys.transpose(1, 2) # Shape: (b, num_heads, num_tokens, head_dim)\n",
" values = values.transpose(1, 2) # Shape: (b, num_heads, num_tokens, head_dim)\n",
" queries = queries.transpose(1, 2) # Shape: (b, num_query_groups, num_tokens, head_dim)\n",
" keys = keys.transpose(1, 2) # Shape: (b, num_kv_groups, num_tokens, head_dim)\n",
" values = values.transpose(1, 2) # Shape: (b, num_kv_groups, num_tokens, head_dim)\n",
" queries = queries.transpose(1, 2) # Shape: (b, num_heads, num_tokens, head_dim)\n",
"\n",
" ##################### NEW #####################\n",
" # Apply RoPE\n",

6
ch05/07_gpt_to_llama/standalone-llama32.ipynb
View File

@@ -257,9 +257,9 @@
" values = values.view(b, num_tokens, self.num_kv_groups, self.head_dim)\n",
"\n",
" # Transpose keys, values, and queries\n",
" keys = keys.transpose(1, 2) # Shape: (b, num_heads, num_tokens, head_dim)\n",
" values = values.transpose(1, 2) # Shape: (b, num_heads, num_tokens, head_dim)\n",
" queries = queries.transpose(1, 2) # Shape: (b, num_query_groups, num_tokens, head_dim)\n",
" keys = keys.transpose(1, 2) # Shape: (b, num_kv_groups, num_tokens, head_dim)\n",
" values = values.transpose(1, 2) # Shape: (b, num_kv_groups, num_tokens, head_dim)\n",
" queries = queries.transpose(1, 2) # Shape: (b, num_heads, num_tokens, head_dim)\n",
"\n",
" # Apply RoPE\n",
" keys = apply_rope(keys, cos, sin)\n",