books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed gqa qkv code comments (#660)

Browse Source
This commit is contained in:
Daniel Kleine
2025-06-13 15:21:28 +02:00
committed by GitHub
parent 7632eb018b
commit c2cfb47b1a
3 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/llms_from_scratch/llama3.py
View File

@@ -166,9 +166,9 @@ class GroupedQueryAttention(nn.Module):
values = values.view(b, num_tokens, self.num_kv_groups, self.head_dim)
# Transpose keys, values, and queries
keys = keys.transpose(1, 2) # Shape: (b, num_heads, num_tokens, head_dim)
values = values.transpose(1, 2) # Shape: (b, num_heads, num_tokens, head_dim)
queries = queries.transpose(1, 2) # Shape: (b, num_query_groups, num_tokens, head_dim)
keys = keys.transpose(1, 2) # Shape: (b, num_kv_groups, num_tokens, head_dim)
values = values.transpose(1, 2) # Shape: (b, num_kv_groups, num_tokens, head_dim)
queries = queries.transpose(1, 2) # Shape: (b, num_heads, num_tokens, head_dim)
# Apply RoPE
keys = apply_rope(keys, cos, sin)