books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove redundant context_length in GQA

Browse Source
This commit is contained in:
rasbt
2025-03-31 16:49:10 -05:00
parent 06ebac3c34
commit 4715dc3be5
1 changed files with 1 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
ch05/07_gpt_to_llama/standalone-llama32-mem-opt.ipynb
View File

@@ -233,7 +233,7 @@
"source": [ "source": [
"class GroupedQueryAttention(nn.Module):\n", "class GroupedQueryAttention(nn.Module):\n",
" def __init__(\n", " def __init__(\n",
" self, d_in, d_out, context_length, num_heads,\n", " self, d_in, d_out, num_heads,\n",
" num_kv_groups,\n", " num_kv_groups,\n",
" dtype=None\n", " dtype=None\n",
" ):\n", " ):\n",
@@ -320,7 +320,6 @@
" self.att = GroupedQueryAttention(\n", " self.att = GroupedQueryAttention(\n",
" d_in=cfg[\"emb_dim\"],\n", " d_in=cfg[\"emb_dim\"],\n",
" d_out=cfg[\"emb_dim\"],\n", " d_out=cfg[\"emb_dim\"],\n",
" context_length=cfg[\"context_length\"],\n",
" num_heads=cfg[\"n_heads\"],\n", " num_heads=cfg[\"n_heads\"],\n",
" num_kv_groups=cfg[\"n_kv_groups\"],\n", " num_kv_groups=cfg[\"n_kv_groups\"],\n",
" dtype=cfg[\"dtype\"]\n", " dtype=cfg[\"dtype\"]\n",