books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rename variable to context_length to make it easier on readers (#106)

Browse Source 
* rename to context length

* fix spacing
This commit is contained in:
Sebastian Raschka
2024-04-04 07:27:41 -05:00
committed by GitHub
parent a940373a14
commit 2de60d1bfb
25 changed files with 242 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
ch04/01_main-chapter-code/exercise-solutions.ipynb
View File

@@ -34,11 +34,11 @@
"metadata": {},
"outputs": [],
"source": [
"from gpt import TransformerBlock\n",
"from gpt import Transfocontext_lengthmerBlock\n",
"\n",
"GPT_CONFIG_124M = {\n",
" \"vocab_size\": 50257,\n",
" \"ctx_len\": 1024,\n",
" \"context_length\": 1024,\n",
" \"emb_dim\": 768,\n",
" \"n_heads\": 12,\n",
" \"n_layers\": 12,\n",
@@ -139,7 +139,7 @@
"source": [
"GPT_CONFIG_124M = {\n",
" \"vocab_size\": 50257,\n",
" \"ctx_len\": 1024,\n",
" \"context_length\": 1024,\n",
" \"emb_dim\": 768,\n",
" \"n_heads\": 12,\n",
" \"n_layers\": 12,\n",
@@ -260,7 +260,7 @@
"source": [
"GPT_CONFIG_124M = {\n",
" \"vocab_size\": 50257,\n",
" \"ctx_len\": 1024,\n",
" \"context_length\": 1024,\n",
" \"emb_dim\": 768,\n",
" \"n_heads\": 12,\n",
" \"n_layers\": 12,\n",
@@ -288,7 +288,7 @@
" self.att = MultiHeadAttention(\n",
" d_in=cfg[\"emb_dim\"],\n",
" d_out=cfg[\"emb_dim\"],\n",
" block_size=cfg[\"ctx_len\"],\n",
" context_length=cfg[\"context_length\"],\n",
" num_heads=cfg[\"n_heads\"], \n",
" dropout=cfg[\"drop_rate_attn\"], # NEW: dropout for multi-head attention\n",
" qkv_bias=cfg[\"qkv_bias\"])\n",
@@ -319,7 +319,7 @@
" def __init__(self, cfg):\n",
" super().__init__()\n",
" self.tok_emb = nn.Embedding(cfg[\"vocab_size\"], cfg[\"emb_dim\"])\n",
" self.pos_emb = nn.Embedding(cfg[\"ctx_len\"], cfg[\"emb_dim\"])\n",
" self.pos_emb = nn.Embedding(cfg[\"context_length\"], cfg[\"emb_dim\"])\n",
" self.drop_emb = nn.Dropout(cfg[\"drop_rate_emb\"]) # NEW: dropout for embedding layers\n",
"\n",
" self.trf_blocks = nn.Sequential(\n",
@@ -370,7 +370,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.4"
"version": "3.10.6"
}
},
"nbformat": 4,