books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

use block size variable in positional embedding layer

Browse Source
This commit is contained in:
rasbt
2023-12-28 19:05:06 +01:00
parent 10aa40ba6a
commit 4f161bd549
7 changed files with 125 additions and 40106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
ch02/01_main-chapter-code/dataloader.ipynb
View File

@@ -20,7 +20,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 2,
"id": "0ed4b7db-3b47-4fd3-a4a6-5f4ed5dd166e",
"metadata": {},
"outputs": [],
@@ -74,8 +74,11 @@
"\n",
"vocab_size = 50257\n",
"output_dim = 256\n",
"block_size = 1024\n",
"\n",
"\n",
"token_embedding_layer = torch.nn.Embedding(vocab_size, output_dim)\n",
"pos_embedding_layer = torch.nn.Embedding(vocab_size, output_dim)\n",
"pos_embedding_layer = torch.nn.Embedding(block_size, output_dim)\n",
"\n",
"max_length = 4\n",
"dataloader = create_dataloader(raw_text, batch_size=8, max_length=max_length, stride=5)"
@@ -83,7 +86,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 3,
"id": "664397bc-6daa-4b88-90aa-e8fc1fbd5846",
"metadata": {},
"outputs": [],
@@ -101,7 +104,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 4,
"id": "d3664332-e6bb-447e-8b96-203aafde8b24",
"metadata": {},
"outputs": [