<|endoftext|> token in dataset v1

2026-04-10 12:33:42 +00:00 · 2024-01-21 12:03:04 -06:00
parent b1923a3075
commit 8860e16e05
3 changed files with 3 additions and 3 deletions
--- a/ch03/01_main-chapter-code/multihead-attention.ipynb
+++ b/ch03/01_main-chapter-code/multihead-attention.ipynb
@@ -46,7 +46,7 @@
    "        self.target_ids = []\n",
    "\n",
    "        # Tokenize the entire text\n",
-    "        token_ids = tokenizer.encode(txt)\n",
+    "        token_ids = tokenizer.encode(txt, allowed_special={'<|endoftext|>'})\n",
    "\n",
    "        # Use a sliding window to chunk the book into overlapping sequences of max_length\n",
    "        for i in range(0, len(token_ids) - max_length, stride):\n",