books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove leftover instances of self.tokenizer (#201)

Browse Source 
* Remove leftover instances of self.tokenizer

* add endoftext token
This commit is contained in:
Sebastian Raschka
2024-06-08 14:57:34 -05:00
committed by GitHub
parent 98d23751f7
commit 40ba3a4068
13 changed files with 18 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
ch06/01_main-chapter-code/previous_chapters.py
View File

@@ -20,12 +20,11 @@ from torch.utils.data import Dataset, DataLoader
class GPTDatasetV1(Dataset):
def __init__(self, txt, tokenizer, max_length, stride):
self.tokenizer = tokenizer
self.input_ids = []
self.target_ids = []
# Tokenize the entire text
token_ids = tokenizer.encode(txt)
token_ids = tokenizer.encode(txt, allowed_special={"<|endoftext|>"})
# Use a sliding window to chunk the book into overlapping sequences of max_length
for i in range(0, len(token_ids) - max_length, stride):

3
ch06/02_bonus_additional-experiments/previous_chapters.py
View File

@@ -20,12 +20,11 @@ from torch.utils.data import Dataset, DataLoader
class GPTDatasetV1(Dataset):
def __init__(self, txt, tokenizer, max_length, stride):
self.tokenizer = tokenizer
self.input_ids = []
self.target_ids = []
# Tokenize the entire text
token_ids = tokenizer.encode(txt)
token_ids = tokenizer.encode(txt, allowed_special={"<|endoftext|>"})
# Use a sliding window to chunk the book into overlapping sequences of max_length
for i in range(0, len(token_ids) - max_length, stride):

2
ch06/03_bonus_imdb-classification/previous_chapters.py
View File

@@ -25,7 +25,7 @@ class GPTDatasetV1(Dataset):
self.target_ids = []
# Tokenize the entire text
token_ids = tokenizer.encode(txt)
token_ids = tokenizer.encode(txt, allowed_special={"<|endoftext|>"})
# Use a sliding window to chunk the book into overlapping sequences of max_length
for i in range(0, len(token_ids) - max_length, stride):