books/LLMs-from-scratch
1
0
Fork 0
mirror of https://github.com/rasbt/LLMs-from-scratch.git synced 2026-04-10 12:33:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

rename to dataloader v1

Browse Source
This commit is contained in:
rasbt
2024-02-24 07:48:18 -06:00
parent 4e68649f16
commit 6243726ab3
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ch04/01_main-chapter-code/previous_chapters.py
View File

@@ -27,7 +27,7 @@ class GPTDatasetV1(Dataset):
return self.input_ids[idx], self.target_ids[idx] return self.input_ids[idx], self.target_ids[idx]
def create_dataloader(txt, batch_size=4, max_length=256, stride=128, shuffle=True): def create_dataloader_v1(txt, batch_size=4, max_length=256, stride=128, shuffle=True):
# Initialize the tokenizer # Initialize the tokenizer
tokenizer = tiktoken.get_encoding("gpt2") tokenizer = tiktoken.get_encoding("gpt2")